diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 96f4268b..a11e0f8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,11 +32,11 @@ # # ── What ships ──────────────────────────────────────────────────────────────── # -# One build variant per platform/arch: lightning + rex-jit + camera + chd. +# One build variant per platform/arch: lightning + gpujit + camera + chd. # # It is a lightning build (interpreter with breakpoint checks and the traceback -# buffer stripped — no interactive debugging) and carries the REX3 draw-shader -# JIT. It does not carry the experimental MIPS JIT (`jitv2`) or the N64 dev +# buffer stripped — no interactive debugging) and carries the graphics JITs +# (`gpujit`: REX3 draw shaders, IMPACT raster pipelines). It does not carry the experimental MIPS JIT (`jitv2`) or the N64 dev # board (`ultra64`); those are source-build opt-ins (see iris-gui/Cargo.toml). # (The v1 MIPS JIT, `jit`, was removed from the tree in 33c4e68.) The # former "standard" (non-lightning), `pcap`, `-jitv2`, and separate `-r5000` @@ -214,11 +214,11 @@ jobs: - target: x86_64-pc-windows-msvc arch: x64 runs-on: windows-latest - cli_features: "rex-jit,lightning" + cli_features: "gpujit,lightning" - target: aarch64-pc-windows-msvc arch: arm64 runs-on: windows-11-arm - cli_features: "rex-jit,lightning" + cli_features: "gpujit,lightning" steps: - uses: actions/checkout@v5 with: @@ -383,8 +383,8 @@ jobs: - name: Build all binaries run: | cargo build --release --target ${{ matrix.target }} -p iris-gui --features iris/lightning,bundled - cargo build --release --bin iris --target ${{ matrix.target }} --features rex-jit,lightning - cargo build --release --bin iris-ci --target ${{ matrix.target }} --features rex-jit,lightning + cargo build --release --bin iris --target ${{ matrix.target }} --features gpujit,lightning + cargo build --release --bin iris-ci --target ${{ matrix.target }} --features gpujit,lightning # ── Code signing setup ────────────────────────────────────────────────── @@ -522,7 +522,7 @@ jobs: if [ "$IDENTITY" != "-" ]; then # See the GUI bundle signing above: the JIT entitlement keeps the - # notarized CLI binaries (built with rex-jit) from being killed by + # notarized CLI binaries (built with gpujit) from being killed by # the hardened runtime on their first JITed REX3 draw. codesign --force --options runtime --timestamp \ --entitlements installer/iris-gui-notarized.entitlements \ @@ -808,8 +808,8 @@ jobs: - name: Build iris CLI and iris-ci run: | - cargo build --release --bin iris --target ${{ matrix.target }} --features rex-jit,lightning - cargo build --release --bin iris-ci --target ${{ matrix.target }} --features rex-jit,lightning + cargo build --release --bin iris --target ${{ matrix.target }} --features gpujit,lightning + cargo build --release --bin iris-ci --target ${{ matrix.target }} --features gpujit,lightning - name: Package Core tarball run: | @@ -935,8 +935,8 @@ jobs: - name: Build iris CLI + iris-ci (cross) run: | set -eux - cargo build --release --bin iris --target "$TARGET" --features rex-jit,lightning - cargo build --release --bin iris-ci --target "$TARGET" --features rex-jit,lightning + cargo build --release --bin iris --target "$TARGET" --features gpujit,lightning + cargo build --release --bin iris-ci --target "$TARGET" --features gpujit,lightning # The release profile keeps debug info (debug = 1) and the host `strip` # can't read a riscv64 ELF, so strip iris-gui with the cross strip here and diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d356875..6e142bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,8 +42,11 @@ hashes are given where a change is easiest to understand by reading the commit. ### Build / features -- **`gpujit` turns on every graphics JIT** (`rex-jit` and `gr4-jit`): - `--features lightning,gpujit`. +- **`gpujit` turns on every graphics JIT** (`rex-jit` and `gr4-jit`) and is + the recommended build: `--features lightning,gpujit`. The GUI and the + release binaries now carry it, so IMPACT gets its raster JIT out of the + box. `IRIS_NO_JIT` now turns the IMPACT JIT off too (and `mgras jit on` + refuses), which keeps the Mac App Store build interpreter-only. - **`gr2 trace` and `mgras trace` need `gputrace`, like `rex trace`.** On with `developer`; plain and `lightning` builds add `--features gputrace`. Without it the per-access trace check on GR2 and IMPACT register reads diff --git a/CLAUDE.md b/CLAUDE.md index d1d98ba4..a1878ec2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,8 +28,8 @@ system (shell, networking, X11). It is **not** cycle-accurate ``` cargo run --release # interpreter -cargo run --release --features lightning,rex-jit # recommended for speed -cargo run --release --features jitv2,rex-jit # enable MIPS JIT v2 (experimental) +cargo run --release --features lightning,gpujit # recommended for speed (gpujit = rex-jit,gr4-jit) +cargo run --release --features jitv2,gpujit # enable MIPS JIT v2 (experimental) cargo run --release -- --cpu r5000 # CPU is a runtime choice, not a feature cargo run -p iris-gui --release # GUI front-end ``` diff --git a/Cargo.toml b/Cargo.toml index 15fa74a3..b07325e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,6 +157,17 @@ tcache = [] # Panics at the exact diverging access instead of letting corruption surface as # a guest crash much later. Very slow; diagnostic only. tcache_verify = ["tcache"] +# Cache parity/ECC injection and the Cache Error exception, for SGI's IDE +# cache diagnostics (dcache1, icache1, scache1): Status.CE forcing the ECC +# register into stored check bits, Index_Load_Tag loading check bits into CP0 +# ECC, and the Cache Error a mismatch raises. Nothing but a diagnostic ever +# observes any of it, and it costs a check on every D-cache miss and a Status +# sync on every exception, so it is off by default. +cache-ecc = [] +# Decoded-instruction cache for the shadow cache (src/cpu/decode_cache.rs). +# Always built now; this flag does nothing and is kept so command lines that +# still pass `--features decode-cache` keep working. +decode-cache = [] default = ["tlbvmap", "rexdiag"] # Cranelift-based JIT compiler for REX3 graphics draw shaders. rex-jit = ["cranelift-codegen", "cranelift-frontend", "cranelift-jit", "cranelift-module", "cranelift-native"] diff --git a/FEATURES.md b/FEATURES.md index 7df4cae6..9e376009 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -41,7 +41,7 @@ Select with `cargo build --release --features `; add | `default` | Default bundle: `tlbvmap` and `rexdiag`. | `tlbvmap`, `rexdiag` | | `rex-jit` | Cranelift compiler for Newport draw shaders; complements precompiled shaders. | `cranelift-codegen`, `cranelift-frontend`, `cranelift-jit`, `cranelift-module`, `cranelift-native` | | `gr4-jit` | Cranelift compiler for IMPACT (GR4) raster pipelines (RSS/TE1 shaders). | `cranelift-codegen`, `cranelift-frontend`, `cranelift-jit`, `cranelift-module`, `cranelift-native` | -| `gpujit` | Every graphics JIT, so one name covers both. | `rex-jit`, `gr4-jit` | +| `gpujit` | Every graphics JIT; the recommended build. Hosts that forbid JIT code (the Mac App Store sandbox) set `IRIS_NO_JIT` at run time. | `rex-jit`, `gr4-jit` | | `jitv2` | Experimental whole-page MIPS compiler; implies transparent caching. | `tcache`, `cranelift-codegen`, `cranelift-frontend`, `cranelift-jit`, `cranelift-module`, `cranelift-native`, `target-lexicon`, `region`, `wasmtime-jit-icache-coherence` | | `jitv2_opcodefusion` | Optional JIT fusion, off by default after live delay-slot failures. | `jitv2` | | `j2wp` | Compatibility alias for `jitv2`; no separate compiler mode. | `jitv2` | @@ -53,15 +53,15 @@ Select with `cargo build --release --features `; add Cargo also exposes these **implicit optional-dependency features**: `cranelift-codegen`, `cranelift-frontend`, `cranelift-jit`, `cranelift-module`, `cranelift-native`, `region`, `target-lexicon`, `wasmtime-jit-icache-coherence`. They enable library dependencies, -not complete emulator capabilities; use `jitv2` or `rex-jit` for a working +not complete emulator capabilities; use `jitv2` or `gpujit` for a working compiler configuration. `dep:` entries above suppress implicit feature names for PCAP/host-service dependencies. ### iris-gui Source: [iris-gui/Cargo.toml](iris-gui/Cargo.toml). There is no GUI-specific -default feature bundle. Its `iris` dependency always enables `rex-jit` and -retains the core default features. GUI `--no-default-features` alone does not +default feature bundle. Its `iris` dependency always enables `gpujit` (the +App Store build turns the JITs off at run time with `IRIS_NO_JIT`) and retains the core default features. GUI `--no-default-features` alone does not turn off those dependency defaults. | GUI feature | Purpose | Enables | @@ -443,7 +443,7 @@ and offline-tool controls are included so they are not mistaken for GUI settings | `IRIS_NO_FETCH_VERIFY` | 1 suppresses fetchverify checks in a fetchverify build. | [src/cpu/mips_exec.rs](src/cpu/mips_exec.rs) | | `IRIS_NO_IDLE` | Any presence disables idle parking. | [src/cpu/idle_park.rs](src/cpu/idle_park.rs) | | `IRIS_NO_INLINE_MEM` | Any presence disables JIT inline memory emission. | [src/cpu/jitv2/codegen.rs](src/cpu/jitv2/codegen.rs) | -| `IRIS_NO_JIT` | Any presence disables the REX3 shader JIT; does not select the MIPS engine. | [src/dev/ng1/rex3.rs](src/dev/ng1/rex3.rs) | +| `IRIS_NO_JIT` | Any presence disables the REX3 shader JIT and the IMPACT raster JIT; does not select the MIPS engine. | [src/dev/ng1/rex3.rs](src/dev/ng1/rex3.rs) | | `IRIS_OPT_SPEED` | Presence selects speed optimization in offline PCP/corpus measurements. | [src/bin/jitv2_pcp_dump.rs](src/bin/jitv2_pcp_dump.rs) | | `IRIS_PRINT_OFFSETS` | Presence runs the opt-in layout-offset print test. | [src/cpu/jitv2/mod.rs](src/cpu/jitv2/mod.rs) | | `IRIS_QOS` | HostGL QoS: default/off disables promotion; quiet suppresses logging. | [iris-hostgl/src/qos.rs](iris-hostgl/src/qos.rs) | @@ -471,10 +471,10 @@ both crates; these examples and implementation notes explain the tradeoffs. Build variants: ``` -cargo run --release --features lightning,rex-jit # recommended for best speed +cargo run --release --features lightning,gpujit # recommended for best speed cargo run --release --features lightning # disable emulator breakpoints for a little bit more speed -cargo run --release --features rex-jit # enable REX3 graphics JIT compiler -cargo run --release --features jitv2,rex-jit # MIPS JIT v2 (experimental; see "JIT compilers") +cargo run --release --features gpujit # enable the graphics JITs (rex-jit + gr4-jit) +cargo run --release --features jitv2,gpujit # MIPS JIT v2 (experimental; see "JIT compilers") cargo run --release --features idle-pause # park the CPU thread while the guest idles instead of spinning a host core cargo run --release --features ci_clock # synthetic deterministic CP0 Count clock (CI/snapshot validator only; loses realtime desktop timing) cargo run --release --features pcap # bridge guest networking onto a real host interface via libpcap instead of the built-in NAT gateway. See [network] in iris.toml. @@ -575,7 +575,7 @@ JIT v2 section for tuning. (The original speculative, tiered MIPS JIT was removed in August 2026; jitv2 replaces it.) ``` -cargo run --release --features jitv2,rex-jit +cargo run --release --features jitv2,gpujit ``` What it does today: compiles on a pool of background threads (`[jitv2] diff --git a/HELP.md b/HELP.md index 37ffd1ec..c813c4ef 100644 --- a/HELP.md +++ b/HELP.md @@ -35,7 +35,7 @@ emulating the Indy. The **iris-gui status bar MIPS** figure (or the CLI window title `MIPS` readout) is **real throughput**: MIPS instructions executed per wall-clock second on the -host. `lightning` builds and the JIT stack (`--features jitv2,rex-jit`) give +host. `lightning` builds and the JIT stack (`--features jitv2,gpujit`) give higher MIPS; the hinv MHz string stays the same. The guest's MHz comes from the CP0 Count rate, which is fixed: Count ticks at diff --git a/README.md b/README.md index 85802e35..a4e02b64 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ boots to a usable system: shell, networking, X11, the works. - Cranelift JIT compiler for MIPS to host code (`jitv2`, optional, experimental), plus a REX3 draw pipeline of 400+ precompiled specialised draw functions and an optional REX3 shader JIT (`rex-jit`), and an IMPACT raster pipeline JIT - (`gr4-jit`) + (`gr4-jit`); `gpujit` turns on both - Copy-on-write disk overlay, and CHD images with MAME-style `.diff.chd` sidecars. Crash all day, base image stays clean - Hot-swappable CD-ROM with runtime disc switching @@ -181,7 +181,8 @@ are in [FEATURES.md](FEATURES.md#emulated-cpu). Optional `jitv2` compiles MIPS code; `rex-jit` compiles Newport draw shaders alongside the precompiled drawing routines; `gr4-jit` compiles IMPACT raster -pipelines. See +pipelines. `gpujit` turns on both graphics JITs and is the recommended +build (`--features lightning,gpujit`). See [FEATURES.md](FEATURES.md#jit-compilers) for their behavior, defaults, limitations, persistent caches, and corpus-measurement tools. diff --git a/iris-gui-README.md b/iris-gui-README.md index e6c8e809..ecc9da47 100644 --- a/iris-gui-README.md +++ b/iris-gui-README.md @@ -28,7 +28,7 @@ cargo run -p iris-gui --release The first build compiles libchdman-rs, the camera backends (nokhwa / V4L), and Cranelift. CHD and camera support are unconditional in the core; iris-gui -also enables `rex-jit`. Device use is configured at runtime. +also enables `gpujit` (the REX3 and IMPACT graphics JITs). Device use is configured at runtime. A debug build (`cargo run -p iris-gui` without `--release`) is fine for iterating on the GUI itself but uses an unoptimized iris core, which means @@ -338,7 +338,7 @@ iris/ ├── Cargo.toml [workspace] iris, iris-gui, iris-hostcall; iris-hostgl excluded ├── src/ iris library + CLI └── iris-gui/ - ├── Cargo.toml depends on iris with rex-jit; CHD/camera built into core + ├── Cargo.toml depends on iris with gpujit; CHD/camera built into core ├── build.rs APP_VERSION (from RELEASE_VERSION or the crate version) ├── assets/ icons, default NVRAM └── src/ diff --git a/iris-gui/Cargo.toml b/iris-gui/Cargo.toml index 908bbae9..9e3f269f 100644 --- a/iris-gui/Cargo.toml +++ b/iris-gui/Cargo.toml @@ -84,10 +84,12 @@ hostcall = ["iris/hostcall"] hostgl = ["hostcall", "iris/hostgl"] [dependencies] -# The GUI always enables REX3 shader compilation. CHD, camera, DaynaPort, -# and Ultra64 device support are unconditional in the core and selected at -# runtime; Ultra64 requires an external gopher64 bridge peer. -iris = { path = "..", features = ["rex-jit"] } +# The GUI always enables the graphics JITs (`gpujit`: REX3 draw shaders and +# IMPACT raster pipelines); the `appstore` build turns both off at run time +# with IRIS_NO_JIT. CHD, camera, DaynaPort, and Ultra64 device support are +# unconditional in the core and selected at runtime; Ultra64 requires an +# external gopher64 bridge peer. +iris = { path = "..", features = ["gpujit"] } eframe = { version = "0.36", default-features = false, features = ["default_fonts", "glow", "wayland", "x11"] } egui = "0.36" crossbeam-channel = "0.5" diff --git a/iris-gui/src/main.rs b/iris-gui/src/main.rs index 0ee12ba0..f1a06d86 100644 --- a/iris-gui/src/main.rs +++ b/iris-gui/src/main.rs @@ -146,7 +146,7 @@ fn main() -> eframe::Result<()> { std::env::set_var("IRIS_CHD_DIFF_DIR", d.join("iris").join("chd-diffs")); } // Force the interpreter on the Mac App Store build. Cranelift (MIPS JIT and - // the always-on REX3 draw-shader JIT) allocates executable memory with + // the always-on REX3 draw-shader and IMPACT raster JITs) allocates executable memory with // mmap+mprotect, not MAP_JIT. The App Sandbox only permits MAP_JIT pages // (com.apple.security.cs.allow-jit is the sole code-signing entitlement MAS // allows; allow-unsigned-executable-memory / disable-executable-page- diff --git a/profile.sh b/profile.sh index 3d66a3ee..5da13a60 100755 --- a/profile.sh +++ b/profile.sh @@ -1,2 +1,2 @@ #!/bin/bash -PERFFLAGS="-F 200 -g --call-graph dwarf" cargo flamegraph --profile profiling --features rex-jit,lightning,jitv2 --bin iris \ No newline at end of file +PERFFLAGS="-F 200 -g --call-graph dwarf" cargo flamegraph --profile profiling --features gpujit,lightning,jitv2 --bin iris \ No newline at end of file diff --git a/rules/perf/hot-path-inlining-must-be-explicit.md b/rules/perf/hot-path-inlining-must-be-explicit.md new file mode 100644 index 00000000..6cdc7034 --- /dev/null +++ b/rules/perf/hot-path-inlining-must-be-explicit.md @@ -0,0 +1,26 @@ +# Hot-path inlining must be stated, not left to the inliner + +`CpuCache::fetch` runs once per interpreted instruction. It used to inline into +`step_int` because the inliner happened to judge it small enough, with its miss +path `fill_l1i_line` inlined into it. + +1431f560 added a 4-word `ic_raw` snapshot to `fill_l1i_line`, a cold path. That +grew `fetch` past the inliner's budget, and `fetch` became an out-of-line call +on every instruction. Bare-metal iris-bench (lightning, R4400, full cache) fell +to ×0.84 of `main` (`sys/uncached` ×0.51, `int/branch` ×0.75). IRIX dhrystone +went 71428 → 58823. + +Found by bisecting the bare-metal suite, then confirmed with +`nm -C --size-sort … | grep 'CpuCache<16384.*fetch'`: the symbol exists only +in the slow build. + +Rules: +- A per-instruction hit path gets `#[inline(always)]` (`CpuCache::fetch`, + `ShadowCache::fetch`, `DecodeCache::fetch`). +- Its miss path gets `#[inline(never)] #[cold]` (`fill_l1i_line`, + `DecodeCache::fetch_slow`), so changes there cannot move the hot path. +- After touching anything reachable from `fetch`, `read`, `write` or + `step_int`, check the symbol table: a new out-of-line symbol for a hot + function is the regression. + +Fixed in 28d8c01a (back to ×1.02 of `main`). diff --git a/rules/perf/shadow-cache-r4400-r5000.md b/rules/perf/shadow-cache-r4400-r5000.md new file mode 100644 index 00000000..e67e0800 --- /dev/null +++ b/rules/perf/shadow-cache-r4400-r5000.md @@ -0,0 +1,100 @@ +# Shadow cache for R4400/R5000: what it is for, what it costs + +`ShadowCache` (src/cpu/mips_cache_shadow.rs) keeps no line data: memory is the +only store, and its tag arrays exist only to answer CACHE instructions. It now +covers the R4400 and R5000 as well as the R10000, selected by `MODEL` +(`mips_cache_v2::model`, as for `CpuCache`). + +Selected at machine build: `[machine] cache = "auto" | "full" | "shadow"`, or +`--cache` (also `iris-bench run --cache`). `auto` means shadow in a jitv2 +build, full otherwise. The R10000 only has the shadow model. + +## Why jitv2, and not the interpreter + +Measured 2026-10-12, full-length `iris-bench run`, `lightning,jitv2`, shadow vs +full. Accuracy identical on all 46 kernels. + +| | geomean | biggest wins | losses | +|---|---|---|---| +| R4400 | ×1.33 | mem/copy ×10.8, stream_copy ×8.0, tlb_hit ×2.8 | llsc ×0.63, cache_flush ×0.63, exception ×0.73, tlb_miss ×0.77 | +| R5000 | ×1.50 | adler32 ×21.6, stream_scale ×13.4, tlb_hit ×2.8 | llsc ×0.51, tlb_miss ×0.71, cache_flush ×0.75 | + +Compiled loads and stores against a shadow cache need no tag probe (the +tagless inline path, `JitDcGeometry::tagless`). + +The losers are the kernels that spend their time in the interpreter +(exceptions, TLB refills, LL/SC, CACHE ops). `ShadowCache::fetch` decodes +into one scratch slot, so the interpreter re-decodes every instruction; the +full model keeps decoded instructions in its lines. On a pure interpreter +build the shadow is about ×0.55–0.6 (measured 2026-10-11, `--quick`), which is +why `auto` keeps the full cache there. + +`mem/latency_dram` and `mem/random` (×0.7) did not obviously fit that pattern; +both recovered once the shadow read RAM through the ppmem window (below). + +## Decode cache and ppmem window (always on) + +`src/cpu/decode_cache.rs`: decoded instructions by physical page, plugged into +`ShadowCache::fetch` only, never touching the shadow's tag arrays. The shadow +also reads and writes RAM through ppmem's window in every build (the window +hand-over is no longer gated on `tcache`). PROM, MMIO and the low-512KB alias +are not RAM in ppmem's bitmap and go through the bus. + +Shadow vs full, measured 2026-10-12 (interpreter: `--quick`, best of 2; +jitv2: full-length): + +| step | interp R4400 | interp R5000 | +|---|---|---| +| shadow alone | ×0.71 | ×0.53 | +| + decode cache, bus reads | ×0.77 | ×0.57 | +| + ppmem window | ×1.03 | ×0.72 | +| + inlined fetch fast path | **×1.36** | **×1.03** | + +jitv2, shadow + both: **×1.52** on both CPUs; accuracy identical. + +Lessons, each measured: +- **Decode was not the main cost.** Bus reads were: every raw-compare fetch + and every data access through the dynamic `BusDevice` call. Profile first. +- **Validate by compare, not by jitv2's write-generation counters.** Through + the window a compare is one host load, and it measured equal or faster + (raw/gen ×1.05 R4400, ×1.00 R5000). It also works without jitv2 and outside + RAM. +- **Inline the hit path, keep the fill cold.** `ShadowCache::fetch` was a call + taking 34% of R5000 interpreter time; `#[inline(always)]` on a small hit + path, with `#[inline(never)] #[cold]` on the rest, closed most of the gap. +- **Each decoded page keeps a pointer to its window page** (null for bus + pages), taken once at page fill; the cache records the region bitmap it was + filled under and starts over when it changes (remap). + +Still lagging: `sys/tlb_miss` (×0.66–0.76 jitv2, ×0.73 R5000 interp). + +## What software sees + +The same as the full model, by construction and by test: +- `shadow_matches_the_full_models`: geometry, PRId/FIR, ISA, TLB, name. +- `tags_read_back_as_the_full_model_does`: 20,000 random Index_Store_Tag / + Index_Load_Tag pairs per CPU must return what the full model returns. Both + go through `r4k_store_taglo` / `r4k_load_taglo`. +- Index ops pick their line the way `CpuCache` does: R4400 direct-mapped, R5000 + way = VA[14], R10000 way = VA[0] (`tag_slot`). + +cpu-tests: shadow loses only `cache/hit_inv_discards` (no line data to +discard), the trade jitv2's tcache already made. + +## Snapshots + +The manifest records `cpu_cache` ("full" | "shadow"). A mismatch is fatal +(`cache_model_mismatch`): a full cache's state holds line data and dirty lines +a shadow has nowhere to put. Legacy manifests mean the old default: full for +R4400/R5000, shadow for the R10000. + +## Cost + +Two more monomorphised CPUs: `.text` +323 KB (+4.3%, `lightning` iris-bench). + +## Seen along the way, not yet chased + +- Plain R4400 with the full cache runs 256 of 262 cpu-tests (6 `cache/` tests + skipped); with the shadow, all 262. +- `umode/load_miss_third` fails with the full cache on both engines and passes + with the shadow. That suggests a full-model bug. diff --git a/rules/testing/cache-tag-index-and-check-bits.md b/rules/testing/cache-tag-index-and-check-bits.md new file mode 100644 index 00000000..7698bf78 --- /dev/null +++ b/rules/testing/cache-tag-index-and-check-bits.md @@ -0,0 +1,114 @@ +# Cache tags: index ops index by address; check bits are generated, not stored + +Both from `docs/R4000_um2.pdf` and `docs/VR5000 User Manual.pdf`, now in the tree. + +## Index_Store_Tag(SD) indexes by the CACHE address + +An index op takes its line from the CACHE instruction's own address (R4000 UM +A-42: `pAddr[CACHEBITS:LINEBITS]` for the secondary; VR5000 UM §3.4 the same). +MAME's `r4000.cpp` agrees. + +`mips_exec` passes TagHi:TagLo in `cache_op`'s `phys_addr` argument for C_IST, +and the functional model computed the L2 index from `phys_addr` for every op. +So a secondary `Index_Store_Tag` wrote whichever set TagLo[19:7] named. Fixed +in `CpuCache::cache_op`; regression test +`l2_index_store_tag_indexes_by_address_not_by_taglo`. + +It went unnoticed because: + +- PROM/IRIX cache init stores TagLo = 0, so every store hit set 0, and the + other sets were already invalid from power-on. +- `cache_op_ilt_ist_l2` stores back the tag it just read. The store goes to the + wrong set, and the read-back finds the untouched original: a round trip that + passes with the bug. Same trap as `r5000-taglo-is-the-r4400-pstate.md`. + +## P and tag ECC are computed on Index_Load_Tag + +"The P and ECC fields of these registers are ignored on Index Store Tag +operations. Parity and ECC are computed by the store operation." (R4000 UM p. 93, +VR5000 UM §6.5.10.) Appendix A's wording ("the TagLo register in the case of +the secondary cache") reads the other way, but p. 93 and Chapter 16 ("the +processor generates check bits for the tag when it is written") both say +computed. + +Since they are a pure function of the stored fields, the model computes them +when the tag is read back and stores nothing: + +| | TagLo[0] P | covers | +|---|---|---| +| R4400 L1I | even parity | PTag, V (Fig. 11-3) | +| R4400 L1D | even parity | PTag, CS (Fig. 11-4) | +| R5000 L1I/L1D | even parity | PTag only (Fig. 12-3/12-4) | +| R4400 L2 | TagLo[6:0] ECC | CS:PIdx:STag via Fig. 16-4 (`l2_tag_ecc`) | + +The Fig. 16-4 transcription is pinned by `l2_tag_ecc_matrix_is_the_manuals` +(row weights, weight-3 distinct syndromes). The text extraction of that figure +is garbled; render the page (`pdftoppm -f 455 -l 455`) to read it. + +Not modelled: the CP0 ECC register load on Index_Load_Tag (data parity/ECC of +the addressed doubleword), and planted bad check bits (CE bit). + +## Hit_Writeback(I) writes the I-cache's own copy back + +Op 6 on the I-cache writes the line back unconditionally (both manuals). IDE +`icache2` (`icache_fill_writeback`) fills the I-cache, changes memory under it +through the D-cache, and expects the write-back to restore the I-cache's +words. It was a no-op here, so every word failed with `Actual:0xffffffff`. +(The `Expect:` value IDE prints there is an uninitialised variable.) + +The I-cache has to keep what it was filled with: +- **R5000:** `ic_instrs` already is that copy. +- **R4400:** fetch is served from the L2 decode slots, which follow L2 data + after the fill. `ic_raw` snapshots the words at fill time, for the write-back + only. + +Under tcache a cached store goes straight to RAM, so the fill already sees the +new words, and this test cannot pass (same trade as `cache/hit_inv_discards`). + +Snapshots carry I-cache tags but not the words. `load_cache_state` now +rebuilds them (`rebuild_l1i_words`). Before, a restored valid R5000 line was +served from empty slots (zeros, i.e. NOPs) until it was evicted. + +## Parity/ECC injection (`cache-ecc` feature) + +IDE `dcache1`, `icache1` and `scache1` plant bad check bits and expect a Cache +Error. That machinery is behind the `cache-ecc` Cargo feature, off by +default: nothing but a diagnostic ever observes it. With it, every IP22 IDE +(field, 6.5) cache test passes on R4400 and R5000: icache1, icache2, dcache1, +scache1. + +- **Cache Error:** `EXEC_CACHE_ERR` (ExecStatus bit 30). Delivery sets + ErrorEPC and ERL and jumps to `0xA0000100`, or `0xBFC00300` with BEV; Cause, + EPC, EXL and BadVAddr are untouched. CacheErr comes from + `MipsCache::take_cache_err`. +- **Check bits are stored only when forced.** Otherwise they always match the + data, so they are computed on demand. D-cache: per doubleword. L2: + per-doubleword XOR in `ParityState::l2_xor`. I-cache: per line. +- **Zero hit-path cost:** a line that needs checking carries `L1DTag::SLOW` + (ptag bit 1), so it fails the exact compare in `matches_phys`, in the + interpreter and in jitv2's inline guard alike. The miss path resolves it and + returns `L1D_SLOW` (2) inside the existing `r > 1` branch. Cold paths that + must find a marked line use `matches_line`. +- **CE semantics** (R4000 UM ch. 5, VR5000 UM 7.2.9): + - a store takes the ECC register as the check bits for the bytes it writes; + - Fill(I) takes it for the whole line; + - a D-cache write-back by a CACHE op XORs it into the L2 ECC of the + doubleword that op addresses. +- **Detection:** + - a D-cache load checks its bytes; + - a primary refill from L2 checks the transferred line (EC set); + - Hit_Writeback(I) checks the line and raises the error from the CACHE + instruction (`take_op_status`). +- **Index_Load_Tag** loads CP0 ECC with the selected doubleword's check bits: + - L2: data ECC via Fig. 16-1, transcribed in `L2_DATA_ECC_MASKS`; + - D-cache: byte parity; + - I-cache: byte parity on R4400, per word on R5000. +- **Fill(I) on a 2-way cache** refills the way already holding the line. IDE + pins lines into the way VA[14] selects with Index_Store_Tag first. + +Measured before gating (interpreter, `--quick`): no cost distinguishable from +build-layout noise, which is ±10% on single `sys/` kernels. The gate is there +so the cost is zero by construction. + +Not modelled: snapshots do not carry forced bits; I-cache fetch does not check +parity; CacheErr's EB/EI/EW and secondary tag errors. diff --git a/rules/testing/r5000-taglo-is-not-r4400-pstate.md b/rules/testing/r5000-taglo-is-the-r4400-pstate.md similarity index 53% rename from rules/testing/r5000-taglo-is-not-r4400-pstate.md rename to rules/testing/r5000-taglo-is-the-r4400-pstate.md index 251406d7..63e67e20 100644 --- a/rules/testing/r5000-taglo-is-not-r4400-pstate.md +++ b/rules/testing/r5000-taglo-is-the-r4400-pstate.md @@ -1,28 +1,45 @@ -# R5000 L1 TagLo is D/V bits, not R4400 MESI PState +# R5000 L1 TagLo uses the R4400 PState encoding -Addresses `cpucritique.md` CACHE-1, and cleans up CACHE-2/CACHE-3/CACHE-4 alongside it. +Addresses `cpucritique.md` CACHE-1 (twice: see below), and cleans up +CACHE-2/CACHE-3/CACHE-4 alongside it. -## CACHE-1 — the real bug +## CACHE-1 — settled by the software SGI shipped (2026-10-11) -`C_ILT` / `C_IST` in `mips_cache_v2.rs` applied the R4400 tag layout to every model: +The R5000 primary TagLo is the R4000-family layout: `[31:8]` PTag, `[7:6]` +PState, `[0]` P. A valid I-cache line reads 2; a D-cache line reads 2 when +clean, 3 when dirty. Shared (1) never occurs on an R5000, so a stored 1 is +taken as CleanExclusive there. The only R5000-specific parts are P (over PTag +alone) and the way select (VA[14]). -| | R4400 | R5000 | -|---|---|---| -| L1-D `[7:6]` | 2-bit MESI PState: 0=Inv, 1=Shared, 2=CleanExcl, 3=DirtyExcl | bit 7 = **D**, bit 6 = **V** | -| L1-I `[7:6]` | same PState field (2 = valid) | bit 7 = **L** (lock), bit 6 = **V** | +Evidence, from `~/gits/irixhistory` (IP22 builds use one R4000-family binary +for R4400, R4600 and R5000): -The R5000 L1 has no bus snooping and therefore no Shared state at all, so the two encodings -are not reconcilable. Consequences before the fix: +- **IDE `icachetag`** (`stand/arcs/ide/fforward/cache/icache.c`) fails unless + every filled I-cache line reads PState == 2. The Indy script + (`stand/arcs/ide/IP22/ip24`) runs `icache2` with no CPU-type check. +- **IDE `set_tag`** (`.../cache/cachesubs.c`) writes PCLEANEXCL `0x80` to make + an I-cache line valid and PDIRTYEXCL `0xC0` for a dirty D-cache line, then + sets W with a load and store; W is not in TagLo. +- **IDE `icache_fill_writeback`** pins a line into a way on R4600/R4700/R5000 + by storing back exactly what Index_Load_Tag returned. +- **IRIX `mcparity.c`** (IP22 parity recovery) reads primary tags on any IP22 + CPU: PState != 0 is valid, PState > PCLEANEXCL is dirty. -- A valid **clean** R5000 line was emitted as PState=2 → `D=1, V=0` → reads back as - **invalid** to R5000 software. -- Firmware writing `TagLo = 0x40` ("valid, clean") was decoded as PState=1 → **Shared**, a - state R5000 does not have. +### How it went wrong first -Fixed by branching on `Self::IS_R5K` in both directions. `IS_R5K` is `const IC_WAYS == 2`, -so this is compile-time — no runtime branch on the cache path. +The first CACHE-1 fix decided the R5000 used discrete bits, D = 7 and V = 6, +from a code critique with no source behind it. It read a clean line back as +`0x40` (fails `icachetag`) and took IDE's `0x80` for *invalid*. Its tests were +written to the same assumption, so they passed. -We model no lock bit, so R5000 L1-I emits L=0 always. +The NEC VR5000 manual (Table 6-13: 0 = Invalid, 3 = Valid) doesn't match the +shipped software either. Trust what IRIX and IDE do on the real part over a +datasheet table, and never over a critique. + +Tests: `taglo_filled_lines_read_back_the_r4000_pstates_on_both_models`, +`taglo_store_decodes_what_ide_and_irix_write`, and the symmetry check +`taglo_store_load_round_trips_on_both_models`. See also +`cache-tag-index-and-check-bits.md`. ## CACHE-3 — the comment was simply false @@ -63,21 +80,25 @@ TagHi, so zeroing it on `C_ILT` is right. Left as-is. ## Tests, and what each actually proves -- `taglo_valid_clean_line_reads_back_valid_on_both_models` — fills a clean line and checks - R4400 reports PState=2 while R5000 reports V=1/D=0. **Catches the load side.** -- `taglo_r5000_decodes_hardware_bit_patterns` — feeds R5000 the literal patterns firmware - writes (`0x80`, `0x40`, `0xC0`) and checks how they land. **Catches the store side.** +- `taglo_filled_lines_read_back_the_r4000_pstates_on_both_models` — fills I and + D lines (clean, then dirty) on both models and checks PState 2/2/3. + **Catches the load side.** +- `taglo_store_decodes_what_ide_and_irix_write` — feeds the R5000 the patterns + IDE writes (`0x00`, `0x40`, `0x80`, `0xC0`). **Catches the store side.** - `taglo_store_load_round_trips_on_both_models` — symmetry only. -That third one is worth a warning: **it passes even with both halves wrong**, because an -encode/decode pair that shares a mistake is still self-inverse. It was written first and -gave a false green; the other two exist because of it. Keep it (an *asymmetric* pair is its -own bug class) but never treat a round-trip test as evidence that a wire format is correct. +The round trip **passes even with both halves wrong**: an encode/decode pair +that shares a mistake is still self-inverse. Keep it (an *asymmetric* pair is +its own bug class), but never treat a round trip as evidence that a format is +right. The first two only help if their expected values come from a source, +not from the encoder under test. The D/V-era tests failed exactly there: they +asserted the D/V layout itself. -Both fixes were verified by reverting each half independently and confirming the -corresponding test fails. +## Resolved: `C_IST` writes back a dirty L1D victim (cpucritique §5.1) -## Open: `C_IST` writes back a dirty L1D victim (cpucritique §5.1) +**Resolved since:** the L1D arm no longer writes back (see the comment in +`cache_op`'s `C_IST` arm), pinned by the test asserting "Index_Store_Tag wrote +the dirty line back to memory". The history below is kept for the reasoning. Investigated 2026-09-12, **deliberately not changed** — recorded so the next session starts from the finding rather than re-deriving it. diff --git a/src/bench_runner.rs b/src/bench_runner.rs index a391ec9a..a41b1ad7 100644 --- a/src/bench_runner.rs +++ b/src/bench_runner.rs @@ -60,6 +60,8 @@ pub struct BenchOptions { /// Emulated CPU. Runtime config since 96e5ddd, so the in-process runner picks /// it like any other machine setting rather than needing its own build. pub cpu: CpuModel, + /// Cache model, as `[machine] cache`. + pub cache: crate::config::CacheModel, /// Set this to abort. The machine is stopped and `run` returns an error — /// there is no partial report, because the suite prints its block only at /// the end. Exists so an interactive caller's Stop button does something @@ -74,6 +76,7 @@ impl Default for BenchOptions { label: "local".to_string(), banks: BENCH_BANKS, cpu: CpuModel::default(), + cache: crate::config::CacheModel::default(), timeout: Duration::from_secs(1800), cancel: None, } @@ -146,10 +149,9 @@ fn run_inner( }; let started = Instant::now(); - let mut machine = Box::new(Machine::new_with_testdev( - bench_config(opts.banks, opts.cpu), - Some(testdev), - )); + let mut cfg = bench_config(opts.banks, opts.cpu); + cfg.machine.cache = opts.cache; + let mut machine = Box::new(Machine::new_with_testdev(cfg, Some(testdev))); // Deliberately no `register_system_controller`: it hands a raw pointer to // this Machine to a thread that outlives the call, and this one is dropped // when the run finishes. Nothing here needs `reset`/`save`/`load` anyway. diff --git a/src/bin/iris_bench.rs b/src/bin/iris_bench.rs index e9c9a8f1..d26f4a3c 100644 --- a/src/bin/iris_bench.rs +++ b/src/bin/iris_bench.rs @@ -147,12 +147,14 @@ fn run_host(exe: &Path, timeout_s: u64) -> Result { /// for a minute is not mistaken for a hang, and that property is worth keeping /// at the command line — so this echoes every line rather than waiting for the /// end and printing the parsed summary. -fn run_embedded(label: &str, timeout_s: u64, quick: bool, cpu: iris::config::CpuModel) -> Result { +fn run_embedded(label: &str, timeout_s: u64, quick: bool, cpu: iris::config::CpuModel, + cache: iris::config::CacheModel) -> Result { let opts = BenchOptions { quick, label: label.to_string(), timeout: Duration::from_secs(timeout_s), cpu, + cache, ..Default::default() }; bench_runner::run(&opts, |p| { @@ -723,6 +725,10 @@ enum Cmd { /// emulator instead: `-- --cpu r5000`. #[arg(long, default_value = "r4400")] cpu: iris::config::CpuModel, + /// Cache model for the in-process run: auto (shadow in a jitv2 + /// build), full or shadow. With --iris: `-- --cache shadow`. + #[arg(long, default_value = "auto")] + cache: iris::config::CacheModel, /// Extra arguments passed through to --iris. #[arg(last = true)] extra: Vec, @@ -844,7 +850,7 @@ fn dispatch(cmd: Cmd) -> Result<(), String> { Ok(()) } - Cmd::Run { iris, elf, config, label, out, timeout, quick, cpu, extra } => { + Cmd::Run { iris, elf, config, label, out, timeout, quick, cpu, cache, extra } => { let out = out.unwrap_or_else(default_out); let run = match iris { Some(iris) => { @@ -873,7 +879,7 @@ fn dispatch(cmd: Cmd) -> Result<(), String> { "{} only {} to --iris, and `run` is in-process by default. \ Add --iris PATH, or drop {}.", stray.join(" and "), verb, obj)); } - run_embedded(&label, timeout, quick, cpu)? + run_embedded(&label, timeout, quick, cpu, cache)? } }; let path = save(&run, &out)?; diff --git a/src/config.rs b/src/config.rs index a5a726e2..80bba5b0 100644 --- a/src/config.rs +++ b/src/config.rs @@ -634,6 +634,40 @@ impl CpuModel { } } +/// How the CPU's caches are modelled. A runtime choice: each is its own +/// monomorphisation, picked when the machine is built. +/// +/// - `full`: the functional caches (`mips_cache_v2`): tags, line data, LRU, +/// write-back, VCE. What the interpreter is fastest with, since its decoded +/// instructions live in the cache lines. +/// - `shadow`: memory is the only store; tag arrays exist only to answer +/// CACHE instructions (`mips_cache_shadow`). jitv2's inline loads and stores +/// need no tag check against it. +/// - `auto`: `shadow` in a `jitv2` build, `full` otherwise. +/// +/// The R10000 (IP28) only has the shadow model. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, clap::ValueEnum)] +#[serde(rename_all = "snake_case")] +#[clap(rename_all = "lower")] +pub enum CacheModel { + #[default] + Auto, + Full, + Shadow, +} + +impl CacheModel { + /// Whether `cpu` runs the shadow cache under this setting. + pub fn is_shadow(self, cpu: CpuModel) -> bool { + match (cpu, self) { + (CpuModel::R10000, _) => true, + (_, Self::Full) => false, + (_, Self::Shadow) => true, + (_, Self::Auto) => cfg!(feature = "jitv2"), + } + } +} + /// `[machine]` section — platform identity (not performance knobs). #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -642,11 +676,13 @@ pub struct MachineSection { pub profile: MachineProfile, #[serde(default)] pub cpu: CpuModel, + #[serde(default)] + pub cache: CacheModel, } impl Default for MachineSection { fn default() -> Self { - Self { profile: MachineProfile::default(), cpu: CpuModel::default() } + Self { profile: MachineProfile::default(), cpu: CpuModel::default(), cache: CacheModel::default() } } } @@ -1651,6 +1687,12 @@ pub struct Cli { #[arg(long = "cpu", value_name = "MODEL")] pub cpu: Option, + /// Cache model: `auto` (default: `shadow` in a jitv2 build, else `full`), + /// `full` or `shadow`. Overrides `[machine] cache`. The R10000 is always + /// `shadow`. + #[arg(long = "cache", value_name = "MODEL")] + pub cache: Option, + /// Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`). #[arg(long = "graphics", visible_alias = "board", visible_alias = "graphics-board", value_name = "BOARD")] pub graphics: Option, @@ -1780,6 +1822,7 @@ impl Cli { if let Some(p) = self.scsi7.clone() { apply_scsi(&mut cfg.scsi, 7, p, false, vec![]); } if let Some(cpu) = self.cpu { cfg.machine.cpu = cpu; } + if let Some(cache) = self.cache { cfg.machine.cache = cache; } if let Some(b) = self.graphics { cfg.graphics.board = b; } if self.scale2x { cfg.scale = 2; } if self.headless { cfg.headless = true; } diff --git a/src/cpu/decode_cache.rs b/src/cpu/decode_cache.rs new file mode 100644 index 00000000..64f3e3d1 --- /dev/null +++ b/src/cpu/decode_cache.rs @@ -0,0 +1,292 @@ +//! Decoded-instruction cache for the shadow cache. +//! +//! The full cache model keeps decoded instructions in its cache lines, so the +//! interpreter decodes each instruction once per fill. The shadow cache has no +//! lines, and without this decodes every instruction on every fetch: about +//! half the interpreter's speed. +//! +//! This is a host-side memo of decode work and nothing more. It is keyed by +//! physical page, never consults or changes the architectural cache state the +//! shadow keeps for CACHE instructions, and is exactly coherent with memory, +//! like the rest of the shadow: a word served from here is always the word +//! memory holds now. Self-modifying code takes effect at once, with no stale +//! I-cache window — stricter than hardware, never wrong for correct software. +//! +//! Coherence is by compare: every fetch re-reads the word (and, for a fused +//! pair, the next one) and reuses the decode only if it is unchanged. Exact, +//! and cheap because the read is one host load through the page's own pointer +//! into ppmem's window, taken when the page is first used. Pages that are not +//! RAM (the PROM, MMIO, the low-512KB alias) have no pointer and read through +//! the bus. +//! +//! jitv2's per-page write-generation counters were tried as the validator in +//! jitv2 builds and measured no faster (raw/gen geomean ×1.05 on R4400, ×1.00 +//! on R5000, full-length iris-bench), so compare is the only mode: it works +//! the same in every build and every region. +//! +//! A RAM remap (MEMCFG) changes which regions are RAM and which counters back +//! a page: the cache records the region bitmap it was filled under and starts +//! over when it changes. Snapshot restore and power-on flush everything. + +use crate::cpu::mips_exec::{DecodedInstr, FLAG_NOT_DECODED}; +#[cfg(feature = "opcodefusion")] +use crate::cpu::mips_exec::FLAG_IMM_IS_NEXT; + +/// Decoded pages held at once. Direct-mapped on the page number. +pub const PAGES: usize = 64; +const SLOTS: usize = 1024; // 4 KB / 4 +const WORDS: usize = SLOTS / 64; +const EMPTY: u64 = u64::MAX; + +struct Page { + /// Physical page number, or `EMPTY`. + pfn: u64, + /// The page's words in ppmem's window, or null to read through the bus. + host: *const u32, + /// Slot holds a word read from memory (decoded or not yet). + valid: [u64; WORDS], + /// Raw mode: the slot's decode also depends on the following word + /// (opcode fusion), recorded in `next`. + fused: [u64; WORDS], + next: Box<[u32]>, + slots: Box<[DecodedInstr]>, +} + +impl Page { + fn new() -> Self { + Self { + pfn: EMPTY, + host: std::ptr::null(), + valid: [0; WORDS], + fused: [0; WORDS], + next: vec![0; SLOTS].into_boxed_slice(), + slots: (0..SLOTS).map(|_| DecodedInstr::default()).collect(), + } + } + + fn reset(&mut self, pfn: u64, host: *const u32) { + self.pfn = pfn; + self.host = host; + self.valid = [0; WORDS]; + self.fused = [0; WORDS]; + } +} + +pub struct DecodeCache { + pages: Box<[Page]>, + /// The ppmem region bitmap the pages were filled under. + bitmap: u64, +} + +impl Default for DecodeCache { + fn default() -> Self { Self::new() } +} + +impl DecodeCache { + pub fn new() -> Self { + Self { pages: (0..PAGES).map(|_| Page::new()).collect(), bitmap: 0 } + } + + /// Forget everything: memory may have changed without any counter moving. + pub fn flush(&mut self) { + for p in self.pages.iter_mut() { + p.pfn = EMPTY; + } + } + + /// The decoded slot for the instruction at `phys`, filled on a miss. + /// + /// `window` and `bitmap` are ppmem's window base + /// and region bitmap: a page in a RAM region reads its words straight out + /// of the window. `ic_line` is the I-cache line size: opcode fusion pairs a + /// word with the next one only within a line. `word` reads a word through + /// the bus, returning the bus status on failure. + #[inline(always)] + pub fn fetch( + &mut self, + phys: u64, + window: *const u8, + bitmap: u64, + ic_line: usize, + word: impl Fn(u32) -> Result, + ) -> Result<*const DecodedInstr, u32> { + // The common case, kept small enough to inline into the interpreter + // loop: a RAM page already here, the slot filled, its word (and a + // fused pair's second word) unchanged. One or two host loads. + if bitmap == self.bitmap { + let pfn = phys >> 12; + let k = (phys as usize >> 2) & (SLOTS - 1); + let (w, bit) = (k >> 6, 1u64 << (k & 63)); + let page = &self.pages[pfn as usize % PAGES]; + if page.pfn == pfn && !page.host.is_null() && page.valid[w] & bit != 0 { + let now = unsafe { *page.host.add(k) }; + if now == page.slots[k].raw + && (page.fused[w] & bit == 0 || unsafe { *page.host.add(k + 1) } == page.next[k]) + { + return Ok(&page.slots[k] as *const DecodedInstr); + } + } + } + self.fetch_slow(phys, window, bitmap, ic_line, word) + } + + /// Everything else: a new bitmap, a page change, a bus page, a fill. + #[inline(never)] + #[cold] + fn fetch_slow( + &mut self, + phys: u64, + window: *const u8, + bitmap: u64, + ic_line: usize, + word: impl Fn(u32) -> Result, + ) -> Result<*const DecodedInstr, u32> { + if bitmap != self.bitmap { + self.flush(); + self.bitmap = bitmap; + } + let pfn = phys >> 12; + let k = (phys as usize >> 2) & (SLOTS - 1); + let (w, bit) = (k >> 6, 1u64 << (k & 63)); + let page = &mut self.pages[pfn as usize % PAGES]; + let _ = ic_line; + let host_for = || -> *const u32 { + if !window.is_null() && bitmap & (1u64 << (phys >> crate::ppmem::BITMAP_SHIFT)) != 0 { + unsafe { window.add((pfn << 12) as usize) as *const u32 } + } else { + std::ptr::null() + } + }; + + if page.pfn != pfn { + page.reset(pfn, host_for()); + } + if page.valid[w] & bit != 0 { + let now = read(page.host, k, phys, &word)?; + let next_ok = page.fused[w] & bit == 0 + || read(page.host, k + 1, phys + 4, &word)? == page.next[k]; + if now == page.slots[k].raw && next_ok { + return Ok(&page.slots[k] as *const DecodedInstr); + } + } + + // Fill: the executor decodes the slot in place on first use. + let raw = read(page.host, k, phys, &word)?; + #[cfg(feature = "opcodefusion")] + let host = page.host; + let slot = &mut page.slots[k]; + slot.raw = raw; + slot.flags = FLAG_NOT_DECODED; + page.fused[w] &= !bit; + // Opcode fusion: hand decode the following word, as the full model's + // fill does, unless this is the last word of an I-cache line (fusion + // never crosses one, so the pair stays in this page). + #[cfg(feature = "opcodefusion")] + if (phys as usize) & (ic_line - 1) != ic_line - 4 { + let next = read(host, k + 1, phys + 4, &word)?; + slot.imm = next; + slot.flags |= FLAG_IMM_IS_NEXT; + page.next[k] = next; + page.fused[w] |= bit; + } + page.valid[w] |= bit; + Ok(slot as *const DecodedInstr) + } +} + +/// Word `k` of a page: one load from its window pointer, or a bus read at +/// `phys` when it has none. +#[inline(always)] +fn read(host: *const u32, k: usize, phys: u64, word: &impl Fn(u32) -> Result) -> Result { + if host.is_null() { word(phys as u32) } else { Ok(unsafe { *host.add(k) }) } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::collections::HashMap; + + fn mem(words: &[(u32, u32)]) -> RefCell> { + RefCell::new(words.iter().copied().collect()) + } + + /// A cached slot is served while its word is unchanged and refilled once + /// it changes. + #[test] + fn a_changed_word_is_refilled() { + { + let m = mem(&[(0x1000, 0x2400_0001), (0x1004, 0)]); + let read = |a: u32| Ok(*m.borrow().get(&a).unwrap_or(&0)); + let mut dc = DecodeCache::new(); + + let p = dc.fetch(0x1000, std::ptr::null(), 0, 16, read).unwrap(); + unsafe { (*(p as *mut DecodedInstr)).flags = 0 }; // the executor decoded it + let q = dc.fetch(0x1000, std::ptr::null(), 0, 16, read).unwrap(); + assert_eq!(p, q); + assert_eq!(unsafe { (*q).flags }, 0, "served decoded, not refilled"); + + m.borrow_mut().insert(0x1000, 0x2400_0002); + let r = dc.fetch(0x1000, std::ptr::null(), 0, 16, read).unwrap(); + assert_eq!(unsafe { (*r).raw }, 0x2400_0002, "refilled"); + assert_ne!(unsafe { (*r).flags } & FLAG_NOT_DECODED, 0); + } + } + + /// A bus error on the word reaches the caller as its status. + #[test] + fn a_bus_error_is_returned() { + let mut dc = DecodeCache::new(); + assert_eq!(dc.fetch(0x2000, std::ptr::null(), 0, 16, |_| Err(0x0800_001c)).err(), Some(0x0800_001c)); + } + + /// Pages that share a table entry evict each other rather than alias. + #[test] + fn colliding_pages_do_not_alias() { + let a = 0x1000u64; + let b = a + (PAGES as u64) * 4096; + let m = mem(&[(a as u32, 0x1111_1111), (b as u32, 0x2222_2222)]); + let read = |x: u32| Ok(*m.borrow().get(&x).unwrap_or(&0)); + let mut dc = DecodeCache::new(); + assert_eq!(unsafe { (*dc.fetch(a, std::ptr::null(), 0, 16, read).unwrap()).raw }, 0x1111_1111); + assert_eq!(unsafe { (*dc.fetch(b, std::ptr::null(), 0, 16, read).unwrap()).raw }, 0x2222_2222); + assert_eq!(unsafe { (*dc.fetch(a, std::ptr::null(), 0, 16, read).unwrap()).raw }, 0x1111_1111); + } + + /// A page in a RAM region reads its words through the window pointer, not + /// the bus; a changed region bitmap (a remap) starts the cache over. + #[test] + fn ram_pages_read_through_the_window_and_a_remap_flushes() { + let mut ram = vec![0u32; 2 * 1024]; + ram[1024 + 3] = 0x2400_0007; // phys 0x100c, page 1 word 3 + let window = ram.as_ptr() as *const u8; + let bus = |_| -> Result { panic!("RAM must not go through the bus") }; + let mut dc = DecodeCache::new(); + let p = dc.fetch(0x100c, window, 1, 16, bus).unwrap(); + assert_eq!(unsafe { (*p).raw }, 0x2400_0007); + + // Same bitmap: served (and validated) from the window. + ram[1024 + 3] = 0x2400_0008; + assert_eq!(unsafe { (*dc.fetch(0x100c, window, 1, 16, bus).unwrap()).raw }, 0x2400_0008); + + // Region no longer RAM: everything goes to the bus now. + let p = dc.fetch(0x100c, window, 0, 16, |_| Ok(0x2400_0009)).unwrap(); + assert_eq!(unsafe { (*p).raw }, 0x2400_0009); + } + + /// Raw mode: a fused slot depends on the next word too, so changing only + /// the delay slot must refill it. + #[test] + #[cfg(feature = "opcodefusion")] + fn raw_mode_rechecks_the_fused_next_word() { + let m = mem(&[(0x1000, 0x03e0_0008), (0x1004, 0)]); // jr ra; nop + let read = |a: u32| Ok(*m.borrow().get(&a).unwrap_or(&0)); + let mut dc = DecodeCache::new(); + let p = dc.fetch(0x1000, std::ptr::null(), 0, 16, read).unwrap(); + unsafe { (*(p as *mut DecodedInstr)).flags = 0 }; + m.borrow_mut().insert(0x1004, 0x2402_0001); // the delay slot is no longer a NOP + let q = dc.fetch(0x1000, std::ptr::null(), 0, 16, read).unwrap(); + assert_ne!(unsafe { (*q).flags } & FLAG_NOT_DECODED, 0, "refilled"); + assert_eq!(unsafe { (*q).imm }, 0x2402_0001); + } +} diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 16024072..182947ea 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -33,11 +33,10 @@ use std::cell::UnsafeCell; use std::sync::Arc; use crate::cpu::mips_cache_v2::{ - cache_op_name, CpuModel, FetchInstrResult, MipsCache, C_ILT, C_IST, C_R10K_CBARRIER, - C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, + cache_op_name, model, r4k_load_taglo, r4k_store_taglo, CpuModel, FetchInstrResult, MipsCache, + C_ILT, C_IST, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, }; -use crate::cpu::mips_exec::{DecodedInstr, FLAG_NOT_DECODED}; -use crate::traits::{BusDevice, BusRead64}; +use crate::traits::{BusDevice, BusRead64, BUS_OK}; use crate::devlog::{LogModule, CACHE_LOG_HIT, CACHE_LOG_OP, devlog_is_active, devlog_mask}; /// Shadow tag and data arrays for one cache. @@ -86,8 +85,9 @@ impl Shadow { /// nothing, because the way that is read is never the way that was written. const MRU_BIT: u64 = 1 << 63; -/// Ways per set, as CACHE index operations address them. Bit 0 of the index -/// selects the way on an R10000; the set starts above the line offset. +/// R10000 only: ways per set, as CACHE index operations address them. Bit 0 +/// of the index selects the way; the set starts above the line offset. The +/// R4400 and R5000 index differently — see `tag_slot`. const WAYS: usize = 2; /// Bits a secondary-cache tag retains. @@ -122,6 +122,10 @@ fn l2_op_log(bit: u32) -> bool { /// /// Const parameters carry the geometry that software can read back, and the /// processor identity. Nothing here affects the speed of a load. +/// +/// `MODEL` is `mips_cache_v2::model`, as for `CpuCache`, and decides +/// everything shape cannot: how a CACHE index op picks its line and way, what +/// a tag keeps, and the R10000's MRU bit, data arrays and op encodings. pub struct ShadowCache< const IC_SIZE: usize, const IC_LINE: usize, @@ -133,13 +137,11 @@ pub struct ShadowCache< const PRID: u32, const FIR: u32, const TLB_ENTRIES: usize, - const R10K_OPS: bool, + const MODEL: u8, > { downstream: Arc, llbit: UnsafeCell, lladdr: UnsafeCell, - /// Somewhere to decode into. Not a cache line — there is no caching. - fetch_scratch: UnsafeCell, /// CP0 ECC on the way in to a store, and on the way out of a load. ecc_in: UnsafeCell, ecc_out: UnsafeCell, @@ -151,12 +153,13 @@ pub struct ShadowCache< /// reads nothing through them — `read`/`write` go to `downstream`, whose /// ppmem path uses the same window — they are here only so jitv2's inline /// tcache path can serve this model too (`jit_dc_geometry`). - #[cfg(feature = "tcache")] tc_base: UnsafeCell<*mut u8>, - #[cfg(feature = "tcache")] tc_bitmap: UnsafeCell, #[cfg(feature = "jitv2")] tc_gen: UnsafeCell<*mut std::sync::atomic::AtomicU64>, + /// Decoded instructions, by physical page. Never touches the shadow + /// arrays above — see `decode_cache`. + decode: UnsafeCell, } // Safety: the CPU thread is the only accessor, as for every other cache model @@ -164,26 +167,28 @@ pub struct ShadowCache< unsafe impl< const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, - const FIR: u32, const TLB_ENTRIES: usize, const R10K_OPS: bool, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, > Send - for ShadowCache + for ShadowCache { } unsafe impl< const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, - const FIR: u32, const TLB_ENTRIES: usize, const R10K_OPS: bool, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, > Sync - for ShadowCache + for ShadowCache { } impl< const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, - const FIR: u32, const TLB_ENTRIES: usize, const R10K_OPS: bool, - > ShadowCache + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, + > ShadowCache { + const R10K: bool = MODEL == model::R10000; + const R5K: bool = MODEL == model::R5000; const IC_LINES: usize = if IC_LINE == 0 { 0 } else { IC_SIZE / IC_LINE }; const DC_LINES: usize = if DC_LINE == 0 { 0 } else { DC_SIZE / DC_LINE }; const L2_LINES: usize = if L2_LINE == 0 { 0 } else { L2_SIZE / L2_LINE }; @@ -193,20 +198,45 @@ impl< downstream, llbit: UnsafeCell::new(false), lladdr: UnsafeCell::new(0), - fetch_scratch: UnsafeCell::new(DecodedInstr::default()), ecc_in: UnsafeCell::new(0), ecc_out: UnsafeCell::new(0), ic: UnsafeCell::new(Shadow::new(Self::IC_LINES, 0)), dc: UnsafeCell::new(Shadow::new(Self::DC_LINES, 0)), - // Only the secondary keeps a data shadow: it is the one a PROM - // walks with Index_Store_Data, and a 1 MB array is cheap once. - l2: UnsafeCell::new(Shadow::new(Self::L2_LINES, L2_SIZE / 8)), - #[cfg(feature = "tcache")] + // Only the R10000 secondary keeps a data shadow: it is the one + // the IP28 PROM walks with Index_Store_Data, and a 1 MB array is + // cheap once. The R4000 family has no index data ops. + l2: UnsafeCell::new(Shadow::new(Self::L2_LINES, if Self::R10K { L2_SIZE / 8 } else { 0 })), tc_base: UnsafeCell::new(std::ptr::null_mut()), - #[cfg(feature = "tcache")] tc_bitmap: UnsafeCell::new(0), #[cfg(feature = "jitv2")] tc_gen: UnsafeCell::new(std::ptr::null_mut()), + decode: UnsafeCell::new(crate::cpu::decode_cache::DecodeCache::new()), + } + } + + /// ppmem's window base if `phys` is RAM the window maps, as the region + /// bitmap says; `None` for the PROM, MMIO and the low-512KB alias, which go + /// through the bus. The bitmap stays zero until the window is wired, so a + /// set bit also means the base is valid. + #[inline(always)] + fn ram(&self, phys: u64) -> Option<*mut u8> { + if unsafe { *self.tc_bitmap.get() } & (1u64 << (phys >> crate::ppmem::BITMAP_SHIFT)) != 0 { + Some(unsafe { *self.tc_base.get() }) + } else { + None + } + } + + /// A window store skips `BusDevice`, so it bumps the page's jitv2 + /// generation counter itself, as `PpMemory`'s writes do. + #[inline(always)] + fn bump_gen(&self, _phys: u64) { + #[cfg(feature = "jitv2")] + { + let g = unsafe { *self.tc_gen.get() }; + if !g.is_null() { + unsafe { (*g.add((_phys >> 12) as usize)).fetch_add(1, std::sync::atomic::Ordering::Relaxed) }; + } } } @@ -232,6 +262,10 @@ impl< /// line granularity — made the two ways of a set alias onto one slot, so /// a tag written to way 1 overwrote way 0 and the PROM read back the /// wrong one. That is the whole of the "TAG walking 1s" failure. + /// + /// That is the R10000. The R4400 is direct-mapped throughout: the index + /// is the set. The R5000's 2-way primaries take the way from VA[14], above + /// the 16 KB a way spans (VR5000 UM 3.4), as `CpuCache` does. fn tag_slot(&self, sel: u32, virt_addr: u64) -> usize { let (line, lines) = match sel { CACH_PI => (IC_LINE, Self::IC_LINES), @@ -241,6 +275,14 @@ impl< if line == 0 || lines == 0 { return 0; } + if !Self::R10K { + let va = virt_addr as usize; + if Self::R5K && matches!(sel, CACH_PI | CACH_PD) { + let sets = lines / 2; + return ((va >> 14) & 1) * sets + (va / line) % sets; + } + return (va / line) % lines; + } let way = (virt_addr as usize) & (WAYS - 1); let set = ((virt_addr as usize) / line) % (lines / WAYS).max(1); (set * WAYS + way) % lines @@ -264,9 +306,9 @@ impl< impl< const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, - const FIR: u32, const TLB_ENTRIES: usize, const R10K_OPS: bool, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, > From> - for ShadowCache + for ShadowCache { fn from(downstream: Arc) -> Self { Self::new(downstream) @@ -276,9 +318,9 @@ impl< impl< const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, - const FIR: u32, const TLB_ENTRIES: usize, const R10K_OPS: bool, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, > CpuModel - for ShadowCache + for ShadowCache { const MIPS4: bool = MIPS4; const PRID: u32 = PRID; @@ -287,17 +329,26 @@ impl< // The R10000 implements 44 virtual address bits where the R4x00 implements // 40; the shadow cache is only used for it, but key this off the same flag // that selects its cache encodings rather than asserting it unconditionally. - const VA_BITS: u32 = if R10K_OPS { 44 } else { 40 }; - const NAME: &'static str = "shadow"; - const R10K_CACHE_OPS: bool = R10K_OPS; + const VA_BITS: u32 = if MODEL == model::R10000 { 44 } else { 40 }; + // The CPU's name, which snapshots check: the same as the full model's, so + // a mismatch reads as a cache-kind difference rather than a different CPU. + // The R10000 keeps its historical "shadow", which existing IP28 + // snapshots carry. + const NAME: &'static str = match MODEL { + model::R4400 => "R4400", + model::R5000 => "R5000", + _ => "shadow", + }; + const CACHE_KIND: &'static str = "shadow"; + const R10K_CACHE_OPS: bool = MODEL == model::R10000; } impl< const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, - const FIR: u32, const TLB_ENTRIES: usize, const R10K_OPS: bool, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, > MipsCache - for ShadowCache + for ShadowCache { const IC_SIZE: usize = IC_SIZE; const IC_LINE: usize = IC_LINE; @@ -308,15 +359,14 @@ impl< const L2_SIZE: usize = L2_SIZE; const L2_LINE: usize = L2_LINE; - #[cfg(feature = "tcache")] + // ppmem's window, in every build: memory is this cache's only store, and + // reading RAM through the window is one host load instead of a bus call. unsafe fn set_tcache_window(&self, base: *mut u8) { unsafe { *self.tc_base.get() = base }; } - #[cfg(feature = "tcache")] fn tcache_bitmap_ptr(&self) -> *mut u64 { self.tc_bitmap.get() } - #[cfg(feature = "tcache")] fn tcache_base_ptr(&self) -> *mut u8 { unsafe { *self.tc_base.get() } } #[cfg(feature = "jitv2")] @@ -343,15 +393,18 @@ impl< crate::cpu::mips_cache_v2::JitDcGeometry::unsupported() } + #[inline(always)] fn fetch(&self, _virt_addr: u64, phys_addr: u64) -> FetchInstrResult { - let r = self.downstream.read32(phys_addr as u32); - if r.is_ok() { - let slot = unsafe { &mut *self.fetch_scratch.get() }; - slot.flags = FLAG_NOT_DECODED; - slot.raw = r.data; - FetchInstrResult::hit(slot as *const DecodedInstr) - } else { - FetchInstrResult::exception(r.status) + // Decoded instructions by physical page — see `decode_cache`. + let dc = unsafe { &mut *self.decode.get() }; + let word = |a: u32| { + let r = self.downstream.read32(a); + if r.is_ok() { Ok(r.data) } else { Err(r.status) } + }; + let (window, bitmap) = unsafe { (*self.tc_base.get() as *const u8, *self.tc_bitmap.get()) }; + match dc.fetch(phys_addr, window, bitmap, IC_LINE, word) { + Ok(p) => FetchInstrResult::hit(p), + Err(status) => FetchInstrResult::exception(status), } } @@ -359,6 +412,24 @@ impl< const { assert!(SIZE == 1 || SIZE == 2 || SIZE == 4 || SIZE == 8, "invalid memory access SIZE") }; + // RAM: one load out of ppmem's window. Its layout is u32 words in + // native order — byte at `off ^ 3`, halfword at `(off >> 1) ^ 1`, + // u64 as rotate_left(32) — the same as CpuCache::tc_read. + if let Some(base) = self.ram(phys_addr) { + let off = phys_addr as usize; + let data = unsafe { + if SIZE == 8 { + (*(base.add(off) as *const u64)).rotate_left(32) + } else if SIZE == 4 { + *(base.add(off) as *const u32) as u64 + } else if SIZE == 2 { + *((base as *const u16).add((off >> 1) ^ 1)) as u64 + } else { + *base.add(off ^ 3) as u64 + } + }; + return BusRead64::ok(data); + } let a = phys_addr as u32; if SIZE == 1 { let r = self.downstream.read8(a); @@ -378,6 +449,22 @@ impl< const { assert!(SIZE == 1 || SIZE == 2 || SIZE == 4 || SIZE == 8, "invalid memory access SIZE") }; + if let Some(base) = self.ram(phys_addr) { + let off = phys_addr as usize; + unsafe { + if SIZE == 8 { + *(base.add(off) as *mut u64) = val.rotate_left(32); + } else if SIZE == 4 { + *(base.add(off) as *mut u32) = val as u32; + } else if SIZE == 2 { + *((base as *mut u16).add((off >> 1) ^ 1)) = val as u16; + } else { + *base.add(off ^ 3) = val as u8; + } + } + self.bump_gen(phys_addr); + return BUS_OK; + } let a = phys_addr as u32; if SIZE == 1 { self.downstream.write8(a, val as u8) @@ -391,6 +478,15 @@ impl< } fn write64_masked(&self, _virt_addr: u64, phys_addr: u64, val: u64, mask: u64) -> u32 { + if let Some(base) = self.ram(phys_addr) { + let p = unsafe { base.add((phys_addr & !7) as usize) as *mut u64 }; + unsafe { + let cur = (*p).rotate_left(32); + *p = ((cur & !mask) | (val & mask)).rotate_left(32); + } + self.bump_gen(phys_addr); + return BUS_OK; + } let aligned = (phys_addr & !7) as u32; let r = self.downstream.read64(aligned); if !r.is_ok() { @@ -418,6 +514,22 @@ impl< cache_op_name(cache_op)); } + // R4400/R5000: a tag keeps only its fields, and reads back with + // generated check bits — exactly what the full model returns. + if !Self::R10K && matches!(op, C_IST | C_ILT) { + let lines = match sel { CACH_PI => Self::IC_LINES, CACH_PD => Self::DC_LINES, _ => Self::L2_LINES }; + if lines == 0 { + return 0; // no secondary on the R5000 + } + let idx = self.tag_slot(sel, virt_addr); + let s = self.shadow(sel); + if op == C_IST { + s.tags[idx] = r4k_store_taglo(Self::R5K, sel, phys_addr as u32) as u64; + return 0; + } + return r4k_load_taglo(Self::R5K, sel, s.tags[idx] as u32) as u64; + } + match op { // Index_Store_Tag / Index_Load_Tag. Stored and returned verbatim: // a tag test is a round trip, and round trips do not require @@ -461,8 +573,8 @@ impl< } // R10000 reassigns 5/6/7, scoped to particular cache selects. - C_R10K_CBARRIER if R10K_OPS && sel == CACH_PI => 0, - C_R10K_ILD if R10K_OPS && matches!(sel, CACH_PI | CACH_PD | CACH_SD) => { + C_R10K_CBARRIER if Self::R10K && sel == CACH_PI => 0, + C_R10K_ILD if Self::R10K && matches!(sel, CACH_PI | CACH_PD | CACH_SD) => { let s = self.shadow(sel); let slot = self.data_slot(s.data.len(), virt_addr); if s.data.is_empty() { @@ -476,7 +588,7 @@ impl< s.data[slot] } } - C_R10K_ISD if R10K_OPS && matches!(sel, CACH_SI | CACH_SD) => { + C_R10K_ISD if Self::R10K && matches!(sel, CACH_SI | CACH_SD) => { let s = self.shadow(sel); let slot = self.data_slot(s.data.len(), virt_addr); if !s.data.is_empty() { @@ -496,6 +608,24 @@ impl< } } + /// Power-on: tags read back invalid, and nothing decoded survives — RAM + /// may be reloaded without a write counter moving. + fn power_on(&self) { + for sel in [CACH_PI, CACH_PD, CACH_SD] { + let s = self.shadow(sel); + s.tags.fill(0); + s.mru.fill(false); + } + unsafe { (*self.decode.get()).flush() }; + } + + /// A restore replaces RAM wholesale; drop every decoded instruction. The + /// shadow arrays are not snapshotted (nothing reads them but diagnostics). + fn load_cache_state(&self, _v: &toml::Value) -> Result<(), String> { + unsafe { (*self.decode.get()).flush() }; + Ok(()) + } + fn get_config(&self, cache_target: u32) -> (usize, usize) { match cache_target { CACH_PI => (IC_SIZE, IC_LINE), @@ -543,7 +673,19 @@ impl< /// Nothing of the microarchitecture: no ways, no LRU, no out-of-order. /// Reports revision 2.5, matching R10000Cache and enabling the IP28 boot tune. pub type R10000ShadowCache = - ShadowCache<32768, 64, 32768, 32, 1048576, 128, true, 0x0000_0925, 0x0000_0900, 64, true>; + ShadowCache<32768, 64, 32768, 32, 1048576, 128, true, 0x0000_0925, 0x0000_0900, 64, { model::R10000 }>; + +/// SGI Indy / Indigo2 R4400: the full model's geometry and identity — 16 KB +/// direct-mapped primaries with 16-byte lines, a 1 MB secondary with 128-byte +/// lines, PRId/FIR, 48 TLB entries, MIPS III — so CP0 Config and a PROM's +/// cache sizing see the same part. `shadow_matches_the_full_models` pins it. +pub type R4400ShadowCache = + ShadowCache<16384, 16, 16384, 16, 1048576, 128, false, 0x0000_0440, 0x0000_0500, 48, { model::R4400 }>; + +/// SGI Indy R5000: 2-way 32 KB primaries with 32-byte lines, no secondary, +/// MIPS IV, the full model's PRId/FIR and TLB. +pub type R5000ShadowCache = + ShadowCache<32768, 32, 32768, 32, 0, 128, true, 0x0000_2321, 0x0000_2300, 48, { model::R5000 }>; #[cfg(test)] mod tests { @@ -731,6 +873,54 @@ mod tests { assert_eq!(c.get_config(CACH_SD), (1048576, 128)); } + /// The R4400 and R5000 shadows stand in for the full models, so everything + /// software can read — CP0 Config geometry, PRId/FIR, ISA level, TLB size, + /// the name snapshots check — must be the full model's. + #[test] + fn shadow_matches_the_full_models() { + use crate::cpu::mips_cache_v2::{R4400Cache, R5000Cache}; + fn same() { + assert_eq!((S::IC_SIZE, S::IC_LINE, S::DC_SIZE, S::DC_LINE, S::L2_SIZE, S::L2_LINE), + (F::IC_SIZE, F::IC_LINE, F::DC_SIZE, F::DC_LINE, F::L2_SIZE, F::L2_LINE), "geometry"); + assert_eq!((S::PRID, S::FIR, S::MIPS4, S::TLB_ENTRIES, S::VA_BITS, S::R10K_CACHE_OPS), + (F::PRID, F::FIR, F::MIPS4, F::TLB_ENTRIES, F::VA_BITS, F::R10K_CACHE_OPS), "identity"); + assert_eq!(S::NAME, F::NAME, "snapshot name"); + assert_eq!((S::CACHE_KIND, F::CACHE_KIND), ("shadow", "full")); + } + same::(); + same::(); + assert_eq!(::NAME, "shadow", "existing IP28 snapshots"); + } + + /// Index_Store_Tag then Index_Load_Tag returns exactly what the full model + /// returns, for random TagLo words at random indexes: the shadow must not + /// be observably different to a PROM, IDE or IRIX reading tags. Both + /// caches start with every tag invalid, so a store followed by a load is + /// the whole of what software can see. + #[test] + fn tags_read_back_as_the_full_model_does() { + use crate::cpu::mips_cache_v2::{R4400Cache, R5000Cache}; + fn check>, F: MipsCache + From>>( + name: &str, sels: &[u32]) { + let mem = || -> Arc { Arc::new(Memory::new(1)) }; + let (shadow, full) = (S::from(mem()), F::from(mem())); + let mut x = 0x9e37_79b9_7f4a_7c15u64; + for _ in 0..20_000 { + x ^= x << 13; x ^= x >> 7; x ^= x << 17; + let sel = sels[(x >> 60) as usize % sels.len()]; + let index = 0x8000_0000 | (x >> 20 & 0xF_FFF8); + let taglo = x as u32 as u64; + shadow.cache_op(C_IST | sel, index, taglo); + full.cache_op(C_IST | sel, index, taglo); + let probe = 0x8000_0000 | (x >> 40 & 0xF_FFF8); + assert_eq!(shadow.cache_op(C_ILT | sel, probe, probe), full.cache_op(C_ILT | sel, probe, probe), + "{name}: sel {sel} TagLo {taglo:#x} stored at {index:#x}, loaded at {probe:#x}"); + } + } + check::("R4400", &[CACH_PI, CACH_PD, CACH_SD]); + check::("R5000", &[CACH_PI, CACH_PD]); + } + /// Under tcache the JIT serves this model through the window alone, and /// only once both windows are published: the emitted code dereferences /// them without a null check. diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index f6a8d75a..62383e50 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -8,7 +8,7 @@ use crate::traits::{BusRead64, BusDevice, Resettable, BUS_OK, BUS_BUSY, BUS_ERR, BUS_VCE}; use crate::snapshot::{u32_slice_to_toml, u64_slice_to_toml, load_u32_slice, load_u64_slice, get_field, toml_bool, toml_u32, hex_u32}; -use crate::cpu::mips_exec::{DecodedInstr, ExecStatus, EXEC_COMPLETE, EXEC_RETRY, exec_exception_const, EXC_VCEI, EXC_VCED, EXC_IBE, FLAG_NOT_DECODED, FLAG_IMM_IS_NEXT}; +use crate::cpu::mips_exec::{DecodedInstr, ExecStatus, EXEC_COMPLETE, EXEC_RETRY, exec_exception_const, EXC_VCEI, EXC_VCED, EXC_IBE, FLAG_NOT_DECODED, FLAG_IMM_IS_NEXT, EXEC_CACHE_ERR}; use crate::devlog::{LogModule, CACHE_LOG_HIT, CACHE_LOG_MISS, CACHE_LOG_OP, devlog_is_active, devlog_mask}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -256,11 +256,31 @@ impl L1DTag { /// True iff this tag is valid and covers the same physical line as `phys_addr`. /// Branchless: one AND+OR on phys_addr, one 64-bit compare. No cs load, no branch. + /// + /// A line carrying [`Self::SLOW`] never matches here, by design: that is + /// how it reaches the miss path. Anything that must find the line + /// regardless (CACHE hit ops, write-back and invalidation scans) uses + /// [`Self::matches_line`]. #[inline(always)] pub fn matches_phys(&self, phys_addr: u64) -> bool { self.ptag == (phys_addr & !0xFFF) | 1 } + /// Bit 1 of `ptag`: send every access to this line through the miss path. + /// + /// Parity forcing (Status.CE) and forced-parity lines need a check on + /// each access, and the hit path cannot afford one. A marked line fails + /// the exact compare in `matches_phys`, in the interpreter and in jitv2's + /// inline guard alike, so the check lives in the miss path at no cost to + /// anything else. Bit 1 of a page-aligned address is free, like bit 0. + pub const SLOW: u64 = 2; + + /// [`Self::matches_phys`], ignoring the [`Self::SLOW`] mark. + #[inline] + pub fn matches_line(&self, phys_addr: u64) -> bool { + self.ptag & !Self::SLOW == (phys_addr & !0xFFF) | 1 + } + /// True iff this tag is valid (for non-hot-path use only). #[inline(always)] pub fn is_valid(&self) -> bool { self.ptag & 1 != 0 } @@ -302,8 +322,83 @@ pub fn migrate_l1d_tag_word_v0(word: u32) -> u32 { raw_ptag | (cs << L1D_TAG_CS_SHIFT) | (w << L1D_TAG_W_SHIFT) } +/// Even parity over `v`: the bit that makes the total count of ones even. +#[inline] +pub fn even_parity(v: u32) -> u32 { v.count_ones() & 1 } + +/// R4000 secondary-cache tag check matrix, UM Figure 16-4. Entry `i` selects +/// the tag bits (CS:PIdx:STag, `[24:0]` as in Figure 11-6) that check bit `i` +/// covers. Every tag bit has a distinct weight-3 syndrome; see +/// `l2_tag_ecc_matrix_is_the_manuals`. +const L2_TAG_ECC_MASKS: [u32; 7] = [ + 0x1ff1111, 0x045222f, 0x10a40ff, 0x09184f0, 0x0620f42, 0x114ff04, 0x0a8f888, +]; + +/// The 7 check bits the R4400 generates for a secondary tag. +/// +/// Not stored: the hardware computes them whenever a tag is written and +/// ignores the ECC field of TagLo on `Index_Store_Tag` (UM p. 93), so +/// computing them when the tag is read back gives the same value with nothing +/// to keep in sync. +pub fn l2_tag_ecc(tag: u32) -> u32 { + L2_TAG_ECC_MASKS.iter().enumerate() + .fold(0, |ecc, (i, &m)| ecc | even_parity(tag & m) << i) +} + +/// R4000 secondary-cache data check matrix, UM Figure 16-1: entry `i` selects +/// the data bits check bit `i` covers. Pinned by +/// `l2_data_ecc_matrix_is_the_manuals`. +const L2_DATA_ECC_MASKS: [u64; 8] = [ + 0x145011110ff014ff, 0x24ff2222f000249f, 0x4c9f444400ff44d0, 0x84d08888ff0f8c50, + 0x0931f0ff11110b21, 0x0b22ff002222fa32, 0xfa24000f4444ff24, 0xff280ff088880928, +]; + +/// The 8 check bits the R4400 computes for a secondary-cache doubleword. +pub fn l2_data_ecc(v: u64) -> u8 { + L2_DATA_ECC_MASKS.iter().enumerate() + .fold(0, |ecc, (i, &m)| ecc | (((v & m).count_ones() & 1) as u8) << i) +} + +/// R4400/R5000 Index_Store_Tag: the TagLo a later Index_Load_Tag returns, +/// without its check bits. A tag is only as wide as its fields, so this is +/// what a cache keeping raw TagLo words must store to read back what the +/// full model does (see `CpuCache::cache_op`, which keeps them as fields): +/// +/// - primary: PTag[31:8] and PState[7:6]. An invalid line keeps nothing. Any +/// valid I-cache state reads back as 2; on the R5000, which has no Shared +/// state, a stored 1 reads back as 2. +/// - secondary (R4400): STag[31:13], SState[12:10] (reserved 1-3 read back as +/// 0), PIdx[9:7]. +pub fn r4k_store_taglo(r5k: bool, sel: u32, taglo: u32) -> u32 { + if matches!(sel, CACH_SI | CACH_SD) { + let state = match (taglo >> 10) & 7 { s @ 4..=7 => s, _ => 0 }; + return (taglo & 0xFFFF_E000) | state << 10 | (taglo & 0x380); + } + let pstate = match (taglo >> 6) & 3 { + 0 => return 0, + _ if sel == CACH_PI => 2, + 1 if r5k => 2, + p => p, + }; + (taglo & 0xFFFF_FF00) | pstate << 6 +} + +/// [`r4k_store_taglo`]'s word as Index_Load_Tag returns it: with the check +/// bits the hardware generates (TagLo[0] P on a primary, TagLo[6:0] ECC on +/// the R4400 secondary). +pub fn r4k_load_taglo(r5k: bool, sel: u32, stored: u32) -> u32 { + if matches!(sel, CACH_SI | CACH_SD) { + let fields = (stored >> 10 & 7) << 22 | (stored >> 7 & 7) << 19 | stored >> 13; + return stored | l2_tag_ecc(fields); + } + let pstate = (stored >> 6) & 3; + let p = even_parity(stored >> 8) + ^ if r5k { 0 } else if sel == CACH_PI { (pstate != 0) as u32 } else { even_parity(pstate) }; + stored | p +} + // L2 Cache Tag -// [31:25] ECC - Error correction code (ignored) +// [31:25] ECC - Error correction code; not stored, see `l2_tag_ecc` // [24:22] CS - Cache State (0=Invalid, 4=CleanExcl, 5=DirtyExcl, 6=Shared, 7=DirtyShared) // [21:19] PIdx - Virtual address bits [14:12] (primary cache aliasing) // [18:0] PTag - Physical address bits [35:17] @@ -436,6 +531,10 @@ pub trait CpuModel: MipsCache { const VA_BITS: u32 = 40; /// Name as the guest and the benchmark report see it. const NAME: &'static str; + /// "full" for the model whose arrays loads, stores and fetches consult; + /// "shadow" for one that keeps only what CACHE ops observe. Recorded in + /// snapshots: a full cache's state does not load into a shadow one. + const CACHE_KIND: &'static str = "full"; /// Cache ops 5/6/7 carry their R10000 meanings rather than their R4000 ones. /// The executor needs this: `Index_Store_Data` takes its value from TagLo /// and `Index_Load_Data` returns into it, neither of which is true of the @@ -455,17 +554,14 @@ pub trait MipsCache: Send + Sync { /// # Safety /// `base` must be ppmem's window base and `bitmap` a live `u64` that /// outlives this cache. - #[cfg(feature = "tcache")] unsafe fn set_tcache_window(&self, _base: *mut u8) {} /// tcache: pointer to this cache's inline mapped-region bitmap, for /// `MappedMemory::set_bitmap_sink2`. Null when the cache has no such field. - #[cfg(feature = "tcache")] fn tcache_bitmap_ptr(&self) -> *mut u64 { std::ptr::null_mut() } /// tcache: base of the ppmem window this cache reads/writes through /// (`tc_base + phys`). Null until `set_transparent_base` runs. - #[cfg(feature = "tcache")] fn tcache_base_ptr(&self) -> *mut u8 { std::ptr::null_mut() } /// tcache: base of the jitv2 per-page generation array (`[AtomicU64]`, @@ -601,6 +697,18 @@ pub trait MipsCache: Send + Sync { /// takes them from this register and `Index_Load_Data` returns them /// there. Models that do not keep them ignore both of these. fn set_cache_ecc(&self, _v: u32) {} + /// The Status bits that govern cache error checking: CE (store the ECC + /// register as check bits instead of computing them) and whether errors + /// are detected at all (neither DE nor ERL set). Pushed by the executor on + /// every Status change. Models that keep no check bits ignore it. + fn set_error_control(&self, _ce: bool, _detect: bool) {} + /// CacheErr for the Cache Error exception the last access reported + /// (`EXEC_CACHE_ERR`). + fn take_cache_err(&self) -> u32 { 0 } + /// A Cache Error raised by the CACHE instruction just executed (an I-cache + /// write-back of a line with bad parity), as an exception status, or 0. + /// Reading it clears it. + fn take_op_status(&self) -> u32 { 0 } /// CP0 `ECC` after the last cache operation. fn cache_op_ecc(&self) -> u32 { 0 } @@ -1023,6 +1131,78 @@ const DEBUG_TRACK_ADDR: Option = Some(0x080165d4); #[cfg(not(feature = "debug_cache"))] const DEBUG_TRACK_ADDR: Option = None; +/// Primary data parity, as far as software can observe it: what Status.CE +/// and the ECC register do to the D-cache's check bits, and the Cache Error +/// exception a mismatch raises (R4000 UM ch. 5, VR5000 UM 7.2.9). +/// +/// Check bits are not stored for ordinary data: unless forced, they always +/// match the data, so they are computed when needed. Only doublewords whose +/// check bits were forced are recorded, in `forced`, and their lines carry +/// [`L1DTag::SLOW`] so every access reaches the check. While CE is set every +/// valid line is marked, since any store may force bits. +#[derive(Default)] +struct ParityState { + /// Status.CE: stores take the ECC register as check bits. + ce: bool, + /// Errors raise a Cache Error exception: Status.DE and Status.ERL clear. + detect: bool, + /// CP0 ECC, low 8 bits: one even-parity bit per byte, byte 0 the least + /// significant. + ecc: u8, + /// (L1-D extended index, doubleword within the line, its check bits). + forced: Vec<(usize, u8, u8)>, + /// Secondary data: per doubleword (physical address within the cache, + /// `>> 3`), the ECC register value XOR'd into its check bits by a CE + /// write-back. Empty until first used; `l2_xor_live` counts non-zero + /// entries so everything else can skip it. + l2_xor: Vec, + l2_xor_live: usize, + /// CP0 ECC after the last Index_Load_Tag. + ecc_out: u32, + /// I-cache lines a CE Fill gave forced parity: (extended index, the bits, + /// the same for every doubleword of the line). + ic_forced: Vec<(usize, u8)>, + /// See `MipsCache::take_op_status`. + op_status: u32, + /// The way `l1d_slow_resolve` found, for the access it is serving. + way: u32, + /// CacheErr for the error last reported. + cacheerr: u32, +} + +/// `fill_l1d_line` result: the line is resolved (way in `ParityState::way`) +/// and the access must take the parity-checking path. Not a status: real +/// statuses are exceptions (bit 27) or retry (0x100), and a way is 0 or 1. +const L1D_SLOW: u32 = 2; + +/// Parity/ECC injection and checking (`cache-ecc`). Off, every hook below +/// folds away: nothing on a miss, an invalidate or an exception looks at +/// `ParityState`. Only diagnostics ever observe it. +const ECC: bool = cfg!(feature = "cache-ecc"); + +/// `EXEC_IS_CACHE_ERR`, for telling a Cache Error apart from other fill +/// statuses. +const EXEC_IS_CACHE_ERR_BIT: u32 = EXEC_CACHE_ERR & !crate::cpu::mips_exec::EXEC_IS_EXCEPTION; + +/// CacheErr: data reference (ER), data field error (ED). R4000 UM Table 5-10; +/// the VR5000 has both at the same positions. +const CACHEERR_ER: u32 = 1 << 31; +const CACHEERR_EC: u32 = 1 << 30; +const CACHEERR_ED: u32 = 1 << 29; + +/// Even parity per byte of `v`: bit `i` covers byte `i`, least significant +/// first, as the ECC register lays them out. +fn byte_parity(v: u64) -> u8 { + (0..8).fold(0, |p, i| p | ((((v >> (8 * i)) as u8).count_ones() & 1) as u8) << i) +} + +/// The byte lanes (as in [`byte_parity`]) an access of `size` bytes at `addr` +/// touches. Big-endian: the byte at offset 0 is the most significant. +fn lane_mask(addr: u64, size: usize) -> u8 { + let off = (addr & 7) as usize; + (off..off + size).fold(0, |m, k| m | 1 << (7 - k)) +} + /// R4000 cache with proper 2-level hierarchy /// /// This implementation keeps L1-I, L1-D, and L2 in a single object @@ -1064,6 +1244,14 @@ pub struct CpuCache< ic_instrs: CacheVec, // Per-set LRU bitmaps for 2-way L1I/L1D; empty on a direct-mapped model. ic_lru: UnsafeCell>, + /// The words each L1-I line held when it was filled, for Hit_Writeback(I). + /// Direct-mapped models only. Their fetch path serves the L2's decode + /// slots, which follow L2 data after the fill, but the hardware I-cache + /// keeps its own copy until refilled. The 2-way model's `ic_instrs` already + /// is that copy. Empty on a 2-way model. + ic_raw: UnsafeCell>, + /// Status.CE / ECC parity forcing and Cache Error reporting. Cold. + perr: UnsafeCell, dc_lru: UnsafeCell>, /// tcache: base of ppmem's 4GB window, or null when unavailable. @@ -1310,6 +1498,9 @@ impl1 = exception status (EXC_VCEI or EXC_IBE). + /// + /// The miss path: kept out of line and cold so its size never decides + /// whether `fetch` inlines into the interpreter loop. When the `ic_raw` + /// snapshot grew this function, `fetch` stopped inlining and every + /// instruction paid a call — about x0.84 on the whole interpreter suite. + #[inline(never)] + #[cold] fn fill_l1i_line(&self, index_addr: u64, phys_addr: u64) -> u32 { let set = self.ic.get_index(index_addr); - // 2-way picks the LRU way; direct-mapped has one way, so eidx == set. + // 2-way picks the LRU way, unless a way already holds this line: a + // CACHE Fill then refills that way. IDE's icache test relies on it on + // the R4600/R5000, placing a line with Index_Store_Tag into the way + // VA[14] selects and then filling it. A fetch only gets here on a + // miss, so for it this is always the LRU way. Direct-mapped has one + // way, so eidx == set. let ic_eidx = if Self::IS_R5K { - set | (unsafe { Self::lru_get(self.ic_lru.get(), set) } as usize) << Self::IC_NUM_LINES_SHIFT + (0..IC_WAYS) + .map(|w| set | w << Self::IC_NUM_LINES_SHIFT) + .find(|&e| self.ic.get_tag(e).matches_phys(phys_addr)) + .unwrap_or_else(|| set | (unsafe { Self::lru_get(self.ic_lru.get(), set) } as usize) << Self::IC_NUM_LINES_SHIFT) } else { set }; // Invalidate victim slot unconditionally — clears tag before any early return. @@ -2405,6 +2622,17 @@ impl u32 { + if ECC { + let e = self.perr(); + if e.ce || !e.forced.is_empty() || e.l2_xor_live != 0 { + return self.l1d_slow_resolve(virt_addr, phys_addr); + } + } + self.fill_l1d_line_raw(virt_addr, phys_addr) + } + + #[allow(clippy::mut_from_ref)] + #[inline(always)] + fn perr(&self) -> &mut ParityState { + unsafe { &mut *self.perr.get() } + } + + /// A miss while parity state is live: either the line is resident but + /// marked [`L1DTag::SLOW`], or it is not resident and is filled (and + /// marked, if CE is set). Returns [`L1D_SLOW`] with the way in + /// `ParityState::way`, or a fill's exception status. + #[cold] + fn l1d_slow_resolve(&self, virt_addr: u64, phys_addr: u64) -> u32 { + let set = self.dc.get_index(virt_addr); + let found = (0..DC_WAYS) + .find(|&w| self.dc.get_tag(set | w << Self::DC_NUM_LINES_SHIFT).matches_line(phys_addr)); + let way = match found { + Some(w) => w as u32, + None => { + if let Some(status) = self.l2_refill_check(virt_addr, phys_addr, Self::DC_LINE, true) { + return status; + } + let r = self.fill_l1d_line_raw(virt_addr, phys_addr); + if r > 1 { return r; } + r + } + }; + let eidx = set | (way as usize) << Self::DC_NUM_LINES_SHIFT; + let mut tag: L1DTag = self.dc.get_tag(eidx); + let marked = self.perr().ce || self.l1d_forced(eidx); + tag.ptag = if marked { tag.ptag | L1DTag::SLOW } else { tag.ptag & !L1DTag::SLOW }; + self.dc.set_tag(eidx, tag); + self.perr().way = way; + L1D_SLOW + } + + fn l1d_forced(&self, eidx: usize) -> bool { + self.perr().forced.iter().any(|f| f.0 == eidx) + } + + /// Drop forced check bits for line `eidx`: its contents are gone. + #[inline] + fn l1d_unforce(&self, eidx: usize) { + if !ECC { return; } + let e = self.perr(); + if !e.forced.is_empty() { + e.forced.retain(|f| f.0 != eidx); + } + } + + /// Set or clear the [`L1DTag::SLOW`] mark on every valid L1-D line. A line + /// with forced check bits stays marked either way. + fn mark_l1d_lines(&self, on: bool) { + for eidx in 0..DC_WAYS * Self::DC_NUM_LINES { + let mut tag: L1DTag = self.dc.get_tag(eidx); + if !tag.is_valid() { continue; } + tag.ptag = if on || self.l1d_forced(eidx) { tag.ptag | L1DTag::SLOW } else { tag.ptag & !L1DTag::SLOW }; + self.dc.set_tag(eidx, tag); + } + } + + /// The whole doubleword at `virt_addr` in resolved line `eidx`. + fn l1d_dword(&self, eidx: usize, virt_addr: u64, phys_addr: u64) -> u64 { + let _ = (eidx, virt_addr); + #[cfg(feature = "tcache")] + { self.tc_read::<8>(phys_addr & !7) } + #[cfg(not(feature = "tcache"))] + { let _ = phys_addr; self.dc.dc_read::<8>(Self::dc_data_addr(eidx, virt_addr & !7)) } + } + + /// Check a load of the `lanes` of the doubleword at `virt_addr` against its + /// forced check bits. Returns the Cache Error status if they disagree and + /// errors are being detected. + fn l1d_parity_check(&self, eidx: usize, virt_addr: u64, phys_addr: u64, lanes: u8) -> Option { + let e = self.perr(); + if !e.detect { return None; } + let dw = ((virt_addr as usize & Self::DC_LINE_MASK) >> 3) as u8; + let forced = e.forced.iter().find(|f| f.0 == eidx && f.1 == dw)?.2; + if (byte_parity(self.l1d_dword(eidx, virt_addr, phys_addr)) ^ forced) & lanes == 0 { + return None; + } + // SIdx: pAddr[21:3]; PIdx: vAddr[14:12]. + self.perr().cacheerr = CACHEERR_ER | CACHEERR_ED + | (phys_addr as u32 & 0x003F_FFF8) | ((virt_addr >> 12) as u32 & 7); + Some(EXEC_CACHE_ERR) + } + + /// After a store to the `lanes` of the doubleword at `virt_addr` (`before` + /// is the doubleword it replaced): with CE set those lanes take the ECC + /// register's bits, otherwise freshly computed ones. A doubleword whose + /// bits agree with its data again is forgotten. + fn l1d_parity_store(&self, eidx: usize, virt_addr: u64, phys_addr: u64, before: u64, lanes: u8) { + let after = byte_parity(self.l1d_dword(eidx, virt_addr, phys_addr)); + let dw = ((virt_addr as usize & Self::DC_LINE_MASK) >> 3) as u8; + let e = self.perr(); + let pos = e.forced.iter().position(|f| f.0 == eidx && f.1 == dw); + let prev = pos.map_or(byte_parity(before), |i| e.forced[i].2); + let src = if e.ce { e.ecc } else { after }; + let bits = (prev & !lanes) | (src & lanes); + match (pos, bits == after) { + (Some(i), true) => { e.forced.swap_remove(i); } + (Some(i), false) => e.forced[i].2 = bits, + (None, false) => e.forced.push((eidx, dw, bits)), + (None, true) => {} + } + if !e.ce && !self.l1d_forced(eidx) { + let mut tag: L1DTag = self.dc.get_tag(eidx); + tag.ptag &= !L1DTag::SLOW; + self.dc.set_tag(eidx, tag); + } + } + + #[cold] + fn read_slow(&self, virt_addr: u64, phys_addr: u64) -> BusRead64 { + let eidx = self.dc.get_index(virt_addr) | (self.perr().way as usize) << Self::DC_NUM_LINES_SHIFT; + if let Some(status) = self.l1d_parity_check(eidx, virt_addr, phys_addr, lane_mask(virt_addr, SIZE)) { + return BusRead64 { status, data: 0 }; + } + #[cfg(feature = "tcache")] + let result = self.tc_read::(phys_addr); + #[cfg(not(feature = "tcache"))] + let result = self.dc.dc_read::(Self::dc_data_addr(eidx, virt_addr)); + BusRead64::ok(result) + } + + #[cold] + fn write_slow(&self, virt_addr: u64, phys_addr: u64, val: u64, mask: u64) -> u32 { + let eidx = self.dc.get_index(virt_addr) | (self.perr().way as usize) << Self::DC_NUM_LINES_SHIFT; + let before = self.l1d_dword(eidx, virt_addr, phys_addr); + let merged = (before & !mask) | (val & mask); + #[cfg(feature = "tcache")] + { + self.tc_write::<8>(phys_addr & !7, merged); + self.tc_invalidate_l2_code(phys_addr); + } + #[cfg(not(feature = "tcache"))] + self.dc.dc_write::<8>(Self::dc_data_addr(eidx, virt_addr & !7), merged); + self.mark_l1d_dirty(eidx); + let lanes = (0..8).fold(0u8, |l, i| if (mask >> (8 * i)) & 0xff != 0 { l | 1 << i } else { l }); + self.l1d_parity_store(eidx, virt_addr, phys_addr, before, lanes); + BUS_OK + } + + /// The doubleword mask for a `SIZE`-byte store at `virt_addr`, and the + /// value shifted into place. + fn store_mask(virt_addr: u64, val: u64) -> (u64, u64) { + if SIZE == 8 { return (u64::MAX, val); } + let shift = (8 - SIZE - (virt_addr & 7) as usize) * 8; + let bits = (1u64 << (SIZE * 8)) - 1; + (bits << shift, (val & bits) << shift) + } + + /// The check bits Index_Load_Tag loads into CP0 ECC: those of the + /// doubleword vAddr[LINEBITS:3] selects in the indexed line (R4000 UM + /// A-42). Secondary: data ECC, with any CE-forced bits XOR'd in. Primary + /// data: byte parity, or the forced bits. Primary instruction: byte parity + /// on the R4400, one bit per word on the R5000 (VR5000 UM Table 7-8). + fn ilt_ecc(&self, is_icache: bool, is_l2: bool, idx: usize, virt_addr: u64) -> u32 { + if is_l2 { + let tag: L2Tag = self.l2.get_tag(idx); + let off = virt_addr as usize & Self::L2_LINE_MASK & !7; + let phys = l2_tag_to_phys(tag, ((idx << Self::L2_LINE_SHIFT) | off) as u64); + #[cfg(feature = "tcache")] + let data = self.downstream.read64(phys as u32).data; + #[cfg(not(feature = "tcache"))] + let data = self.l2.data()[(idx << Self::L2_CHUNKS_PER_LINE_SHIFT) + (off >> 3)]; + return (l2_data_ecc(data) ^ self.l2_xor(phys)) as u32; + } + if is_icache { + if let Some(&(_, forced)) = self.perr().ic_forced.iter().find(|f| f.0 == idx) { + return forced as u32; + } + let k = (virt_addr as usize & Self::IC_LINE_MASK & !7) >> 2; + let (hi, lo) = (self.l1i_word(idx, k), self.l1i_word(idx, k + 1)); + return if Self::IS_R5K { + (lo.count_ones() & 1) | (hi.count_ones() & 1) << 1 + } else { + byte_parity((hi as u64) << 32 | lo as u64) as u32 + }; + } + let tag: L1DTag = self.dc.get_tag(idx); + let phys = l1d_tag_to_phys(tag, ((idx & Self::DC_NUM_LINES_MASK) << Self::DC_LINE_SHIFT) as u64) + | (virt_addr & Self::DC_LINE_MASK as u64 & !7); + let dw = ((virt_addr as usize & Self::DC_LINE_MASK) >> 3) as u8; + let forced = self.perr().forced.iter().find(|f| f.0 == idx && f.1 == dw).map(|f| f.2); + forced.unwrap_or_else(|| byte_parity(self.l1d_dword(idx, virt_addr, phys))) as u32 + } + + /// Slot of the secondary doubleword holding `phys_addr`. + fn l2_slot(phys_addr: u64) -> usize { + (phys_addr as usize & (L2_CACHE_SIZE - 1)) >> 3 + } + + /// The bits XOR'd into the check bits of the doubleword at `phys_addr`. + fn l2_xor(&self, phys_addr: u64) -> u8 { + let e = self.perr(); + if e.l2_xor_live == 0 { return 0; } + e.l2_xor[Self::l2_slot(phys_addr)] + } + + fn set_l2_xor(&self, phys_addr: u64, v: u8) { + let e = self.perr(); + if e.l2_xor.is_empty() { + if v == 0 { return; } + e.l2_xor = vec![0; L2_CACHE_SIZE / 8]; + } + let slot = &mut e.l2_xor[Self::l2_slot(phys_addr)]; + e.l2_xor_live = e.l2_xor_live + (v != 0) as usize - (*slot != 0) as usize; + *slot = v; + } + + /// Recomputed check bits for `len` bytes of secondary data at `phys_addr`: + /// a write into L2, a fill or an invalidate. + #[inline] + fn clear_l2_xor(&self, phys_addr: u64, len: usize) { + if !ECC || self.perr().l2_xor_live == 0 { return; } + let base = phys_addr & !7; + for i in 0..len / 8 { + self.set_l2_xor(base + 8 * i as u64, 0); + } + } + + /// A primary refill from L2 checks the ECC of the `len`-byte line it + /// transfers. Returns the Cache Error if any doubleword carries forced bits + /// and errors are being detected. + fn l2_refill_check(&self, virt_addr: u64, phys_addr: u64, len: usize, data: bool) -> Option { + if !ECC { return None; } + let e = self.perr(); + if e.l2_xor_live == 0 || !e.detect || !HAS_L2 || self.hit_l2(phys_addr).is_none() { + return None; + } + let base = phys_addr & !(len as u64 - 1); + (0..len / 8).find(|&i| self.l2_xor(base + 8 * i as u64) != 0)?; + self.perr().cacheerr = if data { CACHEERR_ER } else { 0 } | CACHEERR_EC | CACHEERR_ED + | (phys_addr as u32 & 0x003F_FFF8) | ((virt_addr >> 12) as u32 & 7); + Some(EXEC_CACHE_ERR) + } + + /// A D-cache write-back done by a CACHE op. With CE set, the ECC register + /// is XOR'd into the check bits of the doubleword the op addresses as the + /// line is written to L2 (R4000 UM A-45). + fn l1d_writeback_op(&self, eidx: usize, virt_addr: u64, cascade: bool) { + if !ECC { + self.writeback_l1d_line(eidx, cascade); + return; + } + let tag: L1DTag = self.dc.get_tag(eidx); + let phys = l1d_tag_to_phys(tag, ((eidx & Self::DC_NUM_LINES_MASK) << Self::DC_LINE_SHIFT) as u64) + | (virt_addr & Self::DC_LINE_MASK as u64 & !7); + let written = tag.is_valid() && tag.dirty; + self.writeback_l1d_line(eidx, cascade); + let e = self.perr(); + if written && e.ce && HAS_L2 && self.hit_l2(phys).is_some() { + let ecc = e.ecc; + self.set_l2_xor(phys, ecc); + } + } + + fn fill_l1d_line_raw(&self, virt_addr: u64, phys_addr: u64) -> u32 { #[cfg(feature = "tcache")] let transparent = true; #[cfg(not(feature = "tcache"))] @@ -2728,6 +3223,139 @@ impl> 2; + let src = &self.l2.instrs.get()[(l2_idx << Self::L2_INSTR_SHIFT) + word..][..n]; + let dst = &mut unsafe { &mut *self.ic_raw.get() }[eidx * n..][..n]; + for (d, s) in dst.iter_mut().zip(src) { + *d = s.raw; + } + } + + /// Word `i` of L1-I line `eidx`, as the I-cache holds it. + fn l1i_word(&self, eidx: usize, i: usize) -> u32 { + if Self::IS_R5K { + self.ic_instrs.get()[(eidx << Self::IC_INSTR_SHIFT) + i].raw + } else { + unsafe { (&*self.ic_raw.get())[eidx * Self::IC_INSTRS_PER_LINE + i] } + } + } + + /// Hit_Writeback(I): write L1-I line `eidx` back to L2 if L2 holds the + /// line, otherwise to memory. + /// + /// Both manuals define it (op 6 on the I-cache: "data is written back + /// unconditionally"). IDE's `icache2` depends on it: it fills the I-cache, + /// changes memory under it through the D-cache, and expects the write-back + /// to put the I-cache's words back. + fn writeback_l1i_line(&self, eidx: usize, virt_addr: u64, line_base: u64) { + let n = Self::IC_INSTRS_PER_LINE; + // A line a CE Fill gave forced parity fails its check here, which is + // how IDE `icache1` observes it (R4000 UM Table 16-1: primary cache + // to secondary, parity checked, trap on error). + let e = self.perr(); + if ECC && e.detect { + if let Some(&(_, forced)) = e.ic_forced.iter().find(|f| f.0 == eidx) { + let bad = (0..n / 2).any(|c| { + let dw = (self.l1i_word(eidx, 2 * c) as u64) << 32 | self.l1i_word(eidx, 2 * c + 1) as u64; + byte_parity(dw) != forced + }); + if bad { + e.cacheerr = CACHEERR_ED | (line_base as u32 & 0x003F_FFF8) | ((virt_addr >> 12) as u32 & 7); + e.op_status = EXEC_CACHE_ERR; + return; + } + } + } + let mut chunks = [0u64; 16]; + let chunks = &mut chunks[..n / 2]; + for (c, chunk) in chunks.iter_mut().enumerate() { + *chunk = (self.l1i_word(eidx, 2 * c) as u64) << 32 | self.l1i_word(eidx, 2 * c + 1) as u64; + } + + let Some(l2_idx) = self.hit_l2(line_base) else { + // write_block bumps the jitv2 page generations it spans. + self.downstream.write_block(line_base as u32, chunks); + return; + }; + // tcache: RAM is the store, so the words go there; dropping + // `has_code` makes the next L1-I fill re-read them. + #[cfg(feature = "tcache")] + { + self.downstream.write_block(line_base as u32, chunks); + self.tc_invalidate_l2_code(line_base); + } + #[cfg(not(feature = "tcache"))] + { + let off = ((line_base as usize) & Self::L2_LINE_MASK) >> 3; + let base = (l2_idx << Self::L2_CHUNKS_PER_LINE_SHIFT) + off; + self.l2.data_mut()[base..base + chunks.len()].copy_from_slice(chunks); + // Keep the decode slots in step with L2 data, as + // `writeback_l1d_line` does. A whole I-cache line is rewritten and + // fusion never crosses one, so clearing the flags drops every + // fused pair this line took part in. + if !Self::IS_R5K { + let instrs = self.l2.instrs.get_mut(); + let ibase = (l2_idx << Self::L2_INSTR_SHIFT) + off * 2; + for i in 0..n { + let slot = &mut instrs[ibase + i]; + slot.raw = self.l1i_word(eidx, i); + slot.flags = FLAG_NOT_DECODED; + } + } + } + self.clear_l2_xor(line_base, n * 4); + let mut tag: L2Tag = self.l2.get_tag(l2_idx); + let cs = match tag.cs() { + L2_CS_CLEAN_EXCLUSIVE => L2_CS_DIRTY_EXCLUSIVE, + L2_CS_SHARED => L2_CS_DIRTY_SHARED, + cs => cs, + }; + tag.set_cs(cs); + self.l2.set_tag(l2_idx, tag); + } + + /// After a snapshot restore: refill the words of every valid L1-I line, + /// which snapshots do not carry. Without this a restored valid tag served + /// whatever the empty slots held, zeros (NOPs) on a fresh cache, until the + /// line was evicted. + fn rebuild_l1i_words(&self) { + let n = Self::IC_INSTRS_PER_LINE; + for eidx in 0..IC_WAYS * Self::IC_NUM_LINES { + let tag: L1ITag = self.ic.get_tag(eidx); + if !tag.is_valid() { + continue; + } + let set = eidx & Self::IC_NUM_LINES_MASK; + let line_base = tag.line_addr() | (((set << Self::IC_LINE_SHIFT) as u64) & L1_INDEX_MASK); + // The L2 decode slots hold the line if L2 does, except under + // tcache, where only an instruction-origin fill (`has_code`) + // populates them. + let l2_has_words = self.hit_l2(line_base).is_some_and(|l2_idx| { + cfg!(not(feature = "tcache")) || self.l2.get_tag(l2_idx).has_code() + }); + if !Self::IS_R5K && l2_has_words { + self.snapshot_l1i_line(eidx, line_base); + continue; + } + for i in 0..n { + let r = self.downstream.read32((line_base + 4 * i as u64) as u32); + let w = if r.is_ok() { r.data } else { 0 }; + if Self::IS_R5K { + let slot = &mut self.ic_instrs.get_mut()[(eidx << Self::IC_INSTR_SHIFT) + i]; + slot.raw = w; + slot.flags = FLAG_NOT_DECODED; + } else { + unsafe { (&mut *self.ic_raw.get())[eidx * n + i] = w }; + } + } + } + } + /// Return the eidx of the L1-D way that holds `phys_addr`, or `None`. #[inline] fn hit_l1d(&self, virt_addr: u64, phys_addr: u64) -> Option { @@ -2735,7 +3363,7 @@ impl FetchInstrResult { if Self::IS_R5K { #[cfg(feature = "debug_cache")] @@ -3116,7 +3747,10 @@ impl 1 { return BusRead64 { status: r, data: 0 }; } + if r > 1 { + if r == L1D_SLOW { return self.read_slow::(virt_addr, phys_addr); } + return BusRead64 { status: r, data: 0 }; + } } #[cfg(not(feature = "lightning"))] if devlog_is_active(LogModule::L1d) && devlog_mask(LogModule::L1d) & CACHE_LOG_HIT != 0 { @@ -3138,7 +3772,10 @@ impl 1 { return BusRead64 { status: way, data: 0 }; } + if way > 1 { + if way == L1D_SLOW { return self.read_slow::(virt_addr, phys_addr); } + return BusRead64 { status: way, data: 0 }; + } let dc_eidx = self.dc.get_index(virt_addr) | (way as usize) << Self::DC_NUM_LINES_SHIFT; #[cfg(not(feature = "lightning"))] if devlog_is_active(LogModule::L1d) && devlog_mask(LogModule::L1d) & CACHE_LOG_HIT != 0 { @@ -3178,7 +3815,13 @@ impl 1 { return r; } + if r > 1 { + if r == L1D_SLOW { + let (mask, v) = Self::store_mask::(virt_addr, val); + return self.write_slow(virt_addr, phys_addr, v, mask); + } + return r; + } } #[cfg(not(feature = "lightning"))] if devlog_is_active(LogModule::L1d) && devlog_mask(LogModule::L1d) & CACHE_LOG_HIT != 0 { @@ -3213,7 +3856,13 @@ impl 1 { return way; } + if way > 1 { + if way == L1D_SLOW { + let (mask, v) = Self::store_mask::(virt_addr, val); + return self.write_slow(virt_addr, phys_addr, v, mask); + } + return way; + } let dc_eidx = self.dc.get_index(virt_addr) | (way as usize) << Self::DC_NUM_LINES_SHIFT; #[cfg(not(feature = "lightning"))] if devlog_is_active(LogModule::L1d) && devlog_mask(LogModule::L1d) & CACHE_LOG_HIT != 0 { @@ -3250,7 +3899,10 @@ impl 1 { return r; } + if r > 1 { + if r == L1D_SLOW { return self.write_slow(virt_addr, phys_addr, val, mask); } + return r; + } } #[cfg(feature = "tcache")] { @@ -3270,7 +3922,10 @@ impl 1 { return way; } + if way > 1 { + if way == L1D_SLOW { return self.write_slow(virt_addr, phys_addr, val, mask); } + return way; + } let dc_eidx = self.dc.get_index(virt_addr) | (way as usize) << Self::DC_NUM_LINES_SHIFT; #[cfg(feature = "tcache")] { @@ -3366,8 +4021,14 @@ impl> 14) as usize & 1) << Self::IC_NUM_LINES_SHIFT) } else { set } @@ -3400,6 +4061,10 @@ impl 7, _ => 0, }; - ((tag.ptag() << 13) | (state << 10) | (tag.pidx() << 7)) as u64 + let fields = (state << 22) | (tag.pidx() << 19) | tag.ptag(); + // ECC[6:0]: generated, as the R4400 does on every tag write. + // The R10000 tag has its own layout and no such field. + let ecc = if MODEL == model::R4400 { l2_tag_ecc(fields) } else { 0 }; + ((tag.ptag() << 13) | (state << 10) | (tag.pidx() << 7) | ecc) as u64 } else if is_icache { let tag: L1ITag = self.ic.get_tag(idx); let raw_ptag = (tag.ptag >> L1_PTAG_SHIFT) as u32 & L1_PTAG_MASK; - if Self::IS_R5K { - // R5000 L1-I TagLo: [31:8] ptag [7] L (lock) [6] V - // Not the R4400 2-bit PState. We model no lock bit, so L=0. - let v = if tag.is_valid() { 1u32 << 6 } else { 0 }; - ((raw_ptag << 8) | v) as u64 - } else { - // R4400 L1-I TagLo: [31:8] raw_ptag [7:6] pstate (2=valid, 0=invalid) - let pstate = if tag.is_valid() { 2u32 } else { 0u32 }; - ((raw_ptag << 8) | (pstate << 6)) as u64 - } + // L1-I TagLo, R4400 and R5000 alike: [31:8] raw_ptag + // [7:6] pstate (2=valid, 0=invalid) [0] P. + // The V bit reads back as PState 2 (PCLEANEXCL) on both: + // IDE's icachetag requires exactly that after a fill, and + // the Indy script runs it on every CPU it shipped with. + // P is even parity over PTag and V on the R4400 (UM Figure + // 11-3), over PTag alone on the R5000 (VR5000 UM 12-3). + let pstate = if tag.is_valid() { 2u32 } else { 0u32 }; + let p = even_parity(raw_ptag) + ^ if Self::IS_R5K { 0 } else { tag.is_valid() as u32 }; + ((raw_ptag << 8) | (pstate << 6) | p) as u64 } else { let tag: L1DTag = self.dc.get_tag(idx); let raw_ptag = (tag.ptag >> L1_PTAG_SHIFT) as u32 & L1_PTAG_MASK; - if Self::IS_R5K { - // R5000 L1-D TagLo: [31:8] ptag [7] D (dirty) [6] V - // Discrete bits, not a 2-bit MESI PState — the R5000 L1 has no - // bus snooping and therefore no Shared/Exclusive distinction. - // Emitting the R4400 encoding here made a valid-clean line read - // back as D=1,V=0, i.e. *invalid* to R5000 software. (CACHE-1.) - let valid = tag.cs as u32 != L1D_CS_INVALID; - let v = if valid { 1u32 << 6 } else { 0 }; - let d = if valid && (tag.dirty - || tag.cs as u32 == L1D_CS_DIRTY_EXCLUSIVE) { 1u32 << 7 } else { 0 }; - ((raw_ptag << 8) | d | v) as u64 - } else { - // R4400 L1-D TagLo: [31:8] raw_ptag [7:6] pstate - // dirty=true promotes CleanExclusive→DirtyExclusive in TagLo output - let pstate = match tag.cs as u32 { - L1D_CS_INVALID => 0u32, - L1D_CS_SHARED => 1u32, - L1D_CS_CLEAN_EXCLUSIVE => if tag.dirty { 3u32 } else { 2u32 }, - L1D_CS_DIRTY_EXCLUSIVE => 3u32, - _ => 0u32, - }; - ((raw_ptag << 8) | (pstate << 6)) as u64 - } + // L1-D TagLo, R4400 and R5000 alike: [31:8] raw_ptag + // [7:6] pstate [0] P. Dirty reads back as DirtyExclusive (3): + // IRIX's IP22 parity recovery (mcparity.c) takes PState > + // PCLEANEXCL to mean dirty on every IP22 CPU, R5000 included. + // The R5000 never produces Shared (1); it has no snooping. + // W and its parity W' are not part of TagLo. + let pstate = match tag.cs as u32 { + L1D_CS_INVALID => 0u32, + L1D_CS_SHARED => 1u32, + L1D_CS_CLEAN_EXCLUSIVE => if tag.dirty { 3u32 } else { 2u32 }, + L1D_CS_DIRTY_EXCLUSIVE => 3u32, + _ => 0u32, + }; + // P: even parity over PTag and CS on the R4400 (UM Figure + // 11-4), over PTag alone on the R5000 (VR5000 UM 12-4). + let p = even_parity(raw_ptag) + ^ if Self::IS_R5K { 0 } else { even_parity(pstate) }; + ((raw_ptag << 8) | (pstate << 6) | p) as u64 } } @@ -3511,42 +4176,31 @@ impl> 8) & L1_PTAG_MASK; let ptag_line = (raw_ptag as u64) << L1_PTAG_SHIFT; // convert to line-base form let pstate = (tag_lo >> 6) & 0x3; - // R5000: bit 6 alone is Valid. R4400: any non-zero PState is valid. - let valid = if Self::IS_R5K { (tag_lo & (1 << 6)) != 0 } else { pstate != 0 }; + let valid = pstate != 0; if is_icache { // Evict existing line first to maintain L1I data pointer integrity. self.invalidate_l1i_line(idx, cascade); self.ic.set_tag(idx, if valid { L1ITag::valid(ptag_line) } else { L1ITag::default() }); } else { - // R5000 has no Shared state (no L1 snooping): a valid line is - // Clean- or Dirty-Exclusive per bit 7. - let (cs, dirty) = if Self::IS_R5K { - let d = (tag_lo & (1 << 7)) != 0; - if !valid { - (L1D_CS_INVALID as u8, false) - } else if d { - (L1D_CS_DIRTY_EXCLUSIVE as u8, true) - } else { - (L1D_CS_CLEAN_EXCLUSIVE as u8, false) - } - } else { - let cs = match pstate { - 0 => L1D_CS_INVALID as u8, - 1 => L1D_CS_SHARED as u8, - 2 => L1D_CS_CLEAN_EXCLUSIVE as u8, - 3 => L1D_CS_DIRTY_EXCLUSIVE as u8, - _ => L1D_CS_INVALID as u8, - }; - (cs, cs == L1D_CS_DIRTY_EXCLUSIVE as u8) + // The R5000 has no Shared state (no L1 snooping), so a + // stored 1 lands as CleanExclusive there. + let cs = match pstate { + 0 => L1D_CS_INVALID as u8, + 1 if Self::IS_R5K => L1D_CS_CLEAN_EXCLUSIVE as u8, + 1 => L1D_CS_SHARED as u8, + 2 => L1D_CS_CLEAN_EXCLUSIVE as u8, + _ => L1D_CS_DIRTY_EXCLUSIVE as u8, }; + let dirty = cs == L1D_CS_DIRTY_EXCLUSIVE as u8; // No writeback. Index_Store_Tag installs a tag and // discards whatever the line held — that is what makes // it usable to initialise a cache whose tags are @@ -3557,7 +4211,9 @@ impl { // same as C_FILL if is_icache { // Fill operation: L1I is virtually indexed, use virt_addr for index. - let _ = self.fill_l1i_line(virt_addr, phys_addr); + // With CE set the ECC register replaces the computed + // parity of the filled line (R4000 UM ch. 5, VR5000 UM 7.2.9). + let r = self.fill_l1i_line(virt_addr, phys_addr); + if ECC && r <= 1 && self.perr().ce { + if let Some(eidx) = self.hit_l1i(virt_addr, phys_addr) { + let ecc = self.perr().ecc; + self.perr().ic_forced.push((eidx, ecc)); + } + } else if r > 1 && r & EXEC_IS_CACHE_ERR_BIT != 0 { + self.perr().op_status = r; + } } else if is_l2 { if let Some(idx) = self.hit_l2(phys_addr) { // Triton C_INVPAGE (SD, hit op): invalidate all L2 lines in the page. @@ -3623,7 +4292,7 @@ impl { - if !is_icache { + if is_icache { + if let Some(eidx) = self.hit_l1i(virt_addr, phys_addr) { + self.writeback_l1i_line(eidx, virt_addr, phys_addr & !(Self::IC_LINE_MASK as u64)); + } + } else { if is_l2 { if let Some(idx) = self.hit_l2(phys_addr) { self.writeback_l2_line(idx); } } else { if let Some(eidx) = self.hit_l1d(virt_addr, phys_addr) { - self.writeback_l1d_line(eidx, cascade); + self.l1d_writeback_op(eidx, virt_addr, cascade); } } } @@ -3666,6 +4339,35 @@ impl u32 { + self.perr().cacheerr + } + + fn take_op_status(&self) -> u32 { + if !ECC { return 0; } + std::mem::take(&mut self.perr().op_status) + } + + fn cache_op_ecc(&self) -> u32 { + self.perr().ecc_out + } + fn get_config(&self, cache_target: u32) -> (usize, usize) { match cache_target { CACH_PI => (IC_SIZE, IC_LINE), @@ -3746,14 +4448,14 @@ impl "Invalid", L1D_CS_SHARED => "Shared", @@ -4124,7 +4826,7 @@ impl Vec { (0..num_sets.div_ceil(32)).map(|i| { @@ -4174,7 +4876,7 @@ impl 2 to mean dirty, so a clean line must not + /// read 3. /// - /// Emitting the R4400 encoding on R5000 made a valid *clean* line read back - /// as D=1,V=0 — i.e. **invalid** to R5000 software — and a store of the - /// R5000 "valid, clean" pattern (0x40) was decoded as PState=1 (Shared). + /// An earlier "R5000 is D(7)/V(6)" encoding read a clean line back as + /// `0x40`, which fails `icachetag`. #[test] - fn taglo_valid_clean_line_reads_back_valid_on_both_models() { - let mem = Arc::new(Memory::new(MEM_MB)); - let phys: u32 = 0x3000; - let va = kseg0(phys); + fn taglo_filled_lines_read_back_the_r4000_pstates_on_both_models() { + fn check>>(name: &str, ways: u64) { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache: C = make_cache_of(mem.clone()); + let phys: u32 = 0x3000; + let va = kseg0(phys); + // Index ops pick the way with VA[14] on the R5000; probe each. + let pstate_of = |sel: u32| -> u64 { + (0..ways) + .map(|w| cache.cache_op(C_ILT | sel, va | (w << 14), 0)) + .find(|t| t >> 8 == (phys >> L1_PTAG_SHIFT) as u64) + .map_or(0, |t| (t >> 6) & 3) + }; - // R4400: a filled (clean) line reports PState=2 (CleanExclusive), V implied. - { - let cache: R4400Cache = make_cache_of(mem.clone()); - let _ = cache.read::<4>(va, phys as u64); // fill the line clean - let idx = cache.dc.get_index(va); - let tag_lo = cache.cache_op(C_ILT | CACH_PD, va, idx as u64); - assert_eq!((tag_lo >> 6) & 0x3, 2, - "R4400 clean line must report PState=CleanExclusive"); - } + let _ = cache.fetch(va, phys as u64); + assert_eq!(pstate_of(CACH_PI), 2, "{name}: a filled I-cache line reads PState 2"); - // R5000: the same line must report V=1 (bit 6) and D=0 (bit 7). - { - let cache: R5000Cache = make_cache_of(mem.clone()); let _ = cache.read::<4>(va, phys as u64); - // 2-way: the fill picked a way via LRU; probe both and take the valid one. - let set = cache.dc.get_index(va); - let nls = R5000Cache::DC_NUM_LINES_SHIFT as usize; - let tag_lo = (0..R5000Cache::DC_WAYS) - .map(|w| cache.cache_op(C_ILT | CACH_PD, va, (set | (w << nls)) as u64)) - .find(|t| t & (1 << 6) != 0) - .expect("R5000 clean line must set V (bit 6) in TagLo — with the \ - R4400 PState encoding it reads back as D=1,V=0, i.e. invalid"); - assert_eq!(tag_lo & (1 << 7), 0, - "a clean line must not set D (bit 7) on R5000"); + assert_eq!(pstate_of(CACH_PD), 2, "{name}: a clean D-cache line reads CleanExclusive"); + + let _ = cache.write::<4>(va, phys as u64, 0x1234_5678); + assert_eq!(pstate_of(CACH_PD), 3, "{name}: a dirty D-cache line reads DirtyExclusive"); } + check::("R4400", 1); + check::("R5000", 2); } /// `C_IST` must be the exact inverse of `C_ILT` for whichever model is /// active: store a tag, load it back, and get the same state bits. /// - /// Note this is a *symmetry* check and passes even when both halves share - /// the same wrong encoding — `taglo_valid_clean_line_reads_back_valid_on_both_models` - /// and `taglo_r5000_decodes_hardware_bit_patterns` are what pin the absolute - /// bit positions. Kept because an asymmetric encode/decode pair is its own - /// distinct bug class. + /// A *symmetry* check: it passes even when both halves share the same + /// wrong encoding, which is exactly how the D/V encoding survived. The + /// absolute bit positions are pinned by + /// `taglo_filled_lines_read_back_the_r4000_pstates_on_both_models` and + /// `taglo_store_decodes_what_ide_and_irix_write`. Kept because an + /// asymmetric encode/decode pair is its own bug class — and IDE's way + /// pinning on R4600/R5000 stores back exactly what it loaded. #[test] fn taglo_store_load_round_trips_on_both_models() { - let mem = Arc::new(Memory::new(MEM_MB)); - let phys: u32 = 0x5000; - let va = kseg0(phys); - - // R4400: PState=3 (DirtyExclusive) must survive the round trip. - { - let cache: R4400Cache = make_cache_of(mem.clone()); - let idx = cache.dc.get_index(va) as u64; - let raw_ptag = (phys >> L1_PTAG_SHIFT) & L1_PTAG_MASK; - let written = (raw_ptag << 8) | (3 << 6); - cache.cache_op(C_IST | CACH_PD, va, written as u64); - let read_back = cache.cache_op(C_ILT | CACH_PD, va, idx); - assert_eq!((read_back >> 6) & 0x3, 3, "R4400 DirtyExclusive must round-trip"); - assert_eq!(read_back >> 8, raw_ptag as u64, "physical tag must round-trip"); - } - - // R5000: valid+dirty is D=1,V=1 (0xC0). Under the old code this was - // decoded as PState=3 by luck, but valid+clean (0x40) was decoded as - // Shared — so assert the clean case too, which is the one that broke. - { - let cache: R5000Cache = make_cache_of(mem.clone()); - let set = cache.dc.get_index(va); - let nls = R5000Cache::DC_NUM_LINES_SHIFT as usize; - let idx = (set | (0 << nls)) as u64; + fn check>>(name: &str) { + let cache: C = make_cache_of(Arc::new(Memory::new(MEM_MB))); + let phys: u32 = 0x5000; + let va = kseg0(phys); let raw_ptag = (phys >> L1_PTAG_SHIFT) & L1_PTAG_MASK; - - for (d_bit, label) in [(1u32 << 7, "dirty"), (0, "clean")] { - let written = (raw_ptag << 8) | d_bit | (1 << 6); // V=1 - cache.cache_op(C_IST | CACH_PD, va, written as u64); - let read_back = cache.cache_op(C_ILT | CACH_PD, va, idx); - assert_ne!(read_back & (1 << 6), 0, - "R5000 {label} line must read back V=1"); - assert_eq!(read_back & (1 << 7), d_bit as u64, - "R5000 {label} line must round-trip its D bit"); - assert_eq!(read_back >> 8, raw_ptag as u64, "physical tag must round-trip"); + for (sel, pstate) in [(CACH_PI, 2u32), (CACH_PD, 2), (CACH_PD, 3)] { + cache.cache_op(C_IST | sel, va, ((raw_ptag << 8) | (pstate << 6)) as u64); + let read_back = cache.cache_op(C_ILT | sel, va, 0); + assert_eq!((read_back >> 6) & 3, pstate as u64, "{name} sel {sel}: PState"); + assert_eq!(read_back >> 8, raw_ptag as u64, "{name} sel {sel}: physical tag"); } } + check::("R4400"); + check::("R5000"); } - /// R5000 `C_IST` must interpret the bit patterns real R5000 firmware writes, - /// independently of what `C_ILT` would emit. + /// `C_IST` decodes the patterns real software stores. /// - /// This is the half a pure round-trip test cannot catch: with the R4400 - /// PState decoder, TagLo=0x40 ("valid, clean" to R5000 firmware) was read as - /// PState=1 = *Shared*, and TagLo=0x00 vs 0x80 were both "invalid" so a - /// dirty-invalid distinction was lost. Asserting observable *behaviour* - /// (does the line hit? does a writeback occur?) rather than re-reading the - /// tag keeps this honest. + /// IDE's `set_tag` writes PCLEANEXCL (`0x80`) to validate an I-cache line + /// and PDIRTYEXCL (`0xC0`) for a dirty D-cache line, on every IP22 CPU. + /// The old D/V encoding took `0x80` for *invalid* on the R5000. The R5000 + /// has no Shared state, so a stored 1 is taken as CleanExclusive there. #[test] - fn taglo_r5000_decodes_hardware_bit_patterns() { - let mem = Arc::new(Memory::new(MEM_MB)); + fn taglo_store_decodes_what_ide_and_irix_write() { + let cache: R5000Cache = make_cache_of(Arc::new(Memory::new(MEM_MB))); let phys: u32 = 0x7000; let va = kseg0(phys); - let cache: R5000Cache = make_cache_of(mem.clone()); - - let set = cache.dc.get_index(va); - let nls = R5000Cache::DC_NUM_LINES_SHIFT as usize; - let idx = (set | (0 << nls)) as u64; let raw_ptag = (phys >> L1_PTAG_SHIFT) & L1_PTAG_MASK; + let pstate_after = |sel: u32, pattern: u32| { + cache.cache_op(C_IST | sel, va, ((raw_ptag << 8) | pattern) as u64); + (cache.cache_op(C_ILT | sel, va, 0) >> 6) & 3 + }; - // V=0 (whatever bit 7 says) must be Invalid: a subsequent read must miss - // and refill rather than returning whatever the tag pointed at. - cache.cache_op(C_IST | CACH_PD, va, ((raw_ptag << 8) | (1 << 7)) as u64); - let after_invalid = cache.cache_op(C_ILT | CACH_PD, va, idx); - assert_eq!(after_invalid & (1 << 6), 0, - "TagLo with V=0 must store an Invalid line on R5000, \ - regardless of the D bit"); - - // V=1, D=0 — "valid, clean". Must be valid (this is the pattern the - // R4400 decoder mis-read as Shared) and must NOT be dirty. - cache.cache_op(C_IST | CACH_PD, va, ((raw_ptag << 8) | (1 << 6)) as u64); - let clean = cache.cache_op(C_ILT | CACH_PD, va, idx); - assert_ne!(clean & (1 << 6), 0, "V=1,D=0 must store a valid line"); - assert_eq!(clean & (1 << 7), 0, "V=1,D=0 must not be dirty"); - - // V=1, D=1 — "valid, dirty". - cache.cache_op(C_IST | CACH_PD, va, ((raw_ptag << 8) | (1 << 7) | (1 << 6)) as u64); - let dirty = cache.cache_op(C_ILT | CACH_PD, va, idx); - assert_ne!(dirty & (1 << 6), 0, "V=1,D=1 must be valid"); - assert_ne!(dirty & (1 << 7), 0, "V=1,D=1 must be dirty"); + assert_eq!(pstate_after(CACH_PD, 0x00), 0, "D-cache 0x00: invalid"); + assert_eq!(pstate_after(CACH_PD, 0x40), 2, "D-cache 0x40: no Shared, so CleanExclusive"); + assert_eq!(pstate_after(CACH_PD, 0x80), 2, "D-cache 0x80 PCLEANEXCL: valid, clean"); + assert_eq!(pstate_after(CACH_PD, 0xc0), 3, "D-cache 0xC0 PDIRTYEXCL: valid, dirty"); + assert_eq!(pstate_after(CACH_PI, 0x80), 2, "I-cache 0x80 PCLEANEXCL: valid"); + assert_eq!(pstate_after(CACH_PI, 0x00), 0, "I-cache 0x00: invalid"); } /// L1D random read/write: 1M word operations against a shadow copy. @@ -4881,6 +5544,364 @@ mod tests { assert_eq!(tag_lo_read, tag_lo_read2, "ILT/IST(SD) round-trip mismatch"); } + /// `Index_Store_Tag(SD)` writes the line its *index address* names, not the + /// line its TagLo value happens to name. + /// + /// Both manuals say the index op takes the block from the CACHE + /// instruction's address (R4000 UM A-42, VR5000 UM §3.4). The executor + /// passes TagHi:TagLo in `phys_addr` for C_IST, and the L2 index was once + /// computed from that, so a store landed in whichever set TagLo[19:7] + /// selected. `cache_op_ilt_ist_l2` above never saw it: it stores back the + /// tag it just read, so the read-back hits the untouched original. + #[test] + fn l2_index_store_tag_indexes_by_address_not_by_taglo() { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache = make_cache(mem.clone()); + + let line = R4400Cache::L2_LINE as u64; + let target_set = 0x123u64; + let index_addr = 0x8000_0000 | target_set * line; + // CleanExclusive, PIdx 5, STag 0x1abc. TagLo[19:7] — the bits the old + // code read as an index — name set 0xf25 instead. + let tag_lo: u64 = (0x1abc << 13) | (4 << 10) | (5 << 7); + let taglo_set = (tag_lo >> R4400Cache::L2_LINE_SHIFT) & (R4400Cache::L2_SIZE as u64 / line - 1); + assert_ne!(taglo_set, target_set, "test needs TagLo to name a different set"); + + cache.cache_op(C_IST | CACH_SD, index_addr, tag_lo); + + let fields = |v: u64| v & !0x7f; // ECC[6:0] is generated, not stored + assert_eq!(fields(cache.cache_op(C_ILT | CACH_SD, index_addr, index_addr)), tag_lo, + "tag did not land at the index address"); + let other = 0x8000_0000 | taglo_set * line; + assert_eq!(fields(cache.cache_op(C_ILT | CACH_SD, other, other)), 0, + "tag landed in the set TagLo names"); + } + + /// IDE `icache2`'s `icache_fill_writeback`, one line: the I-cache keeps the + /// words it was filled with, and Hit_Writeback(I) puts them back after + /// memory has changed under it. Hit_Writeback(I) was a no-op, so `icache2` + /// reported `Cache err: … Actual:0xffffffff` on every word. + /// + /// Not under tcache: there a cached store goes straight to RAM, so the + /// I-cache fill already sees the new words — the same trade + /// `cache/hit_inv_discards` makes. + #[test] + #[cfg(not(feature = "tcache"))] + fn icache_hit_writeback_restores_the_words_it_was_filled_with() { + fn check>>(name: &str) { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache: C = make_cache_of(mem.clone()); + let phys: u32 = 0x8000; + let va = kseg0(phys); + let words = C::IC_LINE as u32 / 4; + for i in 0..words { mem_write(&mem, phys + 4 * i, 0x1111_0000 + i); } + + // Dirty the D-cache with other words, fill the I-cache, then push + // the D-cache's words all the way to memory. + for i in 0..words { cache.write::<4>(va + 4 * i as u64, (phys + 4 * i) as u64, 0xdead_0000 + i as u64); } + cache.cache_op(C_HWBINV | CACH_PI, va, phys as u64); // Fill(I) + // IDE on R4600/R4700/R5000: pin the line into the way VA[14] + // selects — load its tag, invalidate it, store the tag back by + // index, Fill again. The Fill must refill the pinned way. + let tag = cache.cache_op(C_ILT | CACH_PI, va, 0); + cache.cache_op(C_HINV | CACH_PI, va, phys as u64); + cache.cache_op(C_IST | CACH_PI, va, tag); + cache.cache_op(C_HWBINV | CACH_PI, va, phys as u64); + cache.cache_op(C_HWBINV | CACH_PD, va, phys as u64); + cache.cache_op(C_HWB | CACH_SD, va, phys as u64); + assert_eq!(mem_read(&mem, phys), 0xdead_0000, "{name}: D-cache words reached memory"); + + cache.cache_op(C_HWB | CACH_PI, va, phys as u64); + cache.cache_op(C_HWB | CACH_SD, va, phys as u64); + for i in 0..words { + assert_eq!(mem_read(&mem, phys + 4 * i), 0x1111_0000 + i, + "{name}: word {i} is not what the I-cache was filled with"); + } + } + check::("R4400"); + check::("R5000"); + } + + /// A snapshot carries I-cache tags but not the words, so a restore must + /// rebuild them: a restored valid line is served without a refill. Before, + /// it was served from empty slots, i.e. a line of NOPs. + #[test] + fn restored_icache_lines_hold_their_instructions() { + fn check>>(name: &str) { + let mem = Arc::new(Memory::new(MEM_MB)); + let phys: u32 = 0x9000; + let va = kseg0(phys); + for i in 0..4 { mem_write(&mem, phys + 4 * i, 0x2400_0000 + i); } // addiu + let saved = { + let cache: C = make_cache_of(mem.clone()); + let _ = cache.fetch(va, phys as u64); + cache.save_cache_state() + }; + let cache: C = make_cache_of(mem.clone()); + cache.load_cache_state(&saved).unwrap(); + for i in 0..4u32 { + let r = cache.fetch(va + 4 * i as u64, (phys + 4 * i) as u64); + assert_eq!(unsafe { (*r.instr).raw }, 0x2400_0000 + i, "{name}: word {i}"); + } + } + check::("R4400"); + check::("R5000"); + } + + /// IDE `dcache1`, one word: with Status.CE set a store takes the ECC + /// register as its check bits, and the next load of it, with DE and ERL + /// clear, takes a Cache Error with CacheErr.ED set. Both CPUs implement + /// CE the same way (R4000 UM ch. 5, VR5000 UM 7.2.9). + #[cfg(feature = "cache-ecc")] + #[test] + fn forced_parity_raises_a_cache_error_on_the_next_load() { + fn check>>(name: &str) { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache: C = make_cache_of(mem.clone()); + let phys: u32 = 0xa000; + let va = kseg0(phys); + cache.set_error_control(false, true); + assert_eq!(cache.write::<4>(va, phys as u64, 0), BUS_OK); + + // IDE: set_ecc(0x55), SetSR(SR_FORCE_ON), store, load. + cache.set_cache_ecc(0x55); + cache.set_error_control(true, true); + assert_eq!(cache.write::<4>(va, phys as u64, 0x1010_1010), BUS_OK); + let r = cache.read::<4>(va, phys as u64); + assert_eq!(r.status, EXEC_CACHE_ERR, "{name}: load of a forced word"); + let err = cache.take_cache_err(); + assert_ne!(err & CACHEERR_ED, 0, "{name}: CacheErr.ED"); + assert_ne!(err & CACHEERR_ER, 0, "{name}: CacheErr.ER (data)"); + + // DE (or ERL) masks it: the load completes with the data. + cache.set_error_control(true, false); + assert_eq!(cache.read::<4>(va, phys as u64).data, 0x1010_1010, "{name}: DE masks"); + + // The other word of the doubleword was not stored with CE. + cache.set_error_control(false, true); + assert!(cache.read::<4>(va + 4, phys as u64 + 4).is_ok(), "{name}: untouched lanes"); + + // IDE's handler: clear CE, Hit_Invalidate, store again. + cache.cache_op(C_HINV | CACH_PD, va, phys as u64); + assert_eq!(cache.write::<4>(va, phys as u64, 0x1010_1010), BUS_OK); + assert!(cache.read::<4>(va, phys as u64).is_ok(), "{name}: invalidated, refilled"); + + // Rewriting a forced word with CE clear also repairs it. + cache.set_error_control(true, true); + assert_eq!(cache.write::<4>(va, phys as u64, 0), BUS_OK); + cache.set_error_control(false, true); + assert_eq!(cache.write::<4>(va, phys as u64, 7), BUS_OK); + assert_eq!(cache.read::<4>(va, phys as u64).data, 7, "{name}: repaired by a clean store"); + + } + check::("R4400"); + check::("R5000"); + } + + /// The parity machinery must leave the hit path alone: with CE clear and + /// nothing forced, no line carries the slow mark. + #[cfg(feature = "cache-ecc")] + #[test] + fn no_line_stays_marked_once_ce_clears_and_nothing_is_forced() { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache = make_cache(mem.clone()); + let phys: u32 = 0xb000; + let va = kseg0(phys); + cache.set_error_control(false, true); + let _ = cache.read::<4>(va, phys as u64); + cache.set_error_control(true, true); + let idx = cache.dc.get_index(va); + assert_ne!(cache.dc.get_tag(idx).ptag & L1DTag::SLOW, 0, "CE marks resident lines"); + cache.set_error_control(false, true); + assert!(cache.dc.get_tag(idx).matches_phys(phys as u64), "unmarked again"); + } + + /// IDE `icache1`, one line: with CE set, Fill(I) takes the ECC register as + /// the line's parity, and Hit_Writeback(I) then raises a Cache Error from + /// the CACHE instruction itself. With DE set it writes back normally; a + /// refill without CE clears it. + #[cfg(feature = "cache-ecc")] + #[test] + fn forced_icache_parity_faults_on_writeback() { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache = make_cache(mem.clone()); + let phys: u32 = 0xe000; + let va = kseg0(phys); + cache.set_cache_ecc(0x55); + cache.set_error_control(true, true); // SR_FORCE_ON + cache.cache_op(C_HWBINV | CACH_PI, va, phys as u64); // Fill(I) + cache.set_error_control(false, true); // SR_FORCE_OFF + cache.cache_op(C_HWB | CACH_PI, va, phys as u64); + assert_eq!(cache.take_op_status(), EXEC_CACHE_ERR); + assert_ne!(cache.take_cache_err() & CACHEERR_ED, 0); + assert_eq!(cache.take_op_status(), 0, "reading the status clears it"); + + cache.set_error_control(false, false); // DE: no fault + cache.cache_op(C_HWB | CACH_PI, va, phys as u64); + assert_eq!(cache.take_op_status(), 0); + + cache.set_error_control(false, true); + cache.cache_op(C_HINV | CACH_PI, va, phys as u64); + cache.cache_op(C_HWBINV | CACH_PI, va, phys as u64); // Fill, CE clear + cache.cache_op(C_HWB | CACH_PI, va, phys as u64); + assert_eq!(cache.take_op_status(), 0, "a refill without CE computes parity"); + } + + /// Fill(I) refills the way that already holds the line, not the LRU way: + /// IDE pins a line into the way VA[14] selects with Index_Store_Tag and + /// then fills it. + #[test] + fn icache_fill_refills_the_way_holding_the_line() { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache: R5000Cache = make_cache_of(mem.clone()); + let phys: u32 = 0x8000; + let va = kseg0(phys); + mem_write(&mem, phys, 0x2400_0042); + let raw_ptag = (phys >> L1_PTAG_SHIFT) & L1_PTAG_MASK; + let way1 = va | 1 << 14; + cache.cache_op(C_IST | CACH_PI, way1, ((raw_ptag << 8) | 0x80) as u64); + cache.cache_op(C_HWBINV | CACH_PI, va, phys as u64); // Fill(I) + let eidx1 = cache.ic.get_index(va) | 1 << R5000Cache::IC_NUM_LINES_SHIFT; + assert_eq!(cache.l1i_word(eidx1, 0), 0x2400_0042, "way 1 was refilled"); + let eidx0 = cache.ic.get_index(va); + assert!(!cache.ic.get_tag(eidx0).is_valid(), "way 0 untouched"); + } + + /// `L2_DATA_ECC_MASKS` is R4000 UM Figure 16-1, checked against what the + /// figure states: 27 ones per row with the check bit, weight-3 syndromes + /// except weight 5 for data bits 59, 58, 53, 52, 11, 10, 5 and 4, all + /// distinct, and the text's example (data bit 0 is `0001 0011`). + #[test] + fn l2_data_ecc_matrix_is_the_manuals() { + assert!(L2_DATA_ECC_MASKS.iter().all(|m| m.count_ones() + 1 == 27)); + let mut seen = std::collections::HashSet::new(); + for bit in 0..64 { + let syndrome = l2_data_ecc(1 << bit); + let weight = if [59, 58, 53, 52, 11, 10, 5, 4].contains(&bit) { 5 } else { 3 }; + assert_eq!(syndrome.count_ones(), weight, "data bit {bit}"); + assert!(seen.insert(syndrome), "data bit {bit} shares a syndrome"); + } + assert_eq!(l2_data_ecc(1), 0b0001_0011); + } + + /// IDE `scache1`, both halves, one doubleword. + /// + /// `scache_ecc_ram`: with CE set, a D-cache Index_Writeback_Invalidate XORs + /// the ECC register into the check bits of the doubleword it addresses as + /// the line goes to L2, and Index_Load_Tag(SD) reads them back into ECC. + /// `scache_ecc_intr`: with DE clear, refilling the primary from that line + /// takes a Cache Error with CacheErr.EC and ED set; Hit_Invalidate(SD) + /// clears it. + #[cfg(feature = "cache-ecc")] + #[test] + fn forced_secondary_ecc_reads_back_and_faults_on_refill() { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache = make_cache(mem.clone()); + let phys: u32 = 0xc000; + let va = kseg0(phys); + cache.set_error_control(false, false); // SR_NOERR: DE set + assert_eq!(cache.write::<4>(va, phys as u64, 0), BUS_OK); + cache.set_cache_ecc(0x55); + cache.set_error_control(true, false); // SR_FORCE_NOERR + cache.cache_op(C_IWBINV | CACH_PD, va, va); + cache.set_error_control(false, false); + let _ = cache.read::<4>(va, phys as u64); // DE: no fault + cache.set_cache_ecc(0); + cache.cache_op(C_ILT | CACH_SD, va, va); + assert_eq!(cache.cache_op_ecc(), 0x55, "data 0 has check bits 0, so ECC reads the forced pattern"); + // Other doublewords of the line were written back with computed bits. + cache.cache_op(C_ILT | CACH_SD, va + 8, va + 8); + assert_eq!(cache.cache_op_ecc(), 0); + + // scache_ecc_intr: refill with detection on. + cache.cache_op(C_HINV | CACH_PD, va, phys as u64); + cache.set_error_control(false, true); + let r = cache.read::<4>(va, phys as u64); + assert_eq!(r.status, EXEC_CACHE_ERR); + let err = cache.take_cache_err(); + assert_eq!(err & (CACHEERR_EC | CACHEERR_ED), CACHEERR_EC | CACHEERR_ED, "secondary, data field"); + + cache.cache_op(C_HINV | CACH_SD, va, phys as u64); + assert!(cache.read::<4>(va, phys as u64).is_ok(), "Hit_Invalidate(SD) clears it"); + } + + /// Index_Load_Tag(SD) on ordinary data loads the computed data ECC. + #[test] + #[cfg(all(feature = "cache-ecc", not(feature = "tcache")))] + fn index_load_tag_returns_the_data_ecc() { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache = make_cache(mem.clone()); + let phys: u32 = 0xd000; + let va = kseg0(phys); + mem_write(&mem, phys, 0x1234_5678); + mem_write(&mem, phys + 4, 0x9abc_def0); + let _ = cache.read::<4>(va, phys as u64); + cache.cache_op(C_ILT | CACH_SD, va, va); + assert_eq!(cache.cache_op_ecc(), l2_data_ecc(0x1234_5678_9abc_def0) as u32); + } + + /// `L2_TAG_ECC_MASKS` is a transcription of R4000 UM Figure 16-4, and + /// these are the properties that figure states for itself: the "number of + /// 1s in syndrome" row says every tag bit has a weight-3 syndrome, the row + /// totals (check bit included) are 14,11,13,10,10,13,11 from check bit 0 + /// up, and a SECDED code needs every syndrome distinct. A mis-copied bit + /// breaks at least one of them. + #[test] + fn l2_tag_ecc_matrix_is_the_manuals() { + let rows: Vec = L2_TAG_ECC_MASKS.iter().map(|m| m.count_ones() + 1).collect(); + assert_eq!(rows, [14, 11, 13, 10, 10, 13, 11]); + let mut seen = std::collections::HashSet::new(); + for bit in 0..25 { + let syndrome = l2_tag_ecc(1 << bit); + assert_eq!(syndrome.count_ones(), 3, "tag bit {bit}"); + assert!(seen.insert(syndrome), "tag bit {bit} shares a syndrome"); + } + assert_eq!(L2_TAG_ECC_MASKS.iter().fold(0, |a, m| a | m), 0x1ff_ffff, "covers [24:0]"); + } + + /// Index_Load_Tag(SD) returns the check bits for the tag it returns, and + /// ignores whatever ECC TagLo carried on the store (R4000 UM p. 93). + #[test] + fn l2_index_load_tag_generates_the_tag_ecc() { + let mem = Arc::new(Memory::new(MEM_MB)); + let cache = make_cache(mem.clone()); + let index_addr = 0x8000_0000 | 0x40 * R4400Cache::L2_LINE as u64; + let fields: u32 = (5 << 22) | (3 << 19) | 0x2_1357; // DirtyExcl, PIdx 3, STag + let tag_lo = ((fields & 0x7_ffff) << 13) | ((fields >> 22) << 10) | (((fields >> 19) & 7) << 7); + + cache.cache_op(C_IST | CACH_SD, index_addr, (tag_lo | 0x55) as u64); + let got = cache.cache_op(C_ILT | CACH_SD, index_addr, index_addr) as u32; + + assert_eq!(got & !0x7f, tag_lo); + assert_eq!(got & 0x7f, l2_tag_ecc(fields)); + assert_ne!(got & 0x7f, 0x55, "the stored ECC field must be ignored"); + } + + /// TagLo[0] is the tag's even parity, generated by the hardware: over PTag + /// and the state on an R4400 (UM Figures 11-3/11-4), over PTag alone on an + /// R5000 (VR5000 UM Figures 12-3/12-4). Whatever TagLo[0] held on the store + /// is ignored. + #[test] + fn l1_index_load_tag_generates_the_tag_parity() { + fn check>>(state_counts: bool) { + let cache: C = make_cache_of(Arc::new(Memory::new(MEM_MB))); + for sel in [CACH_PI, CACH_PD] { + for ptag in [0u32, 1, 3, 0x12_3457] { + // Valid; Dirty on the R5000 D-cache, DirtyExcl on the R4400. + let tag_lo = (ptag << 8) | 0xc0; + cache.cache_op(C_IST | sel, 0x8000_0000, (tag_lo | 1) as u64); + let got = cache.cache_op(C_ILT | sel, 0x8000_0000, 0x8000_0000) as u32; + let state = (got >> 6) & 3; + let want = even_parity(ptag) ^ if state_counts { even_parity(state) } else { 0 }; + assert_eq!(got & 1, want, "sel {sel} ptag {ptag:#x} taglo {got:#x}"); + assert_eq!(got & 0x3e, 0, "TagLo[5:1] reads zero"); + } + } + } + check::(true); + check::(false); + } + /// Two-way independence: Index_WBInv on way 0 should not disturb way 1. /// Way selection for index ops: bit 14 of the index address selects the way. #[test] diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index 0ada05a8..dc7df2a3 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -2563,6 +2563,19 @@ pub fn deliver_exception(core: &mut MipsCore, status: u32) { pub fn deliver_exception_at(core: &mut MipsCore, status: u32, fault_pc: u64, bd: bool) { const EXEC_IS_TLB_REFILL: u32 = 1 << 28; const EXEC_IS_XTLB_REFILL: u32 = 1 << 29; + const EXEC_IS_CACHE_ERR: u32 = 1 << 30; + + // Cache Error: ERL and ErrorEPC, the uncached vector, and nothing else — + // Cause, EPC and EXL are untouched (R4000 UM ch. 5, VR5000 UM 7.3.9). + // The caller has already loaded CacheErr. + if status & EXEC_IS_CACHE_ERR != 0 { + core.cp0_errorepc = if bd { fault_pc.wrapping_sub(4) } else { fault_pc }; + core.cp0_status |= STATUS_ERL; + let base = if core.cp0_status & STATUS_BEV != 0 { 0xFFFFFFFF_BFC00200u64 } else { 0xFFFFFFFF_A0000000u64 }; + core.pc = base + 0x100; + core.in_delay_slot = false; + return; + } let is_tlb_refill = status & EXEC_IS_TLB_REFILL != 0; let is_xtlb_refill = status & EXEC_IS_XTLB_REFILL != 0; diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 4e158323..903fd16e 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -949,6 +949,10 @@ pub static LOCKSTEP_VERIFICATIONS: std::sync::atomic::AtomicU64 = std::sync::ato pub const EXEC_IS_EXCEPTION: ExecStatus = 1 << 27; // 0x0800_0000 pub const EXEC_IS_TLB_REFILL: ExecStatus = 1 << 28; // 0x1000_0000 pub const EXEC_IS_XTLB_REFILL: ExecStatus = 1 << 29; // 0x2000_0000 +/// A Cache Error exception: ErrorEPC/ERL and the uncached vector, not the +/// common one. CacheErr comes from the cache (`MipsCache::take_cache_err`). +pub const EXEC_IS_CACHE_ERR: ExecStatus = 1 << 30; // 0x4000_0000 +pub const EXEC_CACHE_ERR: ExecStatus = EXEC_IS_EXCEPTION | EXEC_IS_CACHE_ERR; // Bus error ExecStatus values — also exported as BUS_ERR / BUS_VCE in traits.rs. // The values MUST stay identical; enforced by compile-time asserts in traits.rs. @@ -2924,13 +2928,11 @@ impl MipsExecutor { /// # Safety /// Same contract as `CpuCache::set_tcache_window`: `base` must be ppmem's /// window base and `bitmap` a live `u64` outliving this cache. - #[cfg(feature = "tcache")] pub unsafe fn set_tcache_window(&self, base: *mut u8) { unsafe { self.cache.set_tcache_window(base) }; } /// tcache: pointer to the cache's inline bitmap field. - #[cfg(feature = "tcache")] pub fn tcache_bitmap_ptr(&self) -> *mut u64 { self.cache.tcache_bitmap_ptr() } @@ -3622,6 +3624,19 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", self.update_translate_fn(); self.update_fpr_mode(); self.nanotlb_invalidate(); + #[cfg(feature = "cache-ecc")] + self.sync_cache_error_control(); + } + + /// Tell the cache which Status bits govern parity: CE forces the ECC + /// register into stored check bits, and DE or ERL turn detection off. + /// Every Status change comes through `resync_privilege_state`. + #[cfg(feature = "cache-ecc")] + #[inline] + fn sync_cache_error_control(&self) { + let s = self.core.cp0_status; + self.cache.set_error_control(s & STATUS_CE != 0, s & (STATUS_DE | STATUS_ERL) == 0); + self.cache.set_cache_ecc(self.core.cp0_ecc); } /// Store a new CP0 Status word and re-derive everything cached off it. @@ -4381,6 +4396,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[cfg(feature = "jitv2")] { self.core.syscall_pending = false; } + if status & EXEC_IS_CACHE_ERR != 0 { + self.core.cp0_cacheerr = self.cache.take_cache_err(); + } // Architectural effect (Cause/EPC/Status/vector) — the portable part // shared with jitv2-verify (§4.2 single-implementation delivery). crate::cpu::mips_core::deliver_exception(&mut self.core, status); @@ -4466,6 +4484,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // wrong one; the `ad{e,s}l_*_matches_interpreter` equivalence tests // caught it as a one-field divergence. Every caller that needs // BadVAddr sets it before calling. + if status & EXEC_IS_CACHE_ERR != 0 { + self.core.cp0_cacheerr = self.cache.take_cache_err(); + } crate::cpu::mips_core::deliver_exception_at(&mut self.core, status, fault_pc, bd); self.resync_privilege_state(); status @@ -5086,8 +5107,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", if r.is_ok() { Ok(r.data) } else { - if !DEBUG { self.core.cp0_badvaddr = virt_addr; } - Err(r.status) // BUS_BUSY, BUS_VCE, or BUS_ERR — all valid ExecStatus + // A Cache Error leaves BadVAddr alone; it changes + // only CacheErr, ErrorEPC and ERL. + if !DEBUG && r.status & EXEC_IS_CACHE_ERR == 0 { self.core.cp0_badvaddr = virt_addr; } + Err(r.status) // BUS_BUSY, BUS_VCE, BUS_ERR or EXEC_CACHE_ERR — all valid ExecStatus } } else { // Uncached access @@ -6840,8 +6863,21 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } // Call unified cache interface let result = self.cache.cache_op(cache_op, virt_addr, phys_addr_or_taglo); - if C::R10K_CACHE_OPS && op == C_R10K_ILD { + // An I-cache write-back of a line with bad parity raises a Cache + // Error from the CACHE instruction itself. + #[cfg(feature = "cache-ecc")] + { + let op_status = self.cache.take_op_status(); + if op_status != 0 { + return self.handle_exception(op_status); + } + } + // Index_Load_Tag also loads CP0 ECC with the selected doubleword's + // check bits (R4000 UM A-42); the R10000 does this on Index_Load_Data. + if (C::R10K_CACHE_OPS && op == C_R10K_ILD) + || (cfg!(feature = "cache-ecc") && !C::R10K_CACHE_OPS && op == C_ILT) { self.core.cp0_ecc = self.cache.cache_op_ecc(); + self.cache.set_cache_ecc(self.core.cp0_ecc); } // For Index_Load_Tag, update CP0 TagLo from result @@ -7219,6 +7255,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } // cheritest: a write to CP0 26 triggers the test device's dump. Routed // over the bus, so with no test device mapped it's an ignored GIO access. + // CP0 ECC: what a store writes as check bits while Status.CE is set. + if reg == 26 { + self.cache.set_cache_ecc(self.core.cp0_ecc); + } if reg == 26 && self.cheritest_dump_hook { self.sysad.write32(crate::dev::testdev::TEST_DEV_BASE + crate::dev::testdev::REG_DUMP, self.core.cp0_ecc); @@ -10500,13 +10540,11 @@ impl MipsCpu { /// /// # Safety /// Same contract as `CpuCache::set_tcache_window`. - #[cfg(feature = "tcache")] pub unsafe fn set_tcache_window(&self, base: *mut u8) { unsafe { self.executor.lock().set_tcache_window(base) }; } /// tcache: pointer to the cache's inline bitmap field. - #[cfg(feature = "tcache")] pub fn tcache_bitmap_ptr(&self) -> *mut u64 { self.executor.lock().tcache_bitmap_ptr() } @@ -14205,10 +14243,8 @@ pub trait CpuDevice: Device + Resettable + Saveable + Send + Sync { /// /// # Safety /// Same contract as `CpuCache::set_tcache_window`. - #[cfg(feature = "tcache")] unsafe fn set_tcache_window(&self, base: *mut u8); /// tcache: pointer to the cache's inline mapped-region bitmap. - #[cfg(feature = "tcache")] fn tcache_bitmap_ptr(&self) -> *mut u64; /// tcache + jitv2: hand the cache ppmem's generation-window base. /// @@ -14236,6 +14272,8 @@ pub trait CpuDevice: Device + Resettable + Saveable + Send + Sync { fn set_hw_read_fixup_replay(&self, f: Option>) -> Result<(), String>; /// CPU model name, as the guest and the benchmark report see it. fn model_name(&self) -> &'static str; + /// Cache model: "full" or "shadow" (`CpuModel::CACHE_KIND`). + fn cache_kind(&self) -> &'static str; fn set_cheritest_dump_hook(&self, on: bool); fn running_flag(&self) -> Arc; fn fasttick_count(&self) -> Arc; @@ -14252,11 +14290,9 @@ impl CpuDevice for MipsCp fn cycles_ptr(&self) -> crate::cpu::mips_core::CyclesPtr { MipsCpu::cycles_ptr(self) } fn interrupts_ptr(&self) -> *const AtomicU64 { MipsCpu::interrupts_ptr(self) } fn ppmem_bitmap_ptr(&self) -> *mut u64 { MipsCpu::ppmem_bitmap_ptr(self) } - #[cfg(feature = "tcache")] unsafe fn set_tcache_window(&self, base: *mut u8) { unsafe { MipsCpu::set_tcache_window(self, base) } } - #[cfg(feature = "tcache")] fn tcache_bitmap_ptr(&self) -> *mut u64 { MipsCpu::tcache_bitmap_ptr(self) } #[cfg(feature = "jitv2")] unsafe fn set_tcache_gen_window(&self, gen_base: *mut AtomicU64) { @@ -14285,6 +14321,7 @@ impl CpuDevice for MipsCp MipsCpu::set_hw_read_fixup_replay(self, f) } fn model_name(&self) -> &'static str { C::NAME } + fn cache_kind(&self) -> &'static str { C::CACHE_KIND } fn set_cheritest_dump_hook(&self, on: bool) { MipsCpu::set_cheritest_dump_hook(self, on) } fn running_flag(&self) -> Arc { MipsCpu::running_flag(self) } fn fasttick_count(&self) -> Arc { Arc::clone(&self.fasttick_count) } diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 1a3d8cc4..ad8f040a 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -1774,6 +1774,30 @@ mod tests { "EPC must point at the branch, i.e. fault_pc - 4"); } + /// A Cache Error goes to the uncached vector (0xA000_0100, or 0xBFC0_0300 + /// under BEV) with ERL and ErrorEPC, and changes nothing else: Cause, EPC + /// and EXL keep whatever they held (R4000 UM ch. 5, VR5000 UM 7.3.9). + #[test] + fn test_cache_error_takes_the_uncached_vector_with_erl() { + use crate::cpu::mips_core::{MipsCore, deliver_exception_at, STATUS_BEV, STATUS_ERL, STATUS_EXL}; + use crate::cpu::mips_exec::EXEC_CACHE_ERR; + + for (bev, vector) in [(false, 0xFFFF_FFFF_A000_0100u64), (true, 0xFFFF_FFFF_BFC0_0300)] { + let mut core = MipsCore::new(); + core.cp0_status = if bev { STATUS_BEV } else { 0 }; + core.cp0_cause = 0x1234; + core.cp0_epc = 0x5678; + deliver_exception_at(&mut core, EXEC_CACHE_ERR, 0xFFFF_FFFF_8800_0010, true); + + assert_eq!(core.pc, vector, "BEV={bev}"); + assert_eq!(core.cp0_errorepc, 0xFFFF_FFFF_8800_000C, "ErrorEPC: the branch, for a delay slot"); + assert_ne!(core.cp0_status & STATUS_ERL, 0, "ERL set"); + assert_eq!(core.cp0_status & STATUS_EXL, 0, "EXL untouched"); + assert_eq!((core.cp0_cause, core.cp0_epc), (0x1234, 0x5678), "Cause/EPC untouched"); + assert!(!core.in_delay_slot); + } + } + /// CP0 instructions must be refused in user mode unless Status.CU0 is set. /// /// `exec_cop0` is the single dispatch point for MFC0/MTC0/TLB*/ERET/WAIT, and diff --git a/src/cpu/mod.rs b/src/cpu/mod.rs index 12cefcce..35dd7663 100644 --- a/src/cpu/mod.rs +++ b/src/cpu/mod.rs @@ -8,6 +8,7 @@ pub mod mips_core; pub mod mips_tlb; pub mod mips_cache_v2; pub mod mips_cache_shadow; +pub mod decode_cache; pub mod mips_exec; pub mod mips_exec_test; pub mod mips_instr_stats; diff --git a/src/dev/mgras/debug.rs b/src/dev/mgras/debug.rs index 88081009..fb7f86a5 100644 --- a/src/dev/mgras/debug.rs +++ b/src/dev/mgras/debug.rs @@ -531,6 +531,9 @@ impl Mgras { [] => None, _ => return Err("usage: mgras jit [on|off|sync|list]".into()), }; + if mode.is_some_and(|m| m != MODE_OFF) && rss_jit::forbidden() { + return Err("mgras jit: IRIS_NO_JIT is set (this host does not allow JIT code)".into()); + } if let Some(m) = mode { // The RSS thread reads the mode: change it with the board idle. let _sub = self.submit.lock(); diff --git a/src/dev/mgras/rss_jit/mod.rs b/src/dev/mgras/rss_jit/mod.rs index 4cd9cc1e..be0b671b 100644 --- a/src/dev/mgras/rss_jit/mod.rs +++ b/src/dev/mgras/rss_jit/mod.rs @@ -22,6 +22,8 @@ //! process-wide map; a primitive whose shader is not ready yet runs in the //! interpreter. `IRIS_GR4_JIT=off|sync|async` picks the mode (`sync` //! compiles on first use and waits, for tests and comparisons). +//! `IRIS_NO_JIT` (set by the Mac App Store build, whose sandbox kills +//! Cranelift's executable pages) forces it off, as it does the REX3 JIT. pub mod compiler; pub mod key; @@ -45,9 +47,20 @@ pub const MODE_ASYNC: u32 = 0; /// Compile on first use and wait (tests, comparisons). pub const MODE_SYNC: u32 = 2; -/// The mode a new board starts in: `IRIS_GR4_JIT`, else async (off in unit -/// tests, so the interpreter's tests stay deterministic unless asked). +/// Whether the host forbids JIT code (`IRIS_NO_JIT`): off then, whatever +/// `IRIS_GR4_JIT` or `mgras jit` ask for. Off never reaches `store()`, so +/// no compiler thread or executable page is ever made. +pub fn forbidden() -> bool { + std::env::var_os("IRIS_NO_JIT").is_some() +} + +/// The mode a new board starts in: off if `forbidden`, else `IRIS_GR4_JIT`, +/// else async (off in unit tests, so the interpreter's tests stay +/// deterministic unless asked). pub fn default_mode() -> u32 { + if forbidden() { + return MODE_OFF; + } match std::env::var("IRIS_GR4_JIT").as_deref() { Ok("off") | Ok("0") => MODE_OFF, Ok("sync") => MODE_SYNC, diff --git a/src/machine.rs b/src/machine.rs index bf546c05..f45b75fd 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -31,7 +31,7 @@ use crate::cpu::mips_tlb::MipsTlb; use crate::cpu::mips_exec::{MipsExecutor, MipsCpu, MipsCpuConfig, MipsCpuDebugAdapter}; use crate::gdb_stub::CpuDebug; use crate::cpu::mips_cache_v2::{MipsCache, R4400Cache, R5000Cache}; -use crate::cpu::mips_cache_shadow::R10000ShadowCache; +use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R4400ShadowCache, R5000ShadowCache}; use crate::dev::hpc3::Hpc3; use crate::dev::ioc::{Ioc, GioSlot, GIO_SLOT_MAP, profile_idx}; use crate::monitor::Monitor; @@ -806,7 +806,12 @@ impl Machine { Arc::new(MipsCpu::new(executor)) as Arc }}} + let shadow = cfg.machine.cache.is_shadow(cfg_cpu_model); let cpu: Arc = match cfg_cpu_model { + // R4400/R5000: the functional caches, or the shadow ones that + // keep only what CACHE instructions observe — see CacheModel. + crate::config::CpuModel::R4400 if shadow => build_cpu!(R4400ShadowCache), + crate::config::CpuModel::R5000 if shadow => build_cpu!(R5000ShadowCache), crate::config::CpuModel::R4400 => build_cpu!(R4400Cache), crate::config::CpuModel::R5000 => build_cpu!(R5000Cache), // IP28 uses the shadow cache: out of the data path entirely, with @@ -840,15 +845,20 @@ impl Machine { let phys_ptr = Arc::as_ptr(&phys) as *mut Physical; (*phys_ptr).resync_ppmem_bitmap(); } - // tcache: the cache reads RAM straight out of the same window. - // Must come after set_bitmap_sink, so it captures the CPU's inline - // field rather than the sink the space just abandoned. - #[cfg(feature = "tcache")] + // The cache reads RAM straight out of the same window: the full + // cache under tcache, the shadow cache always. A cache that does + // not (the full cache without tcache) has no bitmap field and + // returns null. Must come after set_bitmap_sink, so it captures + // the CPU's inline field rather than the sink the space just + // abandoned. if let Some(sp) = phys.ppmem_space() { unsafe { cpu.set_tcache_window(sp.window_base()) }; // The cache keeps the bitmap inline; ppmem publishes into it // directly on every remap, so the hot path needs no indirection. - unsafe { sp.set_bitmap_sink2(cpu.tcache_bitmap_ptr()) }; + let bitmap = cpu.tcache_bitmap_ptr(); + if !bitmap.is_null() { + unsafe { sp.set_bitmap_sink2(bitmap) }; + } // Window writes bypass BusDevice, so the cache needs the gen // window to bump page counters itself — otherwise jitv2 keeps // running compiled code for pages the guest has modified. @@ -1798,6 +1808,7 @@ impl Machine { manifest.nvram = Some(self.nvram_path.clone()); // The CPU is a runtime choice now, so it is state the restore must match. manifest.cpu_model = Some(self.cpu.model_name().to_string()); + manifest.cpu_cache = Some(self.cpu.cache_kind().to_string()); snap.write_manifest(&manifest).map_err(|e| e.to_string())?; let sv = manifest.schema_version; @@ -1997,6 +2008,8 @@ impl Machine { // no longer a build flag, so a mismatch is as fatal as a feature one. if let Some(e) = crate::snapshot::cpu_model_mismatch( m.cpu_model.as_deref(), self.cpu.model_name()) { fatal.push(e); } + if let Some(e) = crate::snapshot::cache_model_mismatch( + m.cpu_cache.as_deref(), m.cpu_model.as_deref(), self.cpu.cache_kind()) { fatal.push(e); } // Every recorded disk must still be configured at the same SCSI // id with the same size. The host *path* is only where the file // happens to live (it moves when disks are relocated, e.g. into a diff --git a/src/snapshot.rs b/src/snapshot.rs index 2dbc6aa3..c2c965e9 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -74,6 +74,20 @@ pub fn cpu_model_mismatch(snapshot: Option<&str>, current: &str) -> Option, snapshot_cpu: Option<&str>, current: &str) -> Option { + let snapshot = snapshot.unwrap_or(if snapshot_cpu == Some("shadow") { "shadow" } else { "full" }); + (snapshot != current).then(|| format!( + "cache model differs: snapshot {snapshot} vs current {current} — set [machine] cache = \"{snapshot}\"")) +} + /// Feature names older builds recorded that are no longer cargo features (the /// code they gated is now always built). Ignored when comparing a snapshot's /// recorded features against the running build. @@ -126,6 +140,9 @@ pub struct Manifest { /// Emulated CPU at capture time. None for legacy manifests, and for those /// the check is skipped — the model was a build flag before it was config. pub cpu_model: Option, + /// Cache model at capture time, `CpuModel::CACHE_KIND`: "full" or + /// "shadow". None for manifests from before it was a choice. + pub cpu_cache: Option, } impl Manifest { @@ -150,6 +167,7 @@ impl Manifest { disks: Vec::new(), nvram: None, cpu_model: None, + cpu_cache: None, } } @@ -189,6 +207,9 @@ impl Manifest { if let Some(cm) = &self.cpu_model { tbl.insert("cpu_model".into(), Value::String(cm.clone())); } + if let Some(cc) = &self.cpu_cache { + tbl.insert("cpu_cache".into(), Value::String(cc.clone())); + } if let Some(nv) = &self.nvram { tbl.insert("nvram".into(), Value::String(nv.clone())); } @@ -234,6 +255,7 @@ impl Manifest { .unwrap_or_default(); let nvram = tbl.get("nvram").and_then(|x| x.as_str()).map(String::from); let cpu_model = tbl.get("cpu_model").and_then(|x| x.as_str()).map(String::from); + let cpu_cache = tbl.get("cpu_cache").and_then(|x| x.as_str()).map(String::from); Ok(Self { schema_version, iris_git_rev, @@ -246,6 +268,7 @@ impl Manifest { disks, nvram, cpu_model, + cpu_cache, }) } } @@ -615,6 +638,7 @@ mod tests { disks: vec![DiskRef { id: 1, path: "irix53.raw".into(), size_bytes: 4294967296 }], nvram: Some("nvram-irix53.bin".into()), cpu_model: Some("R5000".into()), + cpu_cache: Some("shadow".into()), }; let v = m.to_toml(); let m2 = Manifest::from_toml(&v).expect("parse"); @@ -645,6 +669,7 @@ mod tests { disks: vec![], nvram: None, cpu_model: None, + cpu_cache: None, }; let v = m.to_toml(); let m2 = Manifest::from_toml(&v).expect("parse"); @@ -672,6 +697,19 @@ mod tests { assert!(cpu_model_mismatch(Some("R5000"), "R4400").is_some()); } + #[test] + fn cache_model_mismatch_is_fatal_and_legacy_means_the_old_default() { + assert!(cache_model_mismatch(Some("full"), Some("R4400"), "full").is_none()); + assert!(cache_model_mismatch(Some("shadow"), Some("R5000"), "shadow").is_none()); + assert!(cache_model_mismatch(Some("full"), Some("R4400"), "shadow").is_some()); + assert!(cache_model_mismatch(Some("shadow"), Some("R4400"), "full").is_some()); + // Before the choice: R4400/R5000 ran the full cache, the R10000 the shadow one. + assert!(cache_model_mismatch(None, Some("R4400"), "full").is_none()); + assert!(cache_model_mismatch(None, Some("R4400"), "shadow").is_some()); + assert!(cache_model_mismatch(None, Some("shadow"), "shadow").is_none()); + assert!(cache_model_mismatch(None, None, "full").is_none()); + } + /// The CPU model must survive a manifest round trip, and a legacy manifest /// (written before the model was config) must still parse as "unknown" so /// old snapshots keep loading instead of being refused outright.