From 7772a9d8eb893d49ef78791f864af7aadc482e22 Mon Sep 17 00:00:00 2001 From: 0xrsydn Date: Wed, 7 Oct 2026 09:05:45 +0700 Subject: [PATCH] fix(bios): return cleanly for empty LZ77 streams --- crates/desktop/tests/rom_cli.rs | 27 +++++++++++ crates/gba-core/src/bios/lz77.rs | 11 +++-- crates/gba-core/tests/bios/lz77.rs | 31 ++++++++++-- docs/hardware/bios.md | 9 +++- docs/references.md | 7 +++ docs/research/emerald-empty-lz.md | 75 ++++++++++++++++++++++++++++++ docs/status.md | 4 ++ docs/testing.md | 3 ++ 8 files changed, 156 insertions(+), 11 deletions(-) create mode 100644 docs/research/emerald-empty-lz.md diff --git a/crates/desktop/tests/rom_cli.rs b/crates/desktop/tests/rom_cli.rs index d542d44..7f090a4 100644 --- a/crates/desktop/tests/rom_cli.rs +++ b/crates/desktop/tests/rom_cli.rs @@ -77,6 +77,27 @@ fn raw_original_program_runs_with_a_bounded_report_without_changing_the_file() { assert_eq!(fs::read(path).unwrap(), original); } +fn empty_lz_setup() -> Vec { + // Original ARM caller, an all-zero header at EOF, and a RAM sentinel. + // Both LZ variants must return without reading tokens or writing the destination. + vec![ + 0xe59f001c, 0xe59f101c, 0xe3a0205a, 0xe5812000, 0xef110000, 0xef120000, 0xe5913000, + 0xe3a0402a, 0xeafffffe, 0x0800002c, 0x02001000, 0, + ] +} + +#[test] +fn empty_lz_calls_return_through_original_bios_without_changing_destination() { + let fixture = Fixture::new(); + let path = fixture.rom(&empty_lz_setup()); + let original = fs::read(&path).unwrap(); + let output = run(&path, "1000"); + assert!(output.status.success(), "{}", stderr(&output)); + assert!(stdout(&output).contains("r3=0x0000005a")); + assert!(stdout(&output).contains("r4=0x0000002a")); + assert_eq!(fs::read(path).unwrap(), original); +} + fn wave_setup() -> Vec { // Original ARM halfword stores and a status load; no external audio or assets. let mut code = Vec::new(); @@ -641,6 +662,12 @@ fn native_rom_window_presents_file_bytes_and_reports_stop_cpu_and_video_errors() ); assert!(stdout(&output).contains("Captured ROM frames: 3")); assert!(stdout(&output).contains("Result: window frame limit reached")); + let lz_path = fixture.rom(&empty_lz_setup()); + let lz_output = window(&lz_path, false, None); + assert!(lz_output.status.success(), "{}", stderr(&lz_output)); + assert!(stdout(&lz_output).contains("Captured ROM frames: 3")); + assert!(stdout(&lz_output).contains("r3=0x0000005a")); + assert!(stdout(&lz_output).contains("r4=0x0000002a")); let wave_path = fixture.rom(&wave_setup()); let wave_output = window(&wave_path, false, None); assert!(wave_output.status.success(), "{}", stderr(&wave_output)); diff --git a/crates/gba-core/src/bios/lz77.rs b/crates/gba-core/src/bios/lz77.rs index df73492..81eb88d 100644 --- a/crates/gba-core/src/bios/lz77.rs +++ b/crates/gba-core/src/bios/lz77.rs @@ -1,6 +1,7 @@ //! Original LZ77 decoder emitted as ARM instructions. //! Invalid streams trap after any earlier successful writes. VRAM output requires -//! an even size and halfword alignment; distance-one references are rejected. +//! an even size and halfword alignment for nonempty output; distance-one references +//! are rejected. A zero-length header returns without reading tokens or checking its type. use super::ArmImage; @@ -18,9 +19,11 @@ pub(super) fn emit(a: &mut ArmImage) { a.branch(3, "invalid_argument"); a.emit(0xe490_2004); // LDR r2,[r0],#4 (header) a.emit(0xe202_b0ff); // AND r11,r2,#0xff - a.emit(0xe35b_0010); // CMP r11,#0x10 (LZ77 type, reserved bits zero) - a.branch(1, "invalid_argument"); a.emit(0xe1a0_2422); // MOV r2,r2,LSR #8 (24-bit output byte count) + a.emit(0xe352_0000); // CMP r2,#0 + a.branch(0, "lz_done"); // Empty streams need neither a type tag nor a usable destination. + a.emit(0xe35b_0010); // CMP r11,#0x10 (nonempty LZ77 type, reserved bits zero) + a.branch(1, "invalid_argument"); a.emit(0xe35c_0000); // CMP r12,#0 a.branch(0, "lz_validate_end"); a.emit(0xe181_b002); // ORR r11,r1,r2 (both destination and size must be even) @@ -29,8 +32,6 @@ pub(super) fn emit(a: &mut ArmImage) { a.label("lz_validate_end"); a.emit(0xe091_b002); // ADDS r11,r1,r2 a.branch(2, "invalid_argument"); // Reject output address wrap. - a.emit(0xe352_0000); // CMP r2,#0 - a.branch(0, "lz_done"); a.emit(0xe1a0_8001); // MOV r8,r1 (start of produced output) a.emit(0xe3a0_9000); // MOV r9,#0 (pending halfword) a.emit(0xe3a0_a000); // MOV r10,#0 (bit position 0 or 8) diff --git a/crates/gba-core/tests/bios/lz77.rs b/crates/gba-core/tests/bios/lz77.rs index f7617ab..94e2663 100644 --- a/crates/gba-core/tests/bios/lz77.rs +++ b/crates/gba-core/tests/bios/lz77.rs @@ -305,16 +305,39 @@ fn decompression_restores_user_mode_masks_and_all_caller_registers() { } #[test] -fn zero_output_length_does_not_read_flags_or_touch_destination() { +fn empty_headers_ignore_type_without_reading_payload_or_accessing_destination() { + // Original synthetic headers, including the all-zero empty-stream case. + // The unmapped, odd destination also proves no output alignment/access is needed. for service in [0x11, 0x12] { - let (mut machine, return_pc) = prepare(service, false, 0x0e00_0000, &[0x10, 0, 0, 0]); - complete(&mut machine, return_pc); // Destination and next source byte are unmapped. + for thumb in [false, true] { + for low_byte in 0..=255 { + let (mut machine, return_pc) = + prepare(service, thumb, 0x0e00_0001, &[low_byte, 0, 0, 0]); + let before = machine.cpu().clone(); + complete(&mut machine, return_pc); // Header ends at the last supplied ROM byte. + assert_eq!(&machine.cpu().registers()[..15], &before.registers()[..15]); + assert_eq!(machine.cpu().cpsr(), before.cpsr()); + } + } + } +} + +#[test] +fn empty_headers_preserve_ram_video_and_stack_sentinels() { + for service in [0x11, 0x12] { + for thumb in [false, true] { + for destination in [DEST, VRAM_START + 4] { + for low_byte in [0, 0x10, 0xff] { + verify(service, thumb, destination, &[low_byte, 0, 0, 0], &[]); + } + } + } } } #[test] fn bad_headers_alignment_and_wrapping_destinations_are_rejected() { - for header in [0_u32, 0x0200, 0x0211, 0x0220, 0x02ff] { + for header in [0x0100_u32, 0x0200, 0x0211, 0x0220, 0x02ff] { let (mut machine, _) = prepare(0x11, false, DEST, &header.to_le_bytes()); assert_invalid(fail(&mut machine)); assert_eq!(machine.memory().read32(DEST).unwrap(), 0); diff --git a/docs/hardware/bios.md b/docs/hardware/bios.md index 6f64945..af85134 100644 --- a/docs/hardware/bios.md +++ b/docs/hardware/bios.md @@ -387,7 +387,9 @@ Exact firmware cycle counts and undocumented side effects are not reproduced. ## LZ77 decompression -Both variants read a word-aligned header at r0. Its low byte must be `0x10`; the upper 24 bits specify output length. +Both variants read a word-aligned header at r0. The upper 24 bits specify output length. +If the length is zero, both return without checking the low byte or accessing the destination or token stream. +This includes an all-zero header. Nonempty streams still require low byte `0x10`. Each flag byte describes eight tokens, most significant bit first. A literal token produces one byte. A reference token copies 3–18 bytes from 1–4,096 bytes behind the output position. References can overlap previously produced output. Reads observe each preceding write through the normal emulated bus. @@ -404,7 +406,10 @@ The emulator does not model firmware quirks for odd output sizes or invalid refe The decoder validates the type byte, source alignment, source protection, output address wrap, and reference bounds. Sources below `0x02000000`, references before produced output, and runs beyond the declared size reach `bios::INVALID_ARGUMENT_TRAP`. Invalid VRAM alignment, odd output size, and distance-one references reach the same diagnostic trap. -Zero output length reads no tokens and writes no output after header/argument validation. +Zero output length returns after source alignment/protection checks and the header read, before type or destination checks. +An odd or unmapped destination is therefore harmless for an empty stream; invalid source addresses still fail. +The [empty-stream regression](../research/emerald-empty-lz.md) records the local reproduction and emulator evidence. +This is a bounded compatibility rule, not physical-hardware validation of malformed nonempty streams. Truncated cartridge input and invalid destination accesses return normal memory diagnostics. Earlier completed writes remain committed on failure; neither the whole service nor the whole output buffer is rolled back. Each call runs as ordinary ARM instructions, so callers can enforce machine-step limits on large streams. diff --git a/docs/references.md b/docs/references.md index eabc4fd..37f14b4 100644 --- a/docs/references.md +++ b/docs/references.md @@ -246,6 +246,13 @@ Disabled sound and disconnected serial reset: - [GBATEK normal serial mode](https://problemkaputt.de/gbatek-sio-normal-mode.htm) and its [general-purpose register chapter](https://mgba-emu.github.io/gbatek/), for data widths, control fields, and disconnected input pull-ups. - [mGBA BIOS implementation at 3a5e34be](https://github.com/mgba-emu/mgba/blob/3a5e34be33dc7f8f707e5bc9db69e8a430046f21/src/gba/bios.c), for functional serial and sound reset defaults. Exact firmware ordering and timing remain unverified. +Empty LZ77 header diagnostic: + +- [GBATEK BIOS decompression](https://problemkaputt.de/gbatek-bios-decompression-functions.htm), for the LZ77 header and 24-bit output length. +- [mGBA BIOS at 3a5e34be](https://github.com/mgba-emu/mgba/blob/3a5e34be33dc7f8f707e5bc9db69e8a430046f21/src/gba/bios.c), for `_unLz77` skipping token reads and writes when length is zero, without checking the signature byte. +- Secondary format references: [Rust gba LZ77 wrapper](https://docs.rs/gba/latest/gba/bios/fn.LZ77UnCompReadNormalWrite8bit.html) and [gbadoc BIOS overview](https://gbadev.net/gbadoc/bios.html). + See [the isolated reproduction and limits](research/emerald-empty-lz.md). These sources do not establish physical malformed-header behavior. + Optional classifier-assisted debugging: - [TypeSafe documentation index](https://docs.typesafe.ai/llms.txt), [API reference](https://docs.typesafe.ai/api.md), [models](https://docs.typesafe.ai/models.md), [confidence](https://docs.typesafe.ai/confidence.md), and [Jev limitations](https://docs.typesafe.ai/model-jaggedness/jev-1.13.md), for typed evidence review and its limits. diff --git a/docs/research/emerald-empty-lz.md b/docs/research/emerald-empty-lz.md new file mode 100644 index 0000000..20e0024 --- /dev/null +++ b/docs/research/emerald-empty-lz.md @@ -0,0 +1,75 @@ +# Emerald starter-selection diagnostic: empty LZ77 input + +## Report and isolation + +A user reported a failure while selecting one of the three starters in a local Emerald run. +The window reported 37,101 captured frames before an intentional replacement-BIOS argument trap: + +```text +PC=0x00000fd4 CPSR=0x80000093 Supervisor/Arm +r0=0x08d66484 r1=0x020057dc r2=0x00000000 +r11=0x00000000 r13=0x03007fa4 r14=0x082e7092 +unsupported instruction 0xe7f000f1 at 0x00000fd4 +``` + +`0xe7f000f1` is our `bios::INVALID_ARGUMENT_TRAP`, not an unimplemented ARM opcode from the game. +The printed instruction address in the user's final error line was truncated; the CPU state supplied the full address. +The run selected `--rtc --save-type flash128 --window --audio` but omitted `--save-file`. +Its save memory was therefore volatile. This diagnosis does not recover progress from that exited process. + +Local read-only inspection found Thumb `SWI 0x11` at the reported link register minus two. +That selects `LZ77UnCompWram`. Our decoder increments r0 by four when reading the header. +The inferred header address, `0x08d66480`, contains `0x00000000` in the locally supplied ROM. +The declared output length is zero. + +A temporary, original ARM caller reproduced the service in isolation: + +1. Map the local ROM unchanged with our original BIOS image. +2. Install an original caller in IWRAM and initialize its Supervisor stack. +3. Call `SWI 0x11` with source `0x08d66480` and destination `0x020057dc`. +4. Put a sentinel at the destination and bound execution to 1,000 machine steps. + +Before the fix, this reproduced the same trap PC and r0/r1/r2/r11 values. +The preceding instruction was at `0x00000c28`, the invalid-header conditional branch. +After the fix, the isolated call returned after 78 machine steps with the destination sentinel unchanged. +The probe did not boot through the game or reconstruct its entire live state. +No game code, graphics, firmware, ROM modifications, or save data were added to the repository. +The reproduction applies to the local file; no cartridge revision was inferred from its filename. + +## Cause and bounded correction + +The decoder required header byte `0x10` before examining the output length. +An empty all-zero header therefore failed validation, although it needed no token reads or output writes. + +Both LZ77 variants now read the header and check its upper 24-bit length first. +A zero length returns through normal register/status restoration, independent of the low header byte or destination. +Source alignment, source protection, and the header read still apply. +Nonempty streams retain the existing type, output alignment, address-wrap, distance, and run-length checks. +Other decompression services are unchanged. No ROM address or game identity selects special behavior. + +GBATEK documents the standard `0x10` LZ77 header and its upper 24-bit length. +It does not establish all malformed-header behavior or physical zero-length timing. +The pinned mGBA implementation extracts the length, assumes the signature, and decodes only while the length is positive. +That provides emulator precedent for an empty-stream return, not a hardware measurement. +We did not copy its implementation or broaden acceptance of nonempty invalid-type streams. +Our replacement BIOS remains original ARM code and does not reproduce exact firmware timing or register clobbers. + +## Validation and remaining check + +Original regression tests failed before the correction and pass afterward. +They cover every low byte of an empty header, both services, ARM/Thumb callers, and caller-register/status restoration. +They preserve RAM, video, and stack sentinels. +Headers ending at ROM EOF and unmapped odd destinations establish that neither tokens nor output are accessed. +Existing nonempty malformed/truncated stream diagnostics still pass. +Original process and native-window tests call both services and verify return markers and unchanged destination data. + +Workspace debug/release tests, strict Clippy, formatting, and warning-free Rust documentation pass on Darwin arm64. +Pinned public ARM/Thumb/memory/BIOS suites and Pong scenarios also pass in both profiles; Pong reports match. +Full starter selection and subsequent gameplay after this fix require user retesting. +The isolated result does not establish Emerald save compatibility or a complete playthrough. + +## References + +- [GBATEK BIOS decompression](https://problemkaputt.de/gbatek-bios-decompression-functions.htm): standard LZ77 header, length, output widths, and reference format. +- [mGBA BIOS at 3a5e34be](https://github.com/mgba-emu/mgba/blob/3a5e34be33dc7f8f707e5bc9db69e8a430046f21/src/gba/bios.c): `_unLz77` extracts the upper 24-bit length and uses `while (remaining > 0)` without checking the low signature byte. +- [Rust gba LZ77 wrapper documentation](https://docs.rs/gba/latest/gba/bios/fn.LZ77UnCompReadNormalWrite8bit.html) and [gbadoc BIOS overview](https://gbadev.net/gbadoc/bios.html): secondary format/calling references found during research, not evidence for physical empty-header behavior. diff --git a/docs/status.md b/docs/status.md index e314b54..9c58db2 100644 --- a/docs/status.md +++ b/docs/status.md @@ -87,6 +87,7 @@ Update this file when a feature lands or a limit is removed. - Raw ROM-file loading with original BIOS boot, terminal diagnostics, and final CPU/step reports. - Firmware boot initializes BG2/BG3 identity scales for bitmap and affine programs that do not program their own matrices. - RegisterRamReset supports disabled sound and disconnected serial initialization, including the common `r0=0xff` request. +- Both LZ77 BIOS services return without output for zero-length headers, including all-zero headers; nonempty validation remains strict. - Disconnected Joybus selection, local data/status registers, and transmit-pending state without invented remote events or IRQs. - Direct Sound A/B support timer-driven FIFO playback, stereo digital levels, and DMA1/A or DMA2/B refill blocks. - PSG pulse channels 1 and 2 support independent nominal frequency/duty playback, length, envelope, status, and stereo mixing. Channel 1 also supports sweep. @@ -192,6 +193,9 @@ A later [fixed-time input probe](research/emerald-wave.md) found a wave trigger Wave support now passes that point and reaches 1,320 frames with scheduled Start/A presses and releases. Output differs from a no-input control, but readable menus and gameplay remain unverified. Flash writing and host persistence now pass original save/restart/load tests. Emerald saving and reloading remain unverified. +A user reached starter selection, then reported a BIOS invalid-argument trap. +An [isolated reproduction](research/emerald-empty-lz.md) identified a zero-length LZ77 header rejected before the length check. +The replacement BIOS now returns for that empty stream. Full starter-selection progress after this correction needs user retesting. Without `--rtc`, cartridge GPIO writes still return read-only diagnostics. No title-screen, gameplay, or audible-game compatibility is established. See [the startup result](research/emerald-reset.md) for the bounded run and next investigation. diff --git a/docs/testing.md b/docs/testing.md index b0bfced..61908a4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -65,6 +65,9 @@ They verify suite selection, pinned URLs, hashes, sizes, checkpoint validation, The [Pong adapter](public-pong.md) has additional offline tests for its pinned downloads, retained license, and bounded gameplay scenarios. Run the external Pong suite separately; ordinary Cargo and Python tests require no public ROM or network access. Device initialization regressions use original instructions to check disabled sound, wave RAM, disconnected serial pins, and reset flags. +LZ77 empty-header tests cover every low header byte, ARM/Thumb callers, both output widths, and unchanged RAM/video/stack sentinels. +An unmapped odd destination and header ending at ROM EOF prove that empty calls access neither output nor tokens. +Original terminal/native-window callers execute both LZ77 SWIs and verify return markers and retained RAM data. Core cartridge GPIO tests use an explicitly attached RTC and original pin sequences, never game code or host time. They cover read overlays, unchanged ROM bytes, masks, directions, widths, framing, control/calendar/reset, aborts, and unsupported commands. Calendar tests cover all dates in 2000–2099, 12/24-hour representation, PM flags, midnight, leap days, and large/split elapsed durations.