fix(video): ignore enable bits for backgrounds a mode lacks

DISPCNT enable bits for BG3 in Mode 1, BG0/BG1 in Mode 2, and every background
but BG2 in Modes 3-5 failed the whole frame with UnsupportedLayers, and the
window turned that into a stopped session. Hardware has no such backgrounds and
ignores the bits, so the renderer now masks them. The error variant is removed.

Prohibited OBJ mode 3 and shape 3 stay diagnostics: their hardware output is
not established here.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HhfHfHEuzv2AUTd6erW1gU

crates/gba-core/src/video.rs

@@ -29,7 +29,6 @@ pub fn rgb555_to_rgb888(color: u16) -> u32 {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum VideoError {
     UnsupportedMode(u8),
-    UnsupportedLayers(u16),
     UnsupportedTileAddress(usize),
     UnsupportedMapAddress(usize),
     UnsupportedObject { index: usize, reason: &'static str },
@@ -51,9 +50,6 @@ impl fmt::Display for VideoError {
                 "background map address {address:#x} exceeds the supported 128 KiB BG VRAM window"
             ),
             Self::UnsupportedObject { index, reason } => write!(f, "OBJ{index}: {reason}"),
-            Self::UnsupportedLayers(control) => {
-                write!(f, "unsupported display layers in DISPCNT {control:#06x}")
-            }
         }
     }
 }
@@ -227,16 +223,15 @@ pub(crate) fn render_rows(
     };
     let mut output = vec![0; length];
     let mode = control & 7;
-    let unsupported = match mode {
+    // Each mode has a fixed background set. Enable bits for absent backgrounds are ignored.
+    let absent = match mode {
         0 => 0,
         1 => 0x800,     // BG0/1 text, BG2 affine.
         2 => 0x300,     // BG2/3 affine.
         3..=5 => 0xb00, // BG2 bitmap.
         _ => return Err(VideoError::UnsupportedMode(mode as u8)),
     };
-    if control & unsupported != 0 {
-        return Err(VideoError::UnsupportedLayers(control));
-    }
+    let control = control & !absent;
     let mut layers: Vec<usize> = (0..4).filter(|bg| control & (0x100 << bg) != 0).collect();
     // Lower priority number wins. BG0 wins ties, then BG1, BG2, BG3.
     layers.sort_by_key(|&bg| (backgrounds[bg].control & 3, bg));

crates/gba-core/tests/affine_backgrounds.rs

@@ -1,7 +1,7 @@
 use gba_core::{
     io::*,
     memory::{Memory, MemoryError, OAM_START as OAM, PALETTE_START as PAL, VRAM_START as VRAM},
-    video::{rgb555_to_rgb888, Framebuffer, VideoError, WIDTH},
+    video::{rgb555_to_rgb888, Framebuffer, WIDTH},
 };
 
 fn setup(bg: u32, size: u16, wrap: bool) -> Memory {
@@ -252,17 +252,16 @@ fn transformed_backgrounds_composite_with_regular_and_affine_sprites() {
 }
 
 #[test]
-fn unsupported_layers_leave_output_unchanged_and_disabled_layers_skip_validation() {
+fn absent_layer_bits_are_ignored_and_disabled_layers_skip_validation() {
     let mut m = setup(2, 3, false);
     let mut f = Framebuffer::default();
-    f.clear(0x001f);
-    for control in [0xc01, 0xd02, 0xe02] {
+    // Mode 1 has no BG3 and mode 2 has no BG0 or BG1; their enable bits change nothing.
+    for (control, present) in [(0xc01, 0x401), (0xd02, 0xc02), (0xe02, 0xc02)] {
+        m.write16(DISPCNT, present).unwrap();
+        let expected = frame(&m);
         m.write16(DISPCNT, control).unwrap();
-        assert_eq!(
-            m.render_frame(&mut f),
-            Err(VideoError::UnsupportedLayers(control))
-        );
-        assert!(f.pixels().iter().all(|&p| p == 0xff0000));
+        m.render_frame(&mut f).unwrap();
+        assert_eq!(f.pixels(), expected.pixels());
     }
     m.write16(DISPCNT, 2).unwrap();
     m.render_frame(&mut f).unwrap();

crates/gba-core/tests/bitmap_modes.rs

@@ -1,7 +1,7 @@
 use gba_core::{
     io::{BG2CNT, BG2PA, BG2X, BG2Y, DISPCNT, GREENSWAP},
     memory::{Memory, OAM_START as OAM, PALETTE_START as PAL, VRAM_START as VRAM},
-    video::{rgb555_to_rgb888, Framebuffer, VideoError, WIDTH},
+    video::{rgb555_to_rgb888, Framebuffer, WIDTH},
 };
 
 fn setup(mode: u16) -> Memory {
@@ -423,20 +423,18 @@ fn cpu_bitmap_controls_pan_rotate_zoom_force_page_and_reset() {
 }
 
 #[test]
-fn unsupported_bitmap_layers_leave_frame_unchanged() {
+fn absent_bitmap_layer_bits_are_ignored() {
     for mode in [4, 5] {
         let mut m = setup(mode);
+        m.write16(VRAM + 0xa000, 0x0201).unwrap(); // Visible on the selected second page.
+        m.write16(DISPCNT, 0x410 | mode).unwrap();
+        let expected = frame(&m);
         let mut f = Framebuffer::default();
-        f.clear(0x001f);
         for extra in [0x100, 0x200, 0x800] {
-            let control = 0x410 | mode | extra;
-            m.write16(DISPCNT, control).unwrap();
-            assert_eq!(
-                m.render_frame(&mut f),
-                Err(VideoError::UnsupportedLayers(control))
-            );
+            m.write16(DISPCNT, 0x410 | mode | extra).unwrap();
+            m.render_frame(&mut f).unwrap();
+            assert_eq!(f.pixels(), expected.pixels());
         }
-        assert!(f.pixels().iter().all(|&p| p == 0xff0000));
         assert_eq!(m.cycles(), 0);
     }
 }

crates/gba-core/tests/graphics.rs

@@ -149,7 +149,7 @@ fn forced_blank_is_white_and_disabled_bg2_uses_palette_backdrop() {
 }
 
 #[test]
-fn unsupported_modes_and_layers_report_errors_without_changing_output() {
+fn unsupported_modes_report_errors_and_absent_layer_bits_are_ignored() {
     let mut memory = mode3();
     let mut frame = Framebuffer::default();
     frame.clear(0x001f);
@@ -160,14 +160,17 @@ fn unsupported_modes_and_layers_report_errors_without_changing_output() {
             Err(VideoError::UnsupportedMode(mode as u8))
         );
     }
-    for layer in [0x100, 0x200, 0x800] {
-        memory.write16(DISPCNT, 0x403 | layer).unwrap();
-        assert_eq!(
-            memory.render_frame(&mut frame),
-            Err(VideoError::UnsupportedLayers(0x403 | layer))
-        );
-    }
     assert!(frame.pixels().iter().all(|&pixel| pixel == 0xff0000));
+    // Mode 3 has only BG2. Stale BG0, BG1, and BG3 enables do not change the image.
+    memory.write16(VRAM_START, 0x03e0).unwrap();
+    memory.write16(DISPCNT, 0x403).unwrap();
+    let mut expected = Framebuffer::default();
+    memory.render_frame(&mut expected).unwrap();
+    for layer in [0x100, 0x200, 0x800, 0xb00] {
+        memory.write16(DISPCNT, 0x403 | layer).unwrap();
+        memory.render_frame(&mut frame).unwrap();
+        assert_eq!(frame.pixels(), expected.pixels());
+    }
 }
 
 #[test]

docs/hardware/video.md

@@ -30,7 +30,7 @@ The renderer samples that bitmap through BG2's matrix and reference-point regist
 Pixel bit 15 and the DISPCNT page-select bit do not affect Mode 3 output. Black pixels are opaque.
 Disabling BG2 leaves palette entry zero behind sprites. Forced blank displays white, regardless of the selected mode.
 All bitmap modes support regular and affine sprites, windows, and color effects.
-Enabling backgrounds other than BG2 in bitmap modes produces errors without changing the output buffer.
+Bitmap modes have only BG2. DISPCNT enable bits for BG0, BG1, and BG3 are ignored there.
 Reserved display modes 6 and 7 return errors. Modes 0 through 5 support snapshots and row capture, not complete hardware implementations.
 
 Mode 4 stores 240×160 byte-sized palette indices. Index zero is transparent and can reveal a lower-priority sprite.
@@ -75,7 +75,7 @@ Affine background snapshots support these configurations:
 - Mode 1: text BG0/BG1 and affine BG2.
 - Mode 2: affine BG2/BG3.
 - Modes 3, 4, and 5: affine bitmap BG2, with out-of-bitmap samples always transparent.
-- Enabling a background unavailable in the selected mode returns a diagnostic rather than silently ignoring it.
+- DISPCNT enable bits for a background the selected mode lacks (BG3 in Mode 1, BG0/BG1 in Mode 2, all but BG2 in Modes 3–5) are ignored, as on hardware.
 
 `BG2PA..BG2PD` occupy `0x04000020..0x04000027`. BG2 reference points X/Y occupy `0x04000028..0x0400002f`.
 BG3 uses the corresponding registers at `0x04000030..0x0400003f`.