0xrsydn/gameboy-rust
1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(test-runner): add deterministic gameplay scenarios

Browse source
This commit is contained in:
0xrsydn 2026-10-06 22:29:27 +07:00
commit bf2ab54fc2
8 changed files with 419 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

166
crates/desktop/src/cartridge/suite.rs
View file

@ -13,12 +13,14 @@ use std::{
use gba_core::{
bios,
cpu::InstructionSet,
input::Buttons,
machine::{Machine, StepKind},
video::{Framebuffer, HEIGHT, WIDTH},
};
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::{json, Value};
use super::{load, Stats, MAX_STEPS};
use super::{load, Stats, MAX_FRAMES, MAX_STEPS};
const MAX_MANIFEST_BYTES: u64 = 1024 * 1024;
const MAX_CASES: usize = 256;
@ -76,15 +78,30 @@ struct Case {
rom: PathBuf,
step_limit: u64,
completion: Completion,
#[serde(default)]
inputs: Option<Vec<Input>>,
checks: Vec<Check>,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(untagged, deny_unknown_fields)]
enum Completion {
Pc {
#[serde(deserialize_with = "word")]
pc: u32,
instruction_set: State,
},
VBlanks {
vblanks: u64,
},
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct Completion {
struct Input {
vblank: u64,
#[serde(deserialize_with = "word")]
pc: u32,
instruction_set: State,
buttons: u32,
}
#[derive(Debug, Clone, Copy, Deserialize, Serialize)]
@ -106,6 +123,12 @@ impl State {
#[derive(Debug, Deserialize, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
enum Check {
Pixel {
x: usize,
y: usize,
#[serde(deserialize_with = "word")]
equals: u32,
},
Register {
index: usize,
#[serde(deserialize_with = "word")]
@ -126,8 +149,8 @@ enum Check {
impl Suite {
fn validate(&self) -> io::Result<()> {
let invalid = |message: &str| io::Error::new(io::ErrorKind::InvalidInput, message);
if self.version != 1 {
return Err(invalid("suite version must be 1"));
if !matches!(self.version, 1 | 2) {
return Err(invalid("suite version must be 1 or 2"));
}
if self.cases.is_empty() || self.cases.len() > MAX_CASES {
return Err(invalid("suite must contain 1..=256 cases"));
@ -150,18 +173,62 @@ impl Suite {
if total > MAX_STEPS {
return Err(invalid("sum of case step limits must not exceed 100000000"));
}
let alignment = match case.completion.instruction_set {
State::Arm => 4,
State::Thumb => 2,
};
if case.completion.pc % alignment != 0 {
return Err(invalid("completion PC must be aligned for its instruction set; do not set the Thumb address bit"));
match case.completion {
Completion::Pc {
pc,
instruction_set,
} => {
let alignment = match instruction_set {
State::Arm => 4,
State::Thumb => 2,
};
if pc % alignment != 0 {
return Err(invalid("completion PC must be aligned for its instruction set; do not set the Thumb address bit"));
}
if case.inputs.is_some() {
return Err(invalid("input schedules require VBlank completion"));
}
}
Completion::VBlanks { vblanks } => {
if self.version != 2 || !(1..=MAX_FRAMES).contains(&vblanks) {
return Err(invalid(
"VBlank completion requires version 2 and 1..=100000 vblanks",
));
}
if let Some(inputs) = &case.inputs {
if inputs.len() > 256 {
return Err(invalid(
"input schedule must contain at most 256 snapshots",
));
}
let mut previous = None;
for input in inputs {
if input.buttons > 0x3ff
|| input.vblank >= vblanks
|| previous.is_some_and(|frame| input.vblank <= frame)
{
return Err(invalid("inputs need 10-bit button masks and strictly increasing VBlanks before completion"));
}
previous = Some(input.vblank);
}
}
}
}
if case.checks.is_empty() || case.checks.len() > MAX_CHECKS {
return Err(invalid("each case must contain 1..=256 checks"));
}
for check in &case.checks {
match check {
Check::Pixel { x, y, equals } => {
if self.version != 2
|| !matches!(case.completion, Completion::VBlanks { .. })
|| *x >= WIDTH
|| *y >= HEIGHT
|| *equals > 0xff_ffff
{
return Err(invalid("pixel checks require version 2, VBlank completion, x < 240, y < 160, and RGB888 values"));
}
}
Check::Register { index, .. } if *index > 15 => {
return Err(invalid("register index must be 0..=15"))
}
@ -205,20 +272,53 @@ enum Outcome {
StepLimit,
Stopped,
EmulationError(String),
RenderError(String),
}
fn run_to_checkpoint(machine: &mut Machine, case: &Case) -> (Stats, Outcome) {
let mut stats = Stats::default();
let capture = matches!(case.completion, Completion::VBlanks { .. });
machine.memory_mut().set_scanline_rendering(capture);
let mut frame = Framebuffer::default();
let mut last_vblank = 0;
let mut inputs = case.inputs.iter().flatten().peekable();
loop {
if machine.stopped() {
return (stats, Outcome::Stopped);
}
// Check after successful progress, before trying to execute the checkpoint.
// The final budgeted step can reach it. No initial/reset-state pass is allowed.
if stats.steps != 0
&& machine.cpu().pc() == case.completion.pc
&& machine.cpu().instruction_set() == case.completion.instruction_set.instruction_set()
{
let vblanks = machine.memory().display_position().vblanks;
if capture && vblanks != last_vblank {
match machine.memory().present_frame(&mut frame) {
Ok(true) => {}
Ok(false) => {
return (
stats,
Outcome::RenderError("no complete captured frame at VBlank".into()),
)
}
Err(error) => return (stats, Outcome::RenderError(error.to_string())),
}
last_vblank = vblanks;
}
// Snapshots apply after this VBlank's capture, before the next machine step.
while inputs.peek().is_some_and(|input| input.vblank <= vblanks) {
let input = inputs.next().unwrap();
machine
.memory_mut()
.set_buttons(Buttons::from_bits(input.buttons as u16));
}
// The final budgeted step can complete a case; reset alone cannot.
let complete = match case.completion {
Completion::Pc {
pc,
instruction_set,
} => {
machine.cpu().pc() == pc
&& machine.cpu().instruction_set() == instruction_set.instruction_set()
}
Completion::VBlanks { vblanks: target } => vblanks >= target,
};
if stats.steps != 0 && complete {
return (stats, Outcome::Checkpoint);
}
if stats.steps == case.step_limit {
@ -234,8 +334,23 @@ fn run_to_checkpoint(machine: &mut Machine, case: &Case) -> (Stats, Outcome) {
fn check(machine: &Machine, check: &Check) -> Value {
let (expected, actual) = match check {
Check::Pixel { x, y, equals } => {
let mut frame = Framebuffer::default();
let actual = match machine.memory().present_frame(&mut frame) {
Ok(true) => Ok(frame.pixels()[y * WIDTH + x]),
Ok(false) => Err("no complete captured frame".to_owned()),
Err(error) => Err(error.to_string()),
};
(*equals, actual)
}
Check::Register { index, equals } => (*equals, Ok(machine.cpu().registers()[*index])),
Check::Memory32 { address, equals } => (*equals, machine.memory().read32(*address)),
Check::Memory32 { address, equals } => (
*equals,
machine
.memory()
.read32(*address)
.map_err(|error| error.to_string()),
),
Check::Cpsr { equals } => (*equals, Ok(machine.cpu().cpsr())),
};
match actual {
@ -285,6 +400,11 @@ fn run_case(case: &Case, base: &Path) -> Value {
};
let (stats, outcome) = run_to_checkpoint(&mut machine, case);
report["state"] = state(&machine, &stats);
if matches!(case.completion, Completion::VBlanks { .. }) {
report["inputs"] = json!(case.inputs.as_deref().unwrap_or_default());
report["state"]["vblanks"] = json!(machine.memory().display_position().vblanks);
report["state"]["captured_vblank"] = json!(machine.memory().captured_vblank());
}
let reason = match outcome {
Outcome::Checkpoint => {
let checks: Vec<_> = case
@ -303,6 +423,10 @@ fn run_case(case: &Case, base: &Path) -> Value {
}
Outcome::StepLimit => "step_limit",
Outcome::Stopped => "stopped",
Outcome::RenderError(error) => {
report["error"] = json!(error);
"render_error"
}
Outcome::EmulationError(error) => {
report["error"] = json!(error);
"emulation_error"
@ -330,7 +454,7 @@ pub fn execute(path: &Path, writer: &mut impl Write) -> Result<(), Box<dyn Error
.collect();
let failed = cases.iter().filter(|case| case["passed"] != true).count();
let report = json!({
"format_version": 1, "bios": "original", "passed": failed == 0,
"format_version": suite.version, "bios": "original", "passed": failed == 0,
"case_count": cases.len(), "failed_count": failed, "cases": cases,
});
write_json_report(writer, &report)?;
@ -340,5 +464,7 @@ pub fn execute(path: &Path, writer: &mut impl Write) -> Result<(), Box<dyn Error
Ok(())
}
#[cfg(test)]
mod gameplay_tests;
#[cfg(test)]
mod tests;

142
crates/desktop/src/cartridge/suite/gameplay_tests.rs Normal file
View file

@ -0,0 +1,142 @@
use super::tests::{boot, suite};
use super::*;
use gba_core::io::{DISPCNT, KEYINPUT};
fn value() -> Value {
json!({"name":"input", "rom":"original.gba", "step_limit":1000000,
"completion":{"vblanks":4},
"inputs":[{"vblank":0,"buttons":1}, {"vblank":1,"buttons":17}, {"vblank":3,"buttons":0}],
"checks":[{"kind":"pixel","x":0,"y":0,"equals":255}]})
}
fn config(value: Value) -> Case {
suite(json!({"version":2,"cases":[value]}))
.unwrap()
.cases
.remove(0)
}
#[test]
fn strict_gameplay_schema_and_bounds() {
assert!(suite(json!({"version":1,"cases":[value()]})).is_err());
assert!(suite(json!({"version":2,"cases":[value()]})).is_ok());
for (pointer, replacement) in [
("/completion", json!({"vblanks":0})),
("/completion", json!({"vblanks":100001})),
(
"/completion",
json!({"vblanks":1,"pc":0,"instruction_set":"arm"}),
),
("/completion", json!({"vblanks":1,"typo":0})),
("/completion", json!({"pc":0})),
("/completion", json!({"pc":0,"instruction_set":"arm"})),
("/inputs", json!([{"vblank":0,"buttons":1024}])),
("/inputs", json!([{"vblank":4,"buttons":0}])),
(
"/inputs",
json!([{"vblank":1,"buttons":0},{"vblank":1,"buttons":1}]),
),
(
"/inputs",
json!([{"vblank":2,"buttons":0},{"vblank":1,"buttons":1}]),
),
("/inputs", json!([{"vblank":0,"buttons":0,"typo":0}])),
("/inputs", json!(vec![json!({"vblank":0,"buttons":0}); 257])),
("/inputs/0/buttons", json!(-1)),
("/inputs/0/vblank", json!(0.5)),
("/checks/0/x", json!(240)),
("/checks/0/y", json!(160)),
("/checks/0/equals", json!(0x1000000)),
] {
let mut input = value();
*input.pointer_mut(pointer).unwrap() = replacement;
assert!(
suite(json!({"version":2,"cases":[input]})).is_err(),
"{pointer}"
);
}
let mut input = value();
input["completion"]["vblanks"] = json!(100000);
input["inputs"] = json!([{"vblank":99999,"buttons":"0x3ff"}]);
input["checks"][0] = json!({"kind":"pixel","x":239,"y":159,"equals":"0xffffff"});
config(input);
}
#[test]
fn cpu_observes_initial_held_replaced_and_released_snapshots() {
for (target, expected, keys) in [
(1, 0xff0000, 0x3fe),
(2, 0x00ff00, 0x3ee),
(3, 0x00ff00, 0x3ee),
(4, 0x0000ff, 0x3ff),
] {
let mut input = value();
input["completion"]["vblanks"] = json!(target);
input["inputs"]
.as_array_mut()
.unwrap()
.retain(|event| event["vblank"].as_u64().unwrap() < target);
input["checks"][0]["equals"] = json!(expected);
let case = config(input);
let mut machine = bios::boot(gba_demos::input_rom()).unwrap();
let (_, outcome) = run_to_checkpoint(&mut machine, &case);
assert_eq!(outcome, Outcome::Checkpoint);
assert_eq!(machine.memory().captured_vblank(), Some(target));
assert_eq!(machine.memory().read16(KEYINPUT).unwrap(), keys);
assert_eq!(check(&machine, &case.checks[0])["passed"], true);
// A post-capture palette write must not change the asserted image.
machine.memory_mut().write16(0x05000000, 0x7fff).unwrap();
assert_eq!(check(&machine, &case.checks[0])["actual"], expected);
let mut frame = Framebuffer::default();
machine.memory().present_frame(&mut frame).unwrap();
assert!(frame.pixels().iter().all(|pixel| *pixel == expected));
}
}
#[test]
fn frame_completion_accepts_final_step_but_not_a_short_budget() {
let mut case = config(value());
let run =
|case: &Case| run_to_checkpoint(&mut bios::boot(gba_demos::input_rom()).unwrap(), case);
let (stats, outcome) = run(&case);
assert_eq!(outcome, Outcome::Checkpoint);
case.step_limit = stats.steps;
assert_eq!(run(&case).1, Outcome::Checkpoint);
case.step_limit -= 1;
assert_eq!(run(&case).1, Outcome::StepLimit);
}
#[test]
fn no_frame_is_not_a_black_pixel_pass_and_mismatch_is_reported() {
let mut machine = bios::boot(gba_demos::input_rom()).unwrap();
let black = Check::Pixel {
x: 0,
y: 0,
equals: 0,
};
let result = check(&machine, &black);
assert_eq!(result["passed"], false);
assert!(result["actual"].is_null());
assert!(result["error"].is_string());
run_to_checkpoint(&mut machine, &config(value()));
let result = check(&machine, &black);
assert_eq!(result["passed"], false);
assert_eq!(result["actual"], 255);
}
#[test]
fn first_render_diagnostic_stops_before_later_checkpoint() {
let mut machine = boot(&[0xeaff_fffe]);
machine.memory_mut().write16(DISPCNT, 7).unwrap();
let (stats, outcome) = run_to_checkpoint(&mut machine, &config(value()));
assert!(matches!(outcome, Outcome::RenderError(_)));
assert_eq!(machine.memory().display_position().vblanks, 1);
assert!(stats.steps > 0);
}
#[test]
fn future_input_does_not_advance_stopped_hardware() {
let mut machine = boot(&[0xef03_0000, 0xeaff_fffe]);
let (_, outcome) = run_to_checkpoint(&mut machine, &config(value()));
assert_eq!(outcome, Outcome::Stopped);
assert_eq!(machine.memory().display_position().vblanks, 0);
}

43
crates/desktop/src/cartridge/suite/tests.rs
View file

@ -13,12 +13,12 @@ fn case_value() -> Value {
fn case() -> Case {
serde_json::from_value(case_value()).unwrap()
}
fn suite(value: Value) -> Result<Suite, Box<dyn Error>> {
pub(super) fn suite(value: Value) -> Result<Suite, Box<dyn Error>> {
let suite: Suite = serde_json::from_value(value)?;
suite.validate()?;
Ok(suite)
}
fn boot(words: &[u32]) -> Machine {
pub(super) fn boot(words: &[u32]) -> Machine {
bios::boot(words.iter().flat_map(|word| word.to_le_bytes()).collect()).unwrap()
}
@ -90,7 +90,7 @@ fn unknown_fields_and_unknown_check_kinds_are_rejected() {
#[test]
fn manifest_rejects_empty_duplicate_excessive_or_invalid_cases() {
assert!(suite(json!({"version": 2, "cases": [case_value()]})).is_err());
assert!(suite(json!({"version": 3, "cases": [case_value()]})).is_err());
assert!(suite(json!({"version": 1, "cases": []})).is_err());
assert!(suite(json!({"version": 1, "cases": [case_value(), case_value()]})).is_err());
for (field, invalid) in [
@ -184,7 +184,10 @@ fn json_report_propagates_write_and_flush_failures() {
#[test]
fn reset_pc_is_not_an_immediate_pass_without_execution() {
let mut config = case();
config.completion.pc = 0;
config.completion = Completion::Pc {
pc: 0,
instruction_set: State::Arm,
};
config.step_limit = 1;
let (stats, outcome) = run_to_checkpoint(&mut boot(&[0xeaff_fffe]), &config);
assert_eq!(stats.steps, 1);
@ -215,14 +218,19 @@ fn checkpoint_is_checked_before_execution_and_on_the_last_budgeted_step() {
#[test]
fn matching_assertions_do_not_pass_without_the_checkpoint() {
let mut config = case();
config.completion.pc = ROM_START + 8;
config.completion = Completion::Pc {
pc: ROM_START + 8,
instruction_set: State::Arm,
};
let mut machine = boot(&[0xe3a0_002a, 0xeaff_fffe]);
let (stats, outcome) = run_to_checkpoint(&mut machine, &config);
assert_eq!(outcome, Outcome::StepLimit);
assert_eq!(stats.steps, config.step_limit);
assert_eq!(check(&machine, &config.checks[0])["passed"], true);
config.completion.pc = ROM_START + 4;
config.completion.instruction_set = State::Thumb;
config.completion = Completion::Pc {
pc: ROM_START + 4,
instruction_set: State::Thumb,
};
assert_eq!(
run_to_checkpoint(&mut boot(&[0xe3a0_002a, 0xeaff_fffe]), &config).1,
Outcome::StepLimit
@ -238,8 +246,10 @@ fn thumb_checkpoint_uses_aligned_architectural_pc() {
bytes.extend([0x2a, 0x20, 0xfe, 0xe7]);
let mut machine = bios::boot(bytes).unwrap();
let mut config = case();
config.completion.pc = ROM_START + 14;
config.completion.instruction_set = State::Thumb;
config.completion = Completion::Pc {
pc: ROM_START + 14,
instruction_set: State::Thumb,
};
assert_eq!(
run_to_checkpoint(&mut machine, &config).1,
Outcome::Checkpoint
@ -250,7 +260,10 @@ fn thumb_checkpoint_uses_aligned_architectural_pc() {
#[test]
fn memory_and_cpsr_checks_report_actual_values_and_read_errors() {
let mut config = case();
config.completion.pc = ROM_START + 12;
config.completion = Completion::Pc {
pc: ROM_START + 12,
instruction_set: State::Arm,
};
let mut machine = boot(&[0xe3a0_002a, 0xe3a0_1402, 0xe581_0000, 0xeaff_fffe]);
assert_eq!(
run_to_checkpoint(&mut machine, &config).1,
@ -291,7 +304,10 @@ fn memory_and_cpsr_checks_report_actual_values_and_read_errors() {
#[test]
fn stop_is_failure_while_halt_consumes_a_bounded_budget() {
let mut config = case();
config.completion.pc = ROM_START + 8;
config.completion = Completion::Pc {
pc: ROM_START + 8,
instruction_set: State::Arm,
};
let (stats, outcome) = run_to_checkpoint(&mut boot(&[0xef03_0000, 0xeaff_fffe]), &config);
assert_eq!(outcome, Outcome::Stopped);
assert!(stats.steps < config.step_limit);
@ -322,7 +338,10 @@ fn dma_and_irq_each_consume_a_budget_step_and_dma_errors_fail() {
let mut machine = boot(&[0xe3a0_002a, 0xeaff_fffe]);
run_to_checkpoint(&mut machine, &case());
let mut config = case();
config.completion.pc = ROM_START + 8;
config.completion = Completion::Pc {
pc: ROM_START + 8,
instruction_set: State::Arm,
};
config.step_limit = 3;
let bus = machine.memory_mut();
bus.write32(DMA_BASE, 0x0200_0000).unwrap();