diff --git a/EFFECTS.md b/EFFECTS.md index 24e0017..e7cca06 100644 --- a/EFFECTS.md +++ b/EFFECTS.md @@ -13,7 +13,7 @@ float4 effect(float2 uv, Inputs in) { ``` - `uv` is the pixel's position: `(0,0)` is top-left, `(1,1)` is bottom-right. -- Return `float4(r, g, b, 1.0)` with components in 0..1. +- Return `float4(r, g, b, 1.0)` with components in 0..1. Alpha is not shown on screen; return 1.0 unless you use it as memory (see below). - Do not write `#include`, `using namespace`, a vertex function or a fragment function. The app adds those. ## What `in` gives you @@ -24,10 +24,11 @@ float4 effect(float2 uv, Inputs in) { | `in.camera` | texture | The untouched camera frame. | | `in.last` | texture | The camera frame before this one. `length(camera - last)` at a pixel tells you how much it changed. | | `in.prev` | texture | This effect's own output from the previous frame. Read it to build trails and feedback. | -| `in.flow` | texture | `.xy` = which way and how far the picture moved at this spot since the last frame, in `uv` units. Typically tiny (0.001 to 0.02). | +| `in.flow` | texture | `.xy` = which way and how far the picture moved at this spot since the last frame, in `uv` units. Typically tiny (0.001 to 0.02): `length(flow) * 50.0` is a rough 0..1 "how fast" value. | | `in.particles` | texture | `.r` = brightness of particles that spawn where something moves and drift along with the motion. | | `in.smp` | sampler | Pass it to every `.sample()` call. | | `in.time` | float | Seconds since the app started. | +| `in.dt` | float | Seconds since the previous frame. Effects run once per camera frame, so this is about 1/30. Use it to make fades and timers last the same in real time whatever the frame rate: `value *= exp(-in.dt / seconds)`. | | `in.resolution` | float2 | Output size in pixels. One pixel is `1.0 / in.resolution` in `uv` units. | | `in.p.` | float | Your sliders (see below). | @@ -44,7 +45,7 @@ Put these at the top of the file. Spacing matters: one space between the parts, ``` - `// slider glow 0.0 4.0 1.5` adds a slider; read its value as `in.p.glow`. Up to 16 per effect. -- `// order 60` sets the position in the chain. Lower runs first; the default is 50. Existing effects use 10 to 90. +- `// order 60` sets the position in the chain. Any whole number; only the relative order matters, lower runs first, and the default is 50. Existing effects use 10 to 90. The motion inputs (`flow`, `particles`) are computed from the camera before any effect runs. - `// on` makes the effect enabled when the app starts. Leave it out for new effects. Slider names must be plain identifiers and must not be Metal built-ins or keywords (`mix`, `length`, `in`, `float`, `step`, `sample` and so on). `amount`, `speed`, `glow`, `hue` are fine. @@ -61,7 +62,8 @@ Slider names must be plain identifiers and must not be Metal built-ins or keywor - Write float numbers with a decimal point: `2.0`, not `2`. Convert loop counters with `float(i)`. - Helper functions go above `effect`, must be marked `static`, and take `Inputs in` as a parameter if they read textures. -- There is no global state. To remember something between frames, write it into your output and read it back from `in.prev`. The alpha channel (`.a`) of your output is free for this: it is not shown on screen. +- There is no global state. To remember something between frames, write it into your output and read it back from `in.prev`. Its `.rgb` is the picture you showed last frame; its alpha channel (`.a`) is free for your own use because it is not shown on screen. That gives one private number per pixel, stored with 8-bit precision (256 steps between 0 and 1), which is enough for a glow or heat level but not for a timestamp or a full-colour image. +- To do something every N seconds, test where `in.time` is inside the current period: `fract(in.time / N) < in.dt / N` is true for exactly one frame per period. ## Two complete examples @@ -112,12 +114,12 @@ swift build -c release .build/release/MotionLab --only --snapshot /tmp/.png ``` -`--check` also lists the sliders it found for each effect. If one of yours is missing, its `// slider` line is malformed. +`--check` also lists the sliders it found for each effect, and warns when a `// slider` line was ignored because it is malformed or over the limit of 16. ## Checklist - The file is `effects/.metal` and defines `float4 effect(float2 uv, Inputs in)`. - It reads its input picture from `in.image`, not `in.camera`, so it works in a chain. -- Every number a user might want to tune is a slider with a sensible range and default. +- The few numbers a user would most want to tune are sliders with a sensible range and default. Internal constants can stay in the code. - The first line is a comment saying what the effect does. - `.build/release/MotionLab --check` prints `ok` for it. diff --git a/Sources/MotionLab/Renderer.swift b/Sources/MotionLab/Renderer.swift index be0818b..40566b9 100644 --- a/Sources/MotionLab/Renderer.swift +++ b/Sources/MotionLab/Renderer.swift @@ -34,6 +34,7 @@ final class Renderer: NSObject, MTKViewDelegate { private var result: MTLTexture? // the finished frame private var lastSerial = -1 private let startTime = CACurrentMediaTime() + private var lastTime: Float? private var frameCount = 0 private var pixels: [UInt8] = [] @@ -224,6 +225,8 @@ final class Renderer: NSObject, MTKViewDelegate { analysis.update(commands, camera: frames.current, particles: layers.contains { $0.enabled && $0.usesParticles }) let time = Float(CACurrentMediaTime() - startTime) + let dt = min(max(time - (lastTime ?? time - 1.0 / 30.0), 0.001), 0.25) + lastTime = time var image = frames.current for layer in layers where layer.enabled { guard let pipeline = layer.pipeline else { continue } @@ -231,8 +234,8 @@ final class Renderer: NSObject, MTKViewDelegate { layer.targets = [makeTarget(device, width: width, height: height), makeTarget(device, width: width, height: height)] for target in layer.targets { commands.pass(into: target, clear: true) } } - // must match `struct Uniforms` in the shader: time, padding, resolution.xy, params[16] - var uniforms: [Float] = [time, 0, Float(width), Float(height)] + // must match `struct Uniforms` in the shader: time, dt, resolution.xy, params[16] + var uniforms: [Float] = [time, dt, Float(width), Float(height)] uniforms += layer.params.map(\.value) + [Float](repeating: 0, count: maxParams - layer.params.count) let textures = [image, frames.current, frames.last, layer.targets[1], analysis.flow, analysis.particles] commands.pass(into: layer.targets[0]) { $0.fullscreen(pipeline, textures: textures, uniforms: uniforms) } diff --git a/Sources/MotionLab/Shaders.swift b/Sources/MotionLab/Shaders.swift index 53e1a8a..b35536b 100644 --- a/Sources/MotionLab/Shaders.swift +++ b/Sources/MotionLab/Shaders.swift @@ -74,6 +74,7 @@ func effectShader(body: String, params: [EffectParam]) -> String { return shaderPrelude + """ struct Uniforms { float time; + float dt; float2 resolution; float params[\(maxParams)]; }; @@ -89,6 +90,7 @@ func effectShader(body: String, params: [EffectParam]) -> String { texture2d particles; // .r = brightness of particles that drift along with the motion sampler smp; float time; // seconds since start + float dt; // seconds since the previous frame (about 1/30) float2 resolution; // output size in pixels Params p; // this effect's sliders }; @@ -106,7 +108,7 @@ func effectShader(body: String, params: [EffectParam]) -> String { constant Uniforms &u [[buffer(0)]]) { constexpr sampler smp(filter::linear, address::clamp_to_edge); Params p = { \(values) }; - Inputs in = { image, camera, last, prev, flow, particles, smp, u.time, u.resolution, p }; + Inputs in = { image, camera, last, prev, flow, particles, smp, u.time, u.dt, u.resolution, p }; return effect(v.uv, in); } """ diff --git a/Sources/MotionLab/main.swift b/Sources/MotionLab/main.swift index a5a23b7..d83c144 100644 --- a/Sources/MotionLab/main.swift +++ b/Sources/MotionLab/main.swift @@ -51,6 +51,10 @@ if CommandLine.arguments.contains("--check") { _ = try makePipeline(device, source: effectShader(body: source, params: info.params), fragment: "effectFragment", format: .bgra8Unorm) print("ok \(name) sliders: \(info.params.isEmpty ? "none" : info.params.map(\.name).joined(separator: ", "))") + let sliderLines = source.split(separator: "\n").filter { $0.hasPrefix("// slider") }.count + if sliderLines != info.params.count { + print(" warning: \(sliderLines - info.params.count) slider line(s) ignored (malformed, or over the limit of \(maxParams))") + } } catch { failures += 1 print("FAIL \(name)\n\(error.localizedDescription)") diff --git a/effects/ghost-stamps.metal b/effects/ghost-stamps.metal new file mode 100644 index 0000000..a553c8d --- /dev/null +++ b/effects/ghost-stamps.metal @@ -0,0 +1,34 @@ +// Every `stamp` seconds the moving parts of the picture are frozen in place as a translucent, +// tinted ghost; each ghost fades over `life` seconds, so moving things leave a row of copies. +// order 70 +// slider stamp 0.2 3.0 0.5 +// slider life 0.2 6.0 2.0 +// slider hue 0.0 1.0 0.55 +// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue) +static float3 hueColor(float h) { + return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0); +} + +float4 effect(float2 uv, Inputs in) { + // The frozen ghosts are one number per pixel (a tinted brightness), so they live in the + // alpha channel of this effect's own output. The colour is only added when displaying, so + // `in.prev.rgb` never feeds the live picture back into itself. + float ghosts = in.prev.sample(in.smp, uv).a; + + float frame = in.dt; + ghosts *= exp(-frame / max(in.p.life, 0.05)); + + float3 live = in.image.sample(in.smp, uv).rgb; + float diff = length(in.camera.sample(in.smp, uv).rgb - in.last.sample(in.smp, uv).rgb); + float moving = smoothstep(0.03, 0.30, diff); + float fresh = dot(live, float3(0.299, 0.587, 0.114)) * moving; + + // A stamp is taken just after each `stamp`-second slot of in.time begins. The window is two + // frames wide, so a stamp is not missed when the frame rate falls below the nominal one. + float slot = max(in.p.stamp, 0.05); + float begins = 1.0 - step(2.0 * frame / slot, fract(in.time / slot)); + ghosts = max(ghosts, fresh * begins); + + float3 tint = hueColor(in.p.hue); + return float4(live + tint * ghosts * 0.85, ghosts); +} diff --git a/effects/glitch-blocks.metal b/effects/glitch-blocks.metal new file mode 100644 index 0000000..87043b3 --- /dev/null +++ b/effects/glitch-blocks.metal @@ -0,0 +1,57 @@ +// Movement breaks the picture into horizontal bands that jump sideways and flash +// with shifted colours. Still frames pass through unchanged. +// order 70 +// slider band 2.0 200.0 48.0 +// slider intensity 0.0 2.0 1.0 +// slider sensitivity 1.0 60.0 24.0 +// slider shift 0.0 0.3 0.08 +// slider split 0.0 0.06 0.012 +// slider flash 0.0 1.0 0.7 +// slider rate 1.0 30.0 12.0 + +// A repeatable pseudo random number in 0..1 from a 2D seed. +static float hash21(float2 p) { + p = fract(p * float2(123.34, 456.21)); + p += dot(p, p + 45.32); + return fract(p.x * p.y); +} + +// How much the picture moved near this spot, in 0..1. +static float motionAt(float2 uv, Inputs in) { + float2 flow = in.flow.sample(in.smp, uv).xy; + float3 now = in.camera.sample(in.smp, uv).rgb; + float3 before = in.last.sample(in.smp, uv).rgb; + float moved = length(now - before); + return clamp(length(flow) * in.p.sensitivity + moved * in.p.sensitivity * 0.05, 0.0, 1.0); +} + +float4 effect(float2 uv, Inputs in) { + float bandPixels = max(in.p.band, 1.0); + float bandHeight = bandPixels / in.resolution.y; + float band = floor(uv.y / bandHeight); + float centre = (band + 0.5) * bandHeight; + + // The whole band reacts to the motion measured at its middle. + float strength = clamp(motionAt(float2(uv.x, centre), in) * in.p.intensity, 0.0, 1.0); + + // Each band gets fresh random values a few times per second. + float tick = floor(in.time * in.p.rate); + float r1 = hash21(float2(band, tick)); + float r2 = hash21(float2(band + 71.3, tick)); + float r3 = hash21(float2(band + 13.7, tick + 5.0)); + + float slide = (r1 * 2.0 - 1.0) * in.p.shift * strength; + float split = (r2 * 2.0 - 1.0) * in.p.split * strength; + float2 base = float2(uv.x + slide, uv.y); + + float3 col = in.image.sample(in.smp, base).rgb; + col.r = in.image.sample(in.smp, float2(base.x + split, base.y)).r; + col.b = in.image.sample(in.smp, float2(base.x - split, base.y)).b; + + // Flash with a shifted copy of the channels. + float3 other = mix(col.gbr, col.brg, step(0.5, r3)); + col = mix(col, other, in.p.flash * strength); + col *= 1.0 + (r1 - 0.5) * in.p.flash * strength; + + return float4(col, 1.0); +}