diff --git a/.gitignore b/.gitignore index d15b778..8eb2b34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ .build/ captures/ -__pycache__/ .DS_Store -headless.png diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..e1f7da7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,14 @@ +# MotionLab + +A small native macOS app: camera in, a chain of Metal shader effects, out to a window, a recording or a virtual camera. + +- **Adding or changing an effect:** read `EFFECTS.md` first. An effect is one file in `effects/`; the Swift code does not need to change. +- **Build:** `swift build -c release` (Command Line Tools are enough, no Xcode). +- **Check effects without a camera or window:** `.build/release/MotionLab --check` +- **Run:** `.build/release/MotionLab` (options are listed at the top of `Sources/MotionLab/main.swift`). + +Layout: + +- `Sources/MotionLab/` the app. `Shaders.swift` holds the wrapper every effect is compiled into; `Analysis.swift` computes the motion inputs; `Renderer.swift` runs the chain. +- `effects/` one `.metal` file per effect. +- `captures/` screenshots and recordings (not in git). diff --git a/EFFECTS.md b/EFFECTS.md new file mode 100644 index 0000000..24e0017 --- /dev/null +++ b/EFFECTS.md @@ -0,0 +1,123 @@ +# Writing a MotionLab effect + +An effect is one text file: `effects/.metal`. The app picks it up automatically (also while it is running) and shows it in the panel with a checkbox and its sliders. No other file needs to change. + +## The contract + +The file defines exactly one function, written in Metal Shading Language. It is called once per pixel, every frame, and returns that pixel's colour: + +```metal +float4 effect(float2 uv, Inputs in) { + return in.image.sample(in.smp, uv); // this one changes nothing +} +``` + +- `uv` is the pixel's position: `(0,0)` is top-left, `(1,1)` is bottom-right. +- Return `float4(r, g, b, 1.0)` with components in 0..1. +- Do not write `#include`, `using namespace`, a vertex function or a fragment function. The app adds those. + +## What `in` gives you + +| Field | Type | Meaning | +|---|---|---| +| `in.image` | texture | The picture so far: the output of the effect above this one in the chain, or the camera if it is the first. **Read this for your input picture.** | +| `in.camera` | texture | The untouched camera frame. | +| `in.last` | texture | The camera frame before this one. `length(camera - last)` at a pixel tells you how much it changed. | +| `in.prev` | texture | This effect's own output from the previous frame. Read it to build trails and feedback. | +| `in.flow` | texture | `.xy` = which way and how far the picture moved at this spot since the last frame, in `uv` units. Typically tiny (0.001 to 0.02). | +| `in.particles` | texture | `.r` = brightness of particles that spawn where something moves and drift along with the motion. | +| `in.smp` | sampler | Pass it to every `.sample()` call. | +| `in.time` | float | Seconds since the app started. | +| `in.resolution` | float2 | Output size in pixels. One pixel is `1.0 / in.resolution` in `uv` units. | +| `in.p.` | float | Your sliders (see below). | + +Reading a texture: `in.image.sample(in.smp, uv)` returns a `float4`; take `.rgb`, `.r`, `.xy` and so on. Reading outside 0..1 repeats the edge pixel. + +## Comment lines the app reads + +Put these at the top of the file. Spacing matters: one space between the parts, nothing after the last value. + +```metal +// slider +// order +// on +``` + +- `// slider glow 0.0 4.0 1.5` adds a slider; read its value as `in.p.glow`. Up to 16 per effect. +- `// order 60` sets the position in the chain. Lower runs first; the default is 50. Existing effects use 10 to 90. +- `// on` makes the effect enabled when the app starts. Leave it out for new effects. + +Slider names must be plain identifiers and must not be Metal built-ins or keywords (`mix`, `length`, `in`, `float`, `step`, `sample` and so on). `amount`, `speed`, `glow`, `hue` are fine. + +## Metal Shading Language, for people who know GLSL + +| GLSL | Metal | +|---|---| +| `vec2`, `vec3`, `vec4` | `float2`, `float3`, `float4` | +| `texture(tex, uv)` | `tex.sample(in.smp, uv)` | +| `mod(a, b)` | `fmod(a, b)` (keeps the sign of `a`; use `a - b * floor(a / b)` for a positive result) | +| `atan(y, x)` | `atan2(y, x)` | +| `mix`, `clamp`, `smoothstep`, `fract`, `floor`, `length`, `dot`, `sin`, `cos`, `pow`, `abs`, `min`, `max` | the same | + +- Write float numbers with a decimal point: `2.0`, not `2`. Convert loop counters with `float(i)`. +- Helper functions go above `effect`, must be marked `static`, and take `Inputs in` as a parameter if they read textures. +- There is no global state. To remember something between frames, write it into your output and read it back from `in.prev`. The alpha channel (`.a`) of your output is free for this: it is not shown on screen. + +## Two complete examples + +A wavy colour split with two sliders: + +```metal +// Splits red and blue apart in a wave that scrolls down the picture. +// order 60 +// slider amount 0.0 0.05 0.012 +// slider waves 1.0 80.0 24.0 +float4 effect(float2 uv, Inputs in) { + float2 d = float2(in.p.amount * sin(uv.y * in.p.waves + in.time * 3.0), 0.0); + return float4(in.image.sample(in.smp, uv + d).r, + in.image.sample(in.smp, uv).g, + in.image.sample(in.smp, uv - d).b, 1.0); +} +``` + +Things that move light up and the glow fades slowly, using `in.prev` alpha as memory: + +```metal +// Whatever moves lights up, and the glow lingers. +// order 50 +// slider sensitivity 1.0 30.0 10.0 +// slider linger 0.5 0.99 0.93 +float4 effect(float2 uv, Inputs in) { + float moved = length(in.camera.sample(in.smp, uv).rgb - in.last.sample(in.smp, uv).rgb); + float fresh = smoothstep(0.3, 1.0, moved * in.p.sensitivity); + float heat = max(in.prev.sample(in.smp, uv).a * in.p.linger, fresh); + float3 base = in.image.sample(in.smp, uv).rgb * 0.5; + return float4(base + float3(1.0, 0.5, 0.1) * heat, heat); +} +``` + +More examples are in `effects/`. + +## Checking your work + +Build once, then check. Both commands print compiler errors with the line number in your file and exit non-zero on failure. + +```bash +swift build -c release + +# compiles every effect; needs no camera and opens no window +.build/release/MotionLab --check + +# runs only your effect on the live camera for 90 frames, saves the last frame, exits +.build/release/MotionLab --only --snapshot /tmp/.png +``` + +`--check` also lists the sliders it found for each effect. If one of yours is missing, its `// slider` line is malformed. + +## Checklist + +- The file is `effects/.metal` and defines `float4 effect(float2 uv, Inputs in)`. +- It reads its input picture from `in.image`, not `in.camera`, so it works in a chain. +- Every number a user might want to tune is a slider with a sensible range and default. +- The first line is a comment saying what the effect does. +- `.build/release/MotionLab --check` prints `ok` for it. diff --git a/native/Package.swift b/Package.swift similarity index 100% rename from native/Package.swift rename to Package.swift diff --git a/native/Sources/MotionLab/Analysis.swift b/Sources/MotionLab/Analysis.swift similarity index 100% rename from native/Sources/MotionLab/Analysis.swift rename to Sources/MotionLab/Analysis.swift diff --git a/native/Sources/MotionLab/Camera.swift b/Sources/MotionLab/Camera.swift similarity index 100% rename from native/Sources/MotionLab/Camera.swift rename to Sources/MotionLab/Camera.swift diff --git a/native/Sources/MotionLab/Output.swift b/Sources/MotionLab/Output.swift similarity index 100% rename from native/Sources/MotionLab/Output.swift rename to Sources/MotionLab/Output.swift diff --git a/native/Sources/MotionLab/Panel.swift b/Sources/MotionLab/Panel.swift similarity index 100% rename from native/Sources/MotionLab/Panel.swift rename to Sources/MotionLab/Panel.swift diff --git a/native/Sources/MotionLab/Renderer.swift b/Sources/MotionLab/Renderer.swift similarity index 100% rename from native/Sources/MotionLab/Renderer.swift rename to Sources/MotionLab/Renderer.swift diff --git a/native/Sources/MotionLab/Shaders.swift b/Sources/MotionLab/Shaders.swift similarity index 100% rename from native/Sources/MotionLab/Shaders.swift rename to Sources/MotionLab/Shaders.swift diff --git a/native/Sources/MotionLab/main.swift b/Sources/MotionLab/main.swift similarity index 80% rename from native/Sources/MotionLab/main.swift rename to Sources/MotionLab/main.swift index eaa7c1c..a5a23b7 100644 --- a/native/Sources/MotionLab/main.swift +++ b/Sources/MotionLab/main.swift @@ -5,6 +5,7 @@ // --effects DIR folder of .metal effect files (default ./effects) // --only a,b,c start with exactly these effects enabled // --virtualcam also send the output to the OBS virtual camera (for Meet, Zoom, ...) +// --check compile every effect file, report errors, and exit (no camera, no window) // --snapshot OUT.png save one frame after --frames N camera frames (default 90) and exit // --record OUT.mp4 record --frames N camera frames (default 90) and exit // @@ -36,6 +37,28 @@ setvbuf(stdout, nil, _IOLBF, 0) // show "loaded" / shader errors immediately, e guard let device = MTLCreateSystemDefaultDevice() else { fail("Metal is not available on this machine.") } let effectsDirectory = URL(fileURLWithPath: option("--effects") ?? "effects") +if CommandLine.arguments.contains("--check") { + // compile every effect file without opening the camera or a window + let files = ((try? FileManager.default.contentsOfDirectory(at: effectsDirectory, includingPropertiesForKeys: nil)) ?? []) + .filter { $0.pathExtension == "metal" } + .sorted { $0.lastPathComponent < $1.lastPathComponent } + var failures = 0 + for url in files { + let name = url.deletingPathExtension().lastPathComponent + do { + let source = try String(contentsOf: url, encoding: .utf8) + let info = parseEffect(source) + _ = try makePipeline(device, source: effectShader(body: source, params: info.params), + fragment: "effectFragment", format: .bgra8Unorm) + print("ok \(name) sliders: \(info.params.isEmpty ? "none" : info.params.map(\.name).joined(separator: ", "))") + } catch { + failures += 1 + print("FAIL \(name)\n\(error.localizedDescription)") + } + } + exit(failures == 0 ? 0 : 1) +} + let renderer: Renderer do { renderer = try Renderer(device: device, camera: Camera(device: device), effectsDirectory: effectsDirectory) diff --git a/effects.py b/effects.py deleted file mode 100644 index 579435f..0000000 --- a/effects.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Effect layers. - -Each layer takes the current BGR frame plus a shared Context (motion analysis -of the raw camera frame) and returns a new BGR frame. Layers run top to bottom -in the order returned by default_layers(), like a chain of TOPs. -""" -import cv2 -import numpy as np - -rng = np.random.default_rng() - - -class Param: - def __init__(self, value, lo=0.0, hi=1.0, integer=False): - self.value, self.lo, self.hi, self.integer = value, lo, hi, integer - - @property - def norm(self): - return (self.value - self.lo) / (self.hi - self.lo) - - @norm.setter - def norm(self, n): - v = self.lo + min(max(n, 0.0), 1.0) * (self.hi - self.lo) - self.value = round(v) if self.integer else v - - -class Layer: - name = "layer" - enabled = True - - def __init__(self, **params): - self.params = params - - def __getitem__(self, key): - return self.params[key].value - - def apply(self, frame, ctx): - return frame - - -class Context: - """Per-frame motion analysis, computed at 1/4 resolution and shared by all layers.""" - - SCALE = 4 - - def __init__(self, size): - self.small = (size[0] // self.SCALE, size[1] // self.SCALE) - self.prev = None - self.t = 0.0 - self.dt = 1 / 30 - - def update(self, frame, dt): - self.dt = dt - self.t += dt - gray = cv2.cvtColor(cv2.resize(frame, self.small, interpolation=cv2.INTER_AREA), cv2.COLOR_BGR2GRAY) - if self.prev is None: - self.prev = gray - self.motion = cv2.absdiff(gray, self.prev) # where things changed - self.energy = float(self.motion.mean()) / 255 # how much changed overall - self._pair = (self.prev, gray) - self._flow = None - self.prev = gray - - @property - def flow(self): - """Dense optical flow (small-res pixels/frame). Lazy: only paid for if a layer asks.""" - if self._flow is None: - self._flow = cv2.calcOpticalFlowFarneback(*self._pair, None, 0.5, 2, 15, 2, 5, 1.1, 0) - return self._flow - - -def hue_bgr(hue): - """Hue in 0..1 (wraps) -> saturated BGR colour with components in 0..1.""" - hsv = np.uint8([[[int(hue % 1.0 * 180), 255, 255]]]) - return cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)[0, 0] / 255.0 - - -def tint(gray, color): - return cv2.merge([cv2.convertScaleAbs(gray, alpha=float(c)) for c in color]) - - -class Colorize(Layer): - name = "Colorize" - enabled = False - PALETTES = [cv2.COLORMAP_INFERNO, cv2.COLORMAP_OCEAN, cv2.COLORMAP_TWILIGHT_SHIFTED, cv2.COLORMAP_TURBO] - - def __init__(self): - super().__init__(mix=Param(0.8), palette=Param(0, 0, len(self.PALETTES) - 1, integer=True)) - - def apply(self, frame, ctx): - gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - mapped = cv2.applyColorMap(gray, self.PALETTES[self["palette"]]) - return cv2.addWeighted(frame, 1 - self["mix"], mapped, self["mix"], 0) - - -class Edges(Layer): - name = "Neon Edges" - - def __init__(self): - super().__init__(detail=Param(0.6), glow=Param(0.35), source=Param(0.35), - hue=Param(0.55), cycle=Param(0.3)) - - def apply(self, frame, ctx): - gray = cv2.GaussianBlur(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), (5, 5), 0) - lo = 20 + 180 * (1 - self["detail"]) - edges = cv2.Canny(gray, lo, lo * 2) - sigma = self["glow"] * 12 - if sigma > 0.5: - edges = cv2.addWeighted(edges, 1, cv2.GaussianBlur(edges, (0, 0), sigma), 2.5, 0) - color = hue_bgr(self["hue"] + ctx.t * self["cycle"] * 0.2) - return cv2.add(cv2.convertScaleAbs(frame, alpha=self["source"]), tint(edges, color)) - - -class Particles(Layer): - """Particles spawn where there is motion and get carried along by the optical flow.""" - - name = "Flow Particles" - MAX = 6000 - - def __init__(self): - super().__init__(count=Param(3000, 200, self.MAX, integer=True), speed=Param(1.5, 0, 4), - trail=Param(0.88, 0, 0.98), hue=Param(0.08)) - self.buf = None - - def apply(self, frame, ctx): - h, w = frame.shape[:2] - if self.buf is None: - self.buf = np.zeros((h, w), np.uint8) - self.pos = (rng.random((self.MAX, 2)) * (w, h)).astype(np.float32) - self.life = rng.integers(20, 90, self.MAX) - - n = self["count"] - pos, life = self.pos[:n], self.life[:n] - flow = ctx.flow - fh, fw = flow.shape[:2] - scale = w / fw - ix = np.clip((pos[:, 0] / scale).astype(int), 0, fw - 1) - iy = np.clip((pos[:, 1] / scale).astype(int), 0, fh - 1) - new = pos + flow[iy, ix] * scale * self["speed"] + rng.normal(0, 0.5, (n, 2)).astype(np.float32) - - life -= 1 - dead = (life <= 0) | (new[:, 0] < 0) | (new[:, 0] >= w) | (new[:, 1] < 0) | (new[:, 1] >= h) - - self.buf = cv2.convertScaleAbs(self.buf, alpha=self["trail"]) - segments = np.stack([pos, new], axis=1).astype(np.int32)[~dead] - cv2.polylines(self.buf, list(segments), False, 255, 1, cv2.LINE_AA) - - k = int(dead.sum()) - if k: - hot = np.flatnonzero(ctx.motion > 24) - if hot.size > 50: - ys, xs = np.divmod(rng.choice(hot, k), fw) - new[dead] = (np.stack([xs, ys], axis=1) + rng.random((k, 2))) * scale - else: - new[dead] = rng.random((k, 2)) * (w, h) - life[dead] = rng.integers(20, 90, k) - self.pos[:n] = new - - glow = cv2.add(self.buf, cv2.GaussianBlur(self.buf, (0, 0), 3)) - return cv2.add(frame, tint(glow, hue_bgr(self["hue"]))) - - -class Feedback(Layer): - """Classic feedback loop: the previous output is transformed, faded and blended back in.""" - - name = "Feedback" - - def __init__(self): - super().__init__(decay=Param(0.92, 0, 0.995), zoom=Param(0.012, -0.05, 0.05), - rotate=Param(0.3, -3, 3), **{"flow push": Param(0.0, 0, 4)}) - self.prev = None - - def apply(self, frame, ctx): - h, w = frame.shape[:2] - if self.prev is None: - self.prev = frame - self.grid = np.stack(np.meshgrid(np.arange(w, dtype=np.float32), - np.arange(h, dtype=np.float32)), axis=-1) - m = cv2.getRotationMatrix2D((w / 2, h / 2), self["rotate"], 1 + self["zoom"]) - fb = cv2.warpAffine(self.prev, m, (w, h), borderMode=cv2.BORDER_REFLECT) - if self["flow push"] > 0: - # smear the feedback along the direction things are moving - flow = cv2.resize(ctx.flow, (w, h)) * (self["flow push"] * Context.SCALE) - fb = cv2.remap(fb, self.grid - flow, None, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT) - out = cv2.max(frame, cv2.convertScaleAbs(fb, alpha=self["decay"])) - self.prev = out - return out - - -class RGBShift(Layer): - name = "RGB Shift" - - def __init__(self): - super().__init__(amount=Param(4, 0, 40), **{"motion react": Param(0.5)}) - - def apply(self, frame, ctx): - d = int(self["amount"] + self["motion react"] * ctx.energy * 400) - if d == 0: - return frame - b, g, r = cv2.split(frame) - return cv2.merge([np.roll(b, d, axis=1), g, np.roll(r, -d, axis=1)]) - - -class Kaleido(Layer): - name = "Kaleidoscope" - enabled = False - - def __init__(self): - super().__init__(segments=Param(6, 2, 16, integer=True), spin=Param(0.2, -2, 2)) - self.radius = None - self.angle = 0.0 - - def apply(self, frame, ctx): - h, w = frame.shape[:2] - if self.radius is None: - ys, xs = np.indices((h, w), dtype=np.float32) - self.radius, self.theta = cv2.cartToPolar(xs - w / 2, ys - h / 2) - self.angle += self["spin"] * ctx.dt - wedge = 2 * np.pi / self["segments"] - # fold every angle into one mirrored wedge, then sample the source from that wedge - a = np.abs(np.mod(self.theta + np.float32(self.angle), np.float32(wedge)) - np.float32(wedge / 2)) - x, y = cv2.polarToCart(self.radius, a - np.float32(np.pi / 2)) - return cv2.remap(frame, x + w / 2, y + h / 2, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT) - - -def default_layers(): - return [Colorize(), Edges(), Particles(), Feedback(), RGBShift(), Kaleido()] diff --git a/native/effects/colorize.metal b/effects/colorize.metal similarity index 100% rename from native/effects/colorize.metal rename to effects/colorize.metal diff --git a/effects/halftone.glsl b/effects/halftone.glsl deleted file mode 100644 index 404aa48..0000000 --- a/effects/halftone.glsl +++ /dev/null @@ -1,17 +0,0 @@ -// Halftone dots sized by brightness; dots turn hot orange wherever something moves. -uniform float cell; // 4.0 .. 40.0 = 10.0 -uniform float heat; // 0.0 .. 6.0 = 3.0 - -void main() { - vec2 grid = resolution / cell; - vec2 centre = (floor(uv * grid) + 0.5) / grid; - vec3 c = texture(camera, centre).rgb; - float luma = dot(c, vec3(0.299, 0.587, 0.114)); - - float radius = 0.5 * sqrt(luma); - float dist = length(fract(uv * grid) - 0.5); - float mask = 1.0 - smoothstep(radius - 0.05, radius + 0.05, dist); - - float moving = clamp(texture(motion, centre).r * heat, 0.0, 1.0); - color = vec4(mix(c, vec3(1.0, 0.35, 0.1), moving) * mask, 1.0); -} diff --git a/native/effects/halftone.metal b/effects/halftone.metal similarity index 100% rename from native/effects/halftone.metal rename to effects/halftone.metal diff --git a/native/effects/kaleidoscope.metal b/effects/kaleidoscope.metal similarity index 100% rename from native/effects/kaleidoscope.metal rename to effects/kaleidoscope.metal diff --git a/effects/liquid.glsl b/effects/liquid.glsl deleted file mode 100644 index 9e75474..0000000 --- a/effects/liquid.glsl +++ /dev/null @@ -1,13 +0,0 @@ -// Liquid trails: the previous output is dragged along the motion, swirled a little, -// and slowly fades back to the live camera. -uniform float persistence; // 0.0 .. 0.99 = 0.92 -uniform float drag; // 0.0 .. 6.0 = 2.5 -uniform float swirl; // 0.0 .. 0.01 = 0.002 - -void main() { - vec2 centred = uv - 0.5; - vec2 from = uv - texture(flow, uv).xy * drag + vec2(-centred.y, centred.x) * swirl; - vec3 trail = texture(prev, from).rgb; - vec3 live = texture(camera, uv).rgb; - color = vec4(mix(live, trail, persistence), 1.0); -} diff --git a/native/effects/liquid.metal b/effects/liquid.metal similarity index 100% rename from native/effects/liquid.metal rename to effects/liquid.metal diff --git a/native/effects/motion-glow.metal b/effects/motion-glow.metal similarity index 100% rename from native/effects/motion-glow.metal rename to effects/motion-glow.metal diff --git a/native/effects/neon-edges.metal b/effects/neon-edges.metal similarity index 100% rename from native/effects/neon-edges.metal rename to effects/neon-edges.metal diff --git a/native/effects/particles.metal b/effects/particles.metal similarity index 100% rename from native/effects/particles.metal rename to effects/particles.metal diff --git a/native/effects/trails.metal b/effects/trails.metal similarity index 100% rename from native/effects/trails.metal rename to effects/trails.metal diff --git a/effects/wave-shift.glsl b/effects/wave-shift.glsl deleted file mode 100644 index 993b467..0000000 --- a/effects/wave-shift.glsl +++ /dev/null @@ -1,12 +0,0 @@ -// RGB split that ripples down the image and gets pushed around by your movement. -uniform float amount; // 0.0 .. 0.05 = 0.012 -uniform float waves; // 1.0 .. 80.0 = 24.0 -uniform float react; // 0.0 .. 3.0 = 1.0 - -void main() { - vec2 d = vec2(amount * sin(uv.y * waves + time * 3.0), 0.0); - d += texture(flow, uv).xy * react; - color = vec4(texture(camera, uv + d).r, - texture(camera, uv).g, - texture(camera, uv - d).b, 1.0); -} diff --git a/native/effects/wave-shift.metal b/effects/wave-shift.metal similarity index 100% rename from native/effects/wave-shift.metal rename to effects/wave-shift.metal diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 0433a0f..0000000 --- a/flake.lock +++ /dev/null @@ -1,27 +0,0 @@ -{ - "nodes": { - "nixpkgs": { - "locked": { - "lastModified": 1791097562, - "narHash": "sha256-K9O9lG3UZNFCnwrz0ViWgP3Kgqukg1DvvzsrkFJfcvU=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "a322c65bf7233456788a9ebf03bc067db7dc090a", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "nixpkgs": "nixpkgs" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 5b51489..0000000 --- a/flake.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ - description = "motionlab - realtime camera + motion graphics playground"; - - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - - outputs = { self, nixpkgs }: - let - systems = [ "aarch64-darwin" "x86_64-darwin" "x86_64-linux" "aarch64-linux" ]; - forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); - - # not in nixpkgs; sends frames to the OBS virtual camera (macOS) or v4l2loopback (Linux) - pyvirtualcamFor = ps: ps.buildPythonPackage rec { - pname = "pyvirtualcam"; - version = "0.15.0"; - pyproject = true; - src = nixpkgs.legacyPackages.${ps.python.stdenv.hostPlatform.system}.fetchFromGitHub { - owner = "letmaik"; - repo = "pyvirtualcam"; - tag = "v${version}"; - fetchSubmodules = true; - hash = "sha256-JtFchhuvVYYhrYOOcsHPC79/Iezyo5QppV4tY9nrjgQ="; - }; - build-system = [ ps.setuptools ps.pybind11 ]; - dependencies = [ ps.numpy ]; - doCheck = false; - pythonImportsCheck = [ "pyvirtualcam" ]; - }; - - pythonFor = pkgs: pkgs.python3.withPackages (ps: [ - ps.numpy - ps.moderngl - (pyvirtualcamFor ps) - # macOS gets a Cocoa window out of the box; Linux needs GTK for cv2.imshow - (if pkgs.stdenv.hostPlatform.isLinux - then ps.opencv4.override { enableGtk3 = true; } - else ps.opencv4) - ]); - in - { - devShells = forAllSystems (pkgs: { - default = pkgs.mkShell { packages = [ (pythonFor pkgs) ]; }; - }); - - apps = forAllSystems (pkgs: { - default = { - type = "app"; - program = toString (pkgs.writeShellScript "motionlab" '' - exec ${pythonFor pkgs}/bin/python ${self}/main.py "$@" - ''); - }; - }); - }; -} diff --git a/main.py b/main.py deleted file mode 100644 index 6f8c402..0000000 --- a/main.py +++ /dev/null @@ -1,191 +0,0 @@ -"""motionlab - a tiny TouchDesigner-style playground: live camera in, motion-reactive graphics out.""" -import argparse -import os -import sys -import time - -import cv2 -import numpy as np - -from effects import Context, default_layers, hue_bgr -from panel import Panel -from shaders import ShaderBank - -SIZE = (960, 540) -VCAM_SIZE = (1280, 720) -WINDOW = "motionlab" - - -class Synthetic: - """Drifting blobs, so the app (and --headless) works without a camera.""" - - def __init__(self): - self.i = 0 - - def read(self): - w, h = SIZE - t = self.i / 30 - self.i += 1 - img = np.full((h, w, 3), (30, 20, 20), np.uint8) - for k in range(5): - x = int(w * (0.5 + 0.4 * np.sin(t * (0.7 + 0.2 * k) + k))) - y = int(h * (0.5 + 0.4 * np.cos(t * (0.5 + 0.3 * k) + 2 * k))) - cv2.circle(img, (x, y), 40 + 12 * k, hue_bgr(k / 5) * 255, -1) - return True, cv2.GaussianBlur(img, (0, 0), 4) - - def release(self): - pass - - -def open_source(args): - if args.demo or args.headless: - return Synthetic() - backend = cv2.CAP_AVFOUNDATION if sys.platform == "darwin" else cv2.CAP_ANY - cap = cv2.VideoCapture(args.camera, backend) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720) - # without camera permission macOS can hand back all-black frames instead of failing - for _ in range(10): - ok, frame = cap.read() - if ok and frame.any(): - return cap - cap.release() - print("Could not read from the camera - falling back to the synthetic demo source.\n" - "On macOS: allow your terminal under System Settings > Privacy & Security > Camera, then rerun.", - file=sys.stderr) - return Synthetic() - - -def open_virtualcam(): - """Virtual camera other apps (Meet, Zoom, ...) can pick as their webcam.""" - import pyvirtualcam - try: - cam = pyvirtualcam.Camera(*VCAM_SIZE, 30, fmt=pyvirtualcam.PixelFormat.BGR) - except RuntimeError as e: - sys.exit(f"Could not start the virtual camera: {e}") - print(f"virtual camera on: pick '{cam.device}' as the camera in your call app") - return cam - - -def fit(frame): - """Centre-crop to the output aspect ratio, then resize.""" - w, h = SIZE - fh, fw = frame.shape[:2] - cw = min(fw, fh * w // h) - ch = min(fh, fw * h // w) - x, y = (fw - cw) // 2, (fh - ch) // 2 - return cv2.resize(frame[y:y + ch, x:x + cw], SIZE, interpolation=cv2.INTER_AREA) - - -def capture_path(ext): - os.makedirs("captures", exist_ok=True) - return os.path.join("captures", time.strftime(f"motionlab-%Y%m%d-%H%M%S.{ext}")) - - -def main(): - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--camera", type=int, default=0, help="camera index (default 0)") - ap.add_argument("--demo", action="store_true", help="use the synthetic source instead of the camera") - ap.add_argument("--virtualcam", action="store_true", help="also send the output to a virtual camera for video calls") - ap.add_argument("--headless", type=int, metavar="N", help="render N synthetic frames without a window, then exit") - ap.add_argument("--out", default="headless.png", help="where --headless writes its last frame") - ap.add_argument("--effects", default="effects", help="folder of .glsl shader effects (default ./effects)") - ap.add_argument("--only", metavar="NAMES", help="enable only these layers, comma separated (e.g. --only liquid,halftone)") - args = ap.parse_args() - - w, h = SIZE - src = open_source(args) - layers = default_layers() - # prefer ./effects so edits are picked up live; fall back to the copy shipped next to this file - effects_dir = args.effects if os.path.isdir(args.effects) else os.path.join(os.path.dirname(__file__), "effects") - bank = ShaderBank(effects_dir, layers) - if args.only: - wanted = {name.strip().lower() for name in args.only.split(",")} - unknown = wanted - {layer.name.lower() for layer in layers} - if unknown: - sys.exit(f"unknown layer(s): {', '.join(sorted(unknown))}. Available: {', '.join(l.name for l in layers)}") - for layer in layers: - layer.enabled = layer.name.lower() in wanted - if args.headless and bank.errors: - sys.exit(f"shader compile failed: {', '.join(bank.errors)}") - ctx = Context(SIZE) - panel = Panel(layers) - canvas = np.empty((h, w + Panel.WIDTH, 3), np.uint8) - show_panel, writer, fps = True, None, 30.0 - vcam = open_virtualcam() if args.virtualcam else None - - if not args.headless: - cv2.namedWindow(WINDOW, cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) - cv2.resizeWindow(WINDOW, w + Panel.WIDTH, h) - cv2.setMouseCallback(WINDOW, panel.on_mouse) - - start = last = time.perf_counter() - frames = 0 - while True: - ok, frame = src.read() - if not ok: - break - now = time.perf_counter() - dt = 1 / 30 if args.headless else max(now - last, 1e-3) - last = now - fps += (1 / dt - fps) * 0.1 - - frame = cv2.flip(fit(frame), 1) # mirror, so it behaves like a mirror - if frames % 15 == 0: - bank.refresh() # pick up new and edited shader files - ctx.update(frame, dt) - out = frame - for layer in layers: - if layer.enabled: - out = layer.apply(out, ctx) - - if writer: - writer.write(out) - if vcam: - # un-mirror: the preview is a mirror, but the people on the call should see the real thing - vcam.send(cv2.resize(cv2.flip(out, 1), VCAM_SIZE)) - view = out - if show_panel: - canvas[:, :w] = out - panel.draw(canvas, w, f"{fps:3.0f} fps" + (" REC" if writer else "") + (" CAM" if vcam else "")) - view = canvas - else: - panel.hits = [] - - frames += 1 - if args.headless: - if frames >= args.headless: - cv2.imwrite(args.out, view) - print(f"{frames} frames, {(time.perf_counter() - start) / frames * 1000:.1f} ms/frame -> {args.out}") - break - continue - - cv2.imshow(WINDOW, view) - key = cv2.waitKey(1) & 0xFF - if key in (ord("q"), 27) or cv2.getWindowProperty(WINDOW, cv2.WND_PROP_VISIBLE) < 1: - break - elif key == ord("h"): - show_panel = not show_panel - elif key == ord("s"): - path = capture_path("png") - cv2.imwrite(path, out) - print("saved", path) - elif key == ord("r"): - if writer: - writer.release() - writer = None - else: - path = capture_path("mp4") - writer = cv2.VideoWriter(path, cv2.VideoWriter_fourcc(*"mp4v"), round(fps), SIZE) - print("recording", path) - - if writer: - writer.release() - if vcam: - vcam.close() - src.release() - cv2.destroyAllWindows() - - -if __name__ == "__main__": - main() diff --git a/panel.py b/panel.py deleted file mode 100644 index 6290a5c..0000000 --- a/panel.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Control panel drawn straight onto the output canvas: click a layer to toggle it, drag its sliders.""" -import cv2 - -BG = (26, 24, 22) -TRACK = (52, 48, 46) -FG = (228, 228, 228) -DIM = (125, 125, 125) -ACCENT = (255, 176, 64) -FONT = cv2.FONT_HERSHEY_SIMPLEX - - -class Panel: - WIDTH = 260 - - def __init__(self, layers): - self.layers = layers - self.hits = [] # (x0, y0, x1, y1, target, bar_x0, bar_x1), rebuilt on every draw - self.drag = None - - def draw(self, canvas, x0, status): - h = canvas.shape[0] - canvas[:, x0:] = BG - self.hits = [] - x, right = x0 + 14, x0 + self.WIDTH - 14 - cv2.putText(canvas, "MOTIONLAB", (x, 26), FONT, 0.55, FG, 1, cv2.LINE_AA) - cv2.putText(canvas, status, (x + 124, 26), FONT, 0.4, DIM, 1, cv2.LINE_AA) - - y = 44 - for layer in self.layers: - on = layer.enabled - cv2.rectangle(canvas, (x, y), (x + 12, y + 12), ACCENT if on else DIM, -1 if on else 1) - cv2.putText(canvas, layer.name, (x + 22, y + 11), FONT, 0.48, FG if on else DIM, 1, cv2.LINE_AA) - self.hits.append((x0, y - 5, x0 + self.WIDTH, y + 17, layer, 0, 0)) - y += 22 - if not on: - continue - bar = x + 100 - for name, p in layer.params.items(): - cv2.putText(canvas, name, (x + 4, y + 8), FONT, 0.36, DIM, 1, cv2.LINE_AA) - cv2.rectangle(canvas, (bar, y), (right, y + 8), TRACK, -1) - cv2.rectangle(canvas, (bar, y), (bar + int((right - bar) * p.norm), y + 8), ACCENT, -1) - self.hits.append((bar - 8, y - 4, right + 8, y + 12, p, bar, right)) - y += 16 - y += 4 - - cv2.putText(canvas, "[h] panel [s] shot [r] rec [q] quit", (x, h - 12), FONT, 0.36, DIM, 1, cv2.LINE_AA) - - def on_mouse(self, event, mx, my, flags, _): - if event == cv2.EVENT_LBUTTONDOWN: - for x0, y0, x1, y1, target, bar_x0, bar_x1 in self.hits: - if x0 <= mx < x1 and y0 <= my < y1: - if bar_x1: - self.drag = (target, bar_x0, bar_x1) - else: - target.enabled = not target.enabled - break - elif event == cv2.EVENT_LBUTTONUP: - self.drag = None - if self.drag and event in (cv2.EVENT_LBUTTONDOWN, cv2.EVENT_MOUSEMOVE): - param, bar_x0, bar_x1 = self.drag - param.norm = (mx - bar_x0) / (bar_x1 - bar_x0) diff --git a/shaders.py b/shaders.py deleted file mode 100644 index 52b7b0f..0000000 --- a/shaders.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Shader layers: every effects/*.glsl file becomes a layer, reloaded whenever the file is saved. - -A shader file only contains its own sliders and a main(). The host supplies the rest: - - in vec2 uv; pixel position, (0,0) top-left .. (1,1) bottom-right - out vec4 color; what to write for this pixel - uniform sampler2D camera; the incoming frame (output of the layer above), RGB - uniform sampler2D prev; this layer's own output from the previous frame - uniform sampler2D flow; .xy = how far this pixel moved since last frame, in uv units - uniform sampler2D motion; .r = how much this pixel changed since last frame, 0..1 - uniform float time; seconds since start - uniform float energy; overall amount of motion in the frame, 0..1 - uniform vec2 resolution; output size in pixels - -Sliders are plain float uniforms with a range comment: - - uniform float amount; // 0.0 .. 0.05 = 0.01 -""" -import os -import re - -import moderngl -import numpy as np - -from effects import Layer, Param - -VERTEX = """#version 410 -out vec2 uv; -void main() { - uv = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); - gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); -} -""" - -# "#line 1" makes compiler errors report line numbers of the user's file -HEADER = """#version 410 -uniform sampler2D camera; -uniform sampler2D prev; -uniform sampler2D flow; -uniform sampler2D motion; -uniform float time; -uniform float energy; -uniform vec2 resolution; -in vec2 uv; -out vec4 color; -#line 1 -""" - -NUM = r"(-?[\d.]+)" -PARAM_RE = re.compile(rf"uniform\s+float\s+(\w+)\s*;(?:[ \t]*//\s*{NUM}\s*\.\.\s*{NUM}\s*=\s*{NUM})?") - - -class ShaderLayer(Layer): - enabled = False - - def __init__(self, gl, path): - super().__init__() - self.gl, self.path = gl, path - self.name = os.path.splitext(os.path.basename(path))[0] - self.program = None - self.error = None - self.mtime = None - self.targets = None - self.reload() - - def reload(self): - """Recompile if the file changed. On a compile error the last working version keeps running.""" - try: - mtime = os.stat(self.path).st_mtime - except FileNotFoundError: - return - if mtime == self.mtime: - return - self.mtime = mtime - with open(self.path) as f: - source = f.read() - try: - program = self.gl.program(vertex_shader=VERTEX, fragment_shader=HEADER + source) - except moderngl.Error as e: - self.error = str(e) - print(f"[{self.name}] shader error:\n{e}", flush=True) - return - self.error = None - self.program = program - self.vao = self.gl.vertex_array(program, []) - old = self.params - self.params = {name: Param(float(default or 0.5), float(lo or 0), float(hi or 1)) - for name, lo, hi, default in PARAM_RE.findall(source)} - for name, p in self.params.items(): # keep slider positions across reloads - if name in old: - p.value = min(max(old[name].value, p.lo), p.hi) - print(f"[{self.name}] loaded", flush=True) - - def _target(self, size): - texture = self.gl.texture(size, 4) - texture.repeat_x = texture.repeat_y = False - return texture, self.gl.framebuffer([texture]) - - def _texture(self, attr, size, components, dtype="f1"): - if not hasattr(self, attr): - texture = self.gl.texture(size, components, dtype=dtype) - texture.repeat_x = texture.repeat_y = False - setattr(self, attr, texture) - return getattr(self, attr) - - def _set(self, name, value): - uniform = self.program.get(name, None) # absent if the shader doesn't use it - if uniform is not None: - uniform.value = value - return uniform is not None - - def apply(self, frame, ctx): - if self.program is None: - return frame - h, w = frame.shape[:2] - if self.targets is None: - # two render targets, swapped every frame, so the shader can read its previous output - self.targets = [self._target((w, h)), self._target((w, h))] - (_, framebuffer), (prev, _) = self.targets - - camera = self._texture("camera", (w, h), 3) - camera.swizzle = "BGR1" # frames are BGR; shaders see RGB - camera.write(np.ascontiguousarray(frame)) - camera.use(0) - self._set("camera", 0) - prev.use(1) - self._set("prev", 1) - if self._set("flow", 2): - fh, fw = ctx.flow.shape[:2] - flow = self._texture("flow_tex", (fw, fh), 2, "f4") - flow.write((ctx.flow / (fw, fh)).astype("f4")) - flow.use(2) - if self._set("motion", 3): - mh, mw = ctx.motion.shape - motion = self._texture("motion_tex", (mw, mh), 1) - motion.write(ctx.motion) - motion.use(3) - self._set("time", ctx.t) - self._set("energy", ctx.energy) - self._set("resolution", (w, h)) - for name, p in self.params.items(): - self._set(name, p.value) - - framebuffer.use() - self.vao.render(moderngl.TRIANGLES, vertices=3) - out = np.frombuffer(framebuffer.read(components=3), np.uint8).reshape(h, w, 3) - self.targets.reverse() - return np.ascontiguousarray(out[:, :, ::-1]) - - -class ShaderBank: - """Watches a folder of .glsl files and keeps one ShaderLayer per file in the shared layer list.""" - - def __init__(self, directory, layers): - self.directory, self.layers = directory, layers - self.known = {} - self.gl = moderngl.create_context(standalone=True, require=410) - self.refresh() - - def refresh(self): - if os.path.isdir(self.directory): - for filename in sorted(os.listdir(self.directory)): - path = os.path.join(self.directory, filename) - if filename.endswith(".glsl") and path not in self.known: - self.known[path] = ShaderLayer(self.gl, path) - self.layers.append(self.known[path]) - for layer in self.known.values(): - layer.reload() - - @property - def errors(self): - return [layer.name for layer in self.known.values() if layer.error]