From 6591138bbf0231fcdf4e770af633e80efa281d54 Mon Sep 17 00:00:00 2001 From: 0xrsydn Date: Mon, 5 Oct 2026 20:26:47 +0700 Subject: [PATCH] Python/OpenCV proof of concept plus first native Swift port Snapshot before the Python POC is removed. The POC (main.py, effects.py, panel.py, shaders.py, effects/*.glsl, flake.nix) is the reference for the native app under native/. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 5 + effects.py | 227 +++++++++++++++++++ effects/halftone.glsl | 17 ++ effects/liquid.glsl | 13 ++ effects/wave-shift.glsl | 12 + flake.lock | 27 +++ flake.nix | 53 +++++ main.py | 191 ++++++++++++++++ native/Package.swift | 10 + native/Sources/MotionLab/Analysis.swift | 180 +++++++++++++++ native/Sources/MotionLab/Camera.swift | 95 ++++++++ native/Sources/MotionLab/Output.swift | 198 ++++++++++++++++ native/Sources/MotionLab/Panel.swift | 92 ++++++++ native/Sources/MotionLab/Renderer.swift | 287 ++++++++++++++++++++++++ native/Sources/MotionLab/Shaders.swift | 161 +++++++++++++ native/Sources/MotionLab/main.swift | 110 +++++++++ native/effects/colorize.metal | 12 + native/effects/halftone.metal | 18 ++ native/effects/kaleidoscope.metal | 13 ++ native/effects/liquid.metal | 13 ++ native/effects/motion-glow.metal | 20 ++ native/effects/neon-edges.metal | 38 ++++ native/effects/particles.metal | 22 ++ native/effects/trails.metal | 16 ++ native/effects/wave-shift.metal | 13 ++ panel.py | 61 +++++ shaders.py | 172 ++++++++++++++ 27 files changed, 2076 insertions(+) create mode 100644 .gitignore create mode 100644 effects.py create mode 100644 effects/halftone.glsl create mode 100644 effects/liquid.glsl create mode 100644 effects/wave-shift.glsl create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 main.py create mode 100644 native/Package.swift create mode 100644 native/Sources/MotionLab/Analysis.swift create mode 100644 native/Sources/MotionLab/Camera.swift create mode 100644 native/Sources/MotionLab/Output.swift create mode 100644 native/Sources/MotionLab/Panel.swift create mode 100644 native/Sources/MotionLab/Renderer.swift create mode 100644 native/Sources/MotionLab/Shaders.swift create mode 100644 native/Sources/MotionLab/main.swift create mode 100644 native/effects/colorize.metal create mode 100644 native/effects/halftone.metal create mode 100644 native/effects/kaleidoscope.metal create mode 100644 native/effects/liquid.metal create mode 100644 native/effects/motion-glow.metal create mode 100644 native/effects/neon-edges.metal create mode 100644 native/effects/particles.metal create mode 100644 native/effects/trails.metal create mode 100644 native/effects/wave-shift.metal create mode 100644 panel.py create mode 100644 shaders.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d15b778 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.build/ +captures/ +__pycache__/ +.DS_Store +headless.png diff --git a/effects.py b/effects.py new file mode 100644 index 0000000..579435f --- /dev/null +++ b/effects.py @@ -0,0 +1,227 @@ +"""Effect layers. + +Each layer takes the current BGR frame plus a shared Context (motion analysis +of the raw camera frame) and returns a new BGR frame. Layers run top to bottom +in the order returned by default_layers(), like a chain of TOPs. +""" +import cv2 +import numpy as np + +rng = np.random.default_rng() + + +class Param: + def __init__(self, value, lo=0.0, hi=1.0, integer=False): + self.value, self.lo, self.hi, self.integer = value, lo, hi, integer + + @property + def norm(self): + return (self.value - self.lo) / (self.hi - self.lo) + + @norm.setter + def norm(self, n): + v = self.lo + min(max(n, 0.0), 1.0) * (self.hi - self.lo) + self.value = round(v) if self.integer else v + + +class Layer: + name = "layer" + enabled = True + + def __init__(self, **params): + self.params = params + + def __getitem__(self, key): + return self.params[key].value + + def apply(self, frame, ctx): + return frame + + +class Context: + """Per-frame motion analysis, computed at 1/4 resolution and shared by all layers.""" + + SCALE = 4 + + def __init__(self, size): + self.small = (size[0] // self.SCALE, size[1] // self.SCALE) + self.prev = None + self.t = 0.0 + self.dt = 1 / 30 + + def update(self, frame, dt): + self.dt = dt + self.t += dt + gray = cv2.cvtColor(cv2.resize(frame, self.small, interpolation=cv2.INTER_AREA), cv2.COLOR_BGR2GRAY) + if self.prev is None: + self.prev = gray + self.motion = cv2.absdiff(gray, self.prev) # where things changed + self.energy = float(self.motion.mean()) / 255 # how much changed overall + self._pair = (self.prev, gray) + self._flow = None + self.prev = gray + + @property + def flow(self): + """Dense optical flow (small-res pixels/frame). Lazy: only paid for if a layer asks.""" + if self._flow is None: + self._flow = cv2.calcOpticalFlowFarneback(*self._pair, None, 0.5, 2, 15, 2, 5, 1.1, 0) + return self._flow + + +def hue_bgr(hue): + """Hue in 0..1 (wraps) -> saturated BGR colour with components in 0..1.""" + hsv = np.uint8([[[int(hue % 1.0 * 180), 255, 255]]]) + return cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)[0, 0] / 255.0 + + +def tint(gray, color): + return cv2.merge([cv2.convertScaleAbs(gray, alpha=float(c)) for c in color]) + + +class Colorize(Layer): + name = "Colorize" + enabled = False + PALETTES = [cv2.COLORMAP_INFERNO, cv2.COLORMAP_OCEAN, cv2.COLORMAP_TWILIGHT_SHIFTED, cv2.COLORMAP_TURBO] + + def __init__(self): + super().__init__(mix=Param(0.8), palette=Param(0, 0, len(self.PALETTES) - 1, integer=True)) + + def apply(self, frame, ctx): + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + mapped = cv2.applyColorMap(gray, self.PALETTES[self["palette"]]) + return cv2.addWeighted(frame, 1 - self["mix"], mapped, self["mix"], 0) + + +class Edges(Layer): + name = "Neon Edges" + + def __init__(self): + super().__init__(detail=Param(0.6), glow=Param(0.35), source=Param(0.35), + hue=Param(0.55), cycle=Param(0.3)) + + def apply(self, frame, ctx): + gray = cv2.GaussianBlur(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), (5, 5), 0) + lo = 20 + 180 * (1 - self["detail"]) + edges = cv2.Canny(gray, lo, lo * 2) + sigma = self["glow"] * 12 + if sigma > 0.5: + edges = cv2.addWeighted(edges, 1, cv2.GaussianBlur(edges, (0, 0), sigma), 2.5, 0) + color = hue_bgr(self["hue"] + ctx.t * self["cycle"] * 0.2) + return cv2.add(cv2.convertScaleAbs(frame, alpha=self["source"]), tint(edges, color)) + + +class Particles(Layer): + """Particles spawn where there is motion and get carried along by the optical flow.""" + + name = "Flow Particles" + MAX = 6000 + + def __init__(self): + super().__init__(count=Param(3000, 200, self.MAX, integer=True), speed=Param(1.5, 0, 4), + trail=Param(0.88, 0, 0.98), hue=Param(0.08)) + self.buf = None + + def apply(self, frame, ctx): + h, w = frame.shape[:2] + if self.buf is None: + self.buf = np.zeros((h, w), np.uint8) + self.pos = (rng.random((self.MAX, 2)) * (w, h)).astype(np.float32) + self.life = rng.integers(20, 90, self.MAX) + + n = self["count"] + pos, life = self.pos[:n], self.life[:n] + flow = ctx.flow + fh, fw = flow.shape[:2] + scale = w / fw + ix = np.clip((pos[:, 0] / scale).astype(int), 0, fw - 1) + iy = np.clip((pos[:, 1] / scale).astype(int), 0, fh - 1) + new = pos + flow[iy, ix] * scale * self["speed"] + rng.normal(0, 0.5, (n, 2)).astype(np.float32) + + life -= 1 + dead = (life <= 0) | (new[:, 0] < 0) | (new[:, 0] >= w) | (new[:, 1] < 0) | (new[:, 1] >= h) + + self.buf = cv2.convertScaleAbs(self.buf, alpha=self["trail"]) + segments = np.stack([pos, new], axis=1).astype(np.int32)[~dead] + cv2.polylines(self.buf, list(segments), False, 255, 1, cv2.LINE_AA) + + k = int(dead.sum()) + if k: + hot = np.flatnonzero(ctx.motion > 24) + if hot.size > 50: + ys, xs = np.divmod(rng.choice(hot, k), fw) + new[dead] = (np.stack([xs, ys], axis=1) + rng.random((k, 2))) * scale + else: + new[dead] = rng.random((k, 2)) * (w, h) + life[dead] = rng.integers(20, 90, k) + self.pos[:n] = new + + glow = cv2.add(self.buf, cv2.GaussianBlur(self.buf, (0, 0), 3)) + return cv2.add(frame, tint(glow, hue_bgr(self["hue"]))) + + +class Feedback(Layer): + """Classic feedback loop: the previous output is transformed, faded and blended back in.""" + + name = "Feedback" + + def __init__(self): + super().__init__(decay=Param(0.92, 0, 0.995), zoom=Param(0.012, -0.05, 0.05), + rotate=Param(0.3, -3, 3), **{"flow push": Param(0.0, 0, 4)}) + self.prev = None + + def apply(self, frame, ctx): + h, w = frame.shape[:2] + if self.prev is None: + self.prev = frame + self.grid = np.stack(np.meshgrid(np.arange(w, dtype=np.float32), + np.arange(h, dtype=np.float32)), axis=-1) + m = cv2.getRotationMatrix2D((w / 2, h / 2), self["rotate"], 1 + self["zoom"]) + fb = cv2.warpAffine(self.prev, m, (w, h), borderMode=cv2.BORDER_REFLECT) + if self["flow push"] > 0: + # smear the feedback along the direction things are moving + flow = cv2.resize(ctx.flow, (w, h)) * (self["flow push"] * Context.SCALE) + fb = cv2.remap(fb, self.grid - flow, None, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT) + out = cv2.max(frame, cv2.convertScaleAbs(fb, alpha=self["decay"])) + self.prev = out + return out + + +class RGBShift(Layer): + name = "RGB Shift" + + def __init__(self): + super().__init__(amount=Param(4, 0, 40), **{"motion react": Param(0.5)}) + + def apply(self, frame, ctx): + d = int(self["amount"] + self["motion react"] * ctx.energy * 400) + if d == 0: + return frame + b, g, r = cv2.split(frame) + return cv2.merge([np.roll(b, d, axis=1), g, np.roll(r, -d, axis=1)]) + + +class Kaleido(Layer): + name = "Kaleidoscope" + enabled = False + + def __init__(self): + super().__init__(segments=Param(6, 2, 16, integer=True), spin=Param(0.2, -2, 2)) + self.radius = None + self.angle = 0.0 + + def apply(self, frame, ctx): + h, w = frame.shape[:2] + if self.radius is None: + ys, xs = np.indices((h, w), dtype=np.float32) + self.radius, self.theta = cv2.cartToPolar(xs - w / 2, ys - h / 2) + self.angle += self["spin"] * ctx.dt + wedge = 2 * np.pi / self["segments"] + # fold every angle into one mirrored wedge, then sample the source from that wedge + a = np.abs(np.mod(self.theta + np.float32(self.angle), np.float32(wedge)) - np.float32(wedge / 2)) + x, y = cv2.polarToCart(self.radius, a - np.float32(np.pi / 2)) + return cv2.remap(frame, x + w / 2, y + h / 2, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT) + + +def default_layers(): + return [Colorize(), Edges(), Particles(), Feedback(), RGBShift(), Kaleido()] diff --git a/effects/halftone.glsl b/effects/halftone.glsl new file mode 100644 index 0000000..404aa48 --- /dev/null +++ b/effects/halftone.glsl @@ -0,0 +1,17 @@ +// Halftone dots sized by brightness; dots turn hot orange wherever something moves. +uniform float cell; // 4.0 .. 40.0 = 10.0 +uniform float heat; // 0.0 .. 6.0 = 3.0 + +void main() { + vec2 grid = resolution / cell; + vec2 centre = (floor(uv * grid) + 0.5) / grid; + vec3 c = texture(camera, centre).rgb; + float luma = dot(c, vec3(0.299, 0.587, 0.114)); + + float radius = 0.5 * sqrt(luma); + float dist = length(fract(uv * grid) - 0.5); + float mask = 1.0 - smoothstep(radius - 0.05, radius + 0.05, dist); + + float moving = clamp(texture(motion, centre).r * heat, 0.0, 1.0); + color = vec4(mix(c, vec3(1.0, 0.35, 0.1), moving) * mask, 1.0); +} diff --git a/effects/liquid.glsl b/effects/liquid.glsl new file mode 100644 index 0000000..9e75474 --- /dev/null +++ b/effects/liquid.glsl @@ -0,0 +1,13 @@ +// Liquid trails: the previous output is dragged along the motion, swirled a little, +// and slowly fades back to the live camera. +uniform float persistence; // 0.0 .. 0.99 = 0.92 +uniform float drag; // 0.0 .. 6.0 = 2.5 +uniform float swirl; // 0.0 .. 0.01 = 0.002 + +void main() { + vec2 centred = uv - 0.5; + vec2 from = uv - texture(flow, uv).xy * drag + vec2(-centred.y, centred.x) * swirl; + vec3 trail = texture(prev, from).rgb; + vec3 live = texture(camera, uv).rgb; + color = vec4(mix(live, trail, persistence), 1.0); +} diff --git a/effects/wave-shift.glsl b/effects/wave-shift.glsl new file mode 100644 index 0000000..993b467 --- /dev/null +++ b/effects/wave-shift.glsl @@ -0,0 +1,12 @@ +// RGB split that ripples down the image and gets pushed around by your movement. +uniform float amount; // 0.0 .. 0.05 = 0.012 +uniform float waves; // 1.0 .. 80.0 = 24.0 +uniform float react; // 0.0 .. 3.0 = 1.0 + +void main() { + vec2 d = vec2(amount * sin(uv.y * waves + time * 3.0), 0.0); + d += texture(flow, uv).xy * react; + color = vec4(texture(camera, uv + d).r, + texture(camera, uv).g, + texture(camera, uv - d).b, 1.0); +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..0433a0f --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1791097562, + "narHash": "sha256-K9O9lG3UZNFCnwrz0ViWgP3Kgqukg1DvvzsrkFJfcvU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a322c65bf7233456788a9ebf03bc067db7dc090a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..5b51489 --- /dev/null +++ b/flake.nix @@ -0,0 +1,53 @@ +{ + description = "motionlab - realtime camera + motion graphics playground"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + outputs = { self, nixpkgs }: + let + systems = [ "aarch64-darwin" "x86_64-darwin" "x86_64-linux" "aarch64-linux" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + + # not in nixpkgs; sends frames to the OBS virtual camera (macOS) or v4l2loopback (Linux) + pyvirtualcamFor = ps: ps.buildPythonPackage rec { + pname = "pyvirtualcam"; + version = "0.15.0"; + pyproject = true; + src = nixpkgs.legacyPackages.${ps.python.stdenv.hostPlatform.system}.fetchFromGitHub { + owner = "letmaik"; + repo = "pyvirtualcam"; + tag = "v${version}"; + fetchSubmodules = true; + hash = "sha256-JtFchhuvVYYhrYOOcsHPC79/Iezyo5QppV4tY9nrjgQ="; + }; + build-system = [ ps.setuptools ps.pybind11 ]; + dependencies = [ ps.numpy ]; + doCheck = false; + pythonImportsCheck = [ "pyvirtualcam" ]; + }; + + pythonFor = pkgs: pkgs.python3.withPackages (ps: [ + ps.numpy + ps.moderngl + (pyvirtualcamFor ps) + # macOS gets a Cocoa window out of the box; Linux needs GTK for cv2.imshow + (if pkgs.stdenv.hostPlatform.isLinux + then ps.opencv4.override { enableGtk3 = true; } + else ps.opencv4) + ]); + in + { + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { packages = [ (pythonFor pkgs) ]; }; + }); + + apps = forAllSystems (pkgs: { + default = { + type = "app"; + program = toString (pkgs.writeShellScript "motionlab" '' + exec ${pythonFor pkgs}/bin/python ${self}/main.py "$@" + ''); + }; + }); + }; +} diff --git a/main.py b/main.py new file mode 100644 index 0000000..6f8c402 --- /dev/null +++ b/main.py @@ -0,0 +1,191 @@ +"""motionlab - a tiny TouchDesigner-style playground: live camera in, motion-reactive graphics out.""" +import argparse +import os +import sys +import time + +import cv2 +import numpy as np + +from effects import Context, default_layers, hue_bgr +from panel import Panel +from shaders import ShaderBank + +SIZE = (960, 540) +VCAM_SIZE = (1280, 720) +WINDOW = "motionlab" + + +class Synthetic: + """Drifting blobs, so the app (and --headless) works without a camera.""" + + def __init__(self): + self.i = 0 + + def read(self): + w, h = SIZE + t = self.i / 30 + self.i += 1 + img = np.full((h, w, 3), (30, 20, 20), np.uint8) + for k in range(5): + x = int(w * (0.5 + 0.4 * np.sin(t * (0.7 + 0.2 * k) + k))) + y = int(h * (0.5 + 0.4 * np.cos(t * (0.5 + 0.3 * k) + 2 * k))) + cv2.circle(img, (x, y), 40 + 12 * k, hue_bgr(k / 5) * 255, -1) + return True, cv2.GaussianBlur(img, (0, 0), 4) + + def release(self): + pass + + +def open_source(args): + if args.demo or args.headless: + return Synthetic() + backend = cv2.CAP_AVFOUNDATION if sys.platform == "darwin" else cv2.CAP_ANY + cap = cv2.VideoCapture(args.camera, backend) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720) + # without camera permission macOS can hand back all-black frames instead of failing + for _ in range(10): + ok, frame = cap.read() + if ok and frame.any(): + return cap + cap.release() + print("Could not read from the camera - falling back to the synthetic demo source.\n" + "On macOS: allow your terminal under System Settings > Privacy & Security > Camera, then rerun.", + file=sys.stderr) + return Synthetic() + + +def open_virtualcam(): + """Virtual camera other apps (Meet, Zoom, ...) can pick as their webcam.""" + import pyvirtualcam + try: + cam = pyvirtualcam.Camera(*VCAM_SIZE, 30, fmt=pyvirtualcam.PixelFormat.BGR) + except RuntimeError as e: + sys.exit(f"Could not start the virtual camera: {e}") + print(f"virtual camera on: pick '{cam.device}' as the camera in your call app") + return cam + + +def fit(frame): + """Centre-crop to the output aspect ratio, then resize.""" + w, h = SIZE + fh, fw = frame.shape[:2] + cw = min(fw, fh * w // h) + ch = min(fh, fw * h // w) + x, y = (fw - cw) // 2, (fh - ch) // 2 + return cv2.resize(frame[y:y + ch, x:x + cw], SIZE, interpolation=cv2.INTER_AREA) + + +def capture_path(ext): + os.makedirs("captures", exist_ok=True) + return os.path.join("captures", time.strftime(f"motionlab-%Y%m%d-%H%M%S.{ext}")) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--camera", type=int, default=0, help="camera index (default 0)") + ap.add_argument("--demo", action="store_true", help="use the synthetic source instead of the camera") + ap.add_argument("--virtualcam", action="store_true", help="also send the output to a virtual camera for video calls") + ap.add_argument("--headless", type=int, metavar="N", help="render N synthetic frames without a window, then exit") + ap.add_argument("--out", default="headless.png", help="where --headless writes its last frame") + ap.add_argument("--effects", default="effects", help="folder of .glsl shader effects (default ./effects)") + ap.add_argument("--only", metavar="NAMES", help="enable only these layers, comma separated (e.g. --only liquid,halftone)") + args = ap.parse_args() + + w, h = SIZE + src = open_source(args) + layers = default_layers() + # prefer ./effects so edits are picked up live; fall back to the copy shipped next to this file + effects_dir = args.effects if os.path.isdir(args.effects) else os.path.join(os.path.dirname(__file__), "effects") + bank = ShaderBank(effects_dir, layers) + if args.only: + wanted = {name.strip().lower() for name in args.only.split(",")} + unknown = wanted - {layer.name.lower() for layer in layers} + if unknown: + sys.exit(f"unknown layer(s): {', '.join(sorted(unknown))}. Available: {', '.join(l.name for l in layers)}") + for layer in layers: + layer.enabled = layer.name.lower() in wanted + if args.headless and bank.errors: + sys.exit(f"shader compile failed: {', '.join(bank.errors)}") + ctx = Context(SIZE) + panel = Panel(layers) + canvas = np.empty((h, w + Panel.WIDTH, 3), np.uint8) + show_panel, writer, fps = True, None, 30.0 + vcam = open_virtualcam() if args.virtualcam else None + + if not args.headless: + cv2.namedWindow(WINDOW, cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) + cv2.resizeWindow(WINDOW, w + Panel.WIDTH, h) + cv2.setMouseCallback(WINDOW, panel.on_mouse) + + start = last = time.perf_counter() + frames = 0 + while True: + ok, frame = src.read() + if not ok: + break + now = time.perf_counter() + dt = 1 / 30 if args.headless else max(now - last, 1e-3) + last = now + fps += (1 / dt - fps) * 0.1 + + frame = cv2.flip(fit(frame), 1) # mirror, so it behaves like a mirror + if frames % 15 == 0: + bank.refresh() # pick up new and edited shader files + ctx.update(frame, dt) + out = frame + for layer in layers: + if layer.enabled: + out = layer.apply(out, ctx) + + if writer: + writer.write(out) + if vcam: + # un-mirror: the preview is a mirror, but the people on the call should see the real thing + vcam.send(cv2.resize(cv2.flip(out, 1), VCAM_SIZE)) + view = out + if show_panel: + canvas[:, :w] = out + panel.draw(canvas, w, f"{fps:3.0f} fps" + (" REC" if writer else "") + (" CAM" if vcam else "")) + view = canvas + else: + panel.hits = [] + + frames += 1 + if args.headless: + if frames >= args.headless: + cv2.imwrite(args.out, view) + print(f"{frames} frames, {(time.perf_counter() - start) / frames * 1000:.1f} ms/frame -> {args.out}") + break + continue + + cv2.imshow(WINDOW, view) + key = cv2.waitKey(1) & 0xFF + if key in (ord("q"), 27) or cv2.getWindowProperty(WINDOW, cv2.WND_PROP_VISIBLE) < 1: + break + elif key == ord("h"): + show_panel = not show_panel + elif key == ord("s"): + path = capture_path("png") + cv2.imwrite(path, out) + print("saved", path) + elif key == ord("r"): + if writer: + writer.release() + writer = None + else: + path = capture_path("mp4") + writer = cv2.VideoWriter(path, cv2.VideoWriter_fourcc(*"mp4v"), round(fps), SIZE) + print("recording", path) + + if writer: + writer.release() + if vcam: + vcam.close() + src.release() + cv2.destroyAllWindows() + + +if __name__ == "__main__": + main() diff --git a/native/Package.swift b/native/Package.swift new file mode 100644 index 0000000..8074b37 --- /dev/null +++ b/native/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version:5.9 +import PackageDescription + +let package = Package( + name: "MotionLab", + platforms: [.macOS(.v13)], + targets: [ + .executableTarget(name: "MotionLab", path: "Sources/MotionLab") + ] +) diff --git a/native/Sources/MotionLab/Analysis.swift b/native/Sources/MotionLab/Analysis.swift new file mode 100644 index 0000000..db4895e --- /dev/null +++ b/native/Sources/MotionLab/Analysis.swift @@ -0,0 +1,180 @@ +import Metal + +/// Host-side motion analysis, shared by all effects. Every camera frame it produces: +/// +/// flow which way things moved (small 2-channel texture, uv units per frame) +/// particles a picture of particles that spawn where something moves and drift along the flow +/// +/// Effects read these as `in.flow` and `in.particles`. +final class Analysis { + private static let flowWidth = 160, flowHeight = 90 + private static let stateWidth = 128, stateHeight = 64 // one texel per particle + private static let particleSpeed: Float = 1.5 + private static let trailFade: Float = 0.88 + + // each pair is [this frame, previous frame], swapped at the start of every update + private var luma: [MTLTexture] + private var flowTextures: [MTLTexture] + private var state: [MTLTexture] + private var trail: [MTLTexture] + + private let lumaPipeline: MTLRenderPipelineState + private let flowPipeline: MTLRenderPipelineState + private let movePipeline: MTLRenderPipelineState + private let fadePipeline: MTLRenderPipelineState + private let dotPipeline: MTLRenderPipelineState + + var flow: MTLTexture { flowTextures[0] } + var particles: MTLTexture { trail[0] } + + init(device: MTLDevice, queue: MTLCommandQueue, width: Int, height: Int) throws { + func pair(_ w: Int, _ h: Int, _ format: MTLPixelFormat) -> [MTLTexture] { + [makeTarget(device, width: w, height: h, format: format), makeTarget(device, width: w, height: h, format: format)] + } + luma = pair(Analysis.flowWidth, Analysis.flowHeight, .r16Float) + flowTextures = pair(Analysis.flowWidth, Analysis.flowHeight, .rg16Float) + state = pair(Analysis.stateWidth, Analysis.stateHeight, .rgba32Float) + trail = pair(width, height, .r8Unorm) + + let source = shaderPrelude + analysisShaders + lumaPipeline = try makePipeline(device, source: source, fragment: "lumaFragment", format: .r16Float) + flowPipeline = try makePipeline(device, source: source, fragment: "flowFragment", format: .rg16Float) + movePipeline = try makePipeline(device, source: source, fragment: "moveFragment", format: .rgba32Float) + fadePipeline = try makePipeline(device, source: source, fragment: "fadeFragment", format: .r8Unorm) + dotPipeline = try makePipeline(device, source: source, vertex: "dotVertex", fragment: "dotFragment", + format: .r8Unorm, additive: true) + + // new textures hold garbage; start everything from zero + if let commands = queue.makeCommandBuffer() { + for texture in luma + flowTextures + state + trail { commands.pass(into: texture, clear: true) } + commands.commit() + } + } + + func update(_ commands: MTLCommandBuffer, camera: MTLTexture, particles wantsParticles: Bool) { + luma.swapAt(0, 1) + commands.pass(into: luma[0]) { $0.fullscreen(lumaPipeline, textures: [camera]) } + + flowTextures.swapAt(0, 1) + commands.pass(into: flowTextures[0]) { + $0.fullscreen(flowPipeline, textures: [luma[0], luma[1], flowTextures[1]]) + } + + guard wantsParticles else { return } + state.swapAt(0, 1) + commands.pass(into: state[0]) { + $0.fullscreen(movePipeline, textures: [state[1], flowTextures[0], luma[0], luma[1]], + uniforms: [Float.random(in: 0..<1000), Analysis.particleSpeed, 0, 0]) + } + trail.swapAt(0, 1) + commands.pass(into: trail[0]) { encoder in + encoder.fullscreen(fadePipeline, textures: [trail[1]], uniforms: [Analysis.trailFade, 0, 0, 0]) + encoder.setRenderPipelineState(dotPipeline) + encoder.setVertexTexture(state[0], index: 0) + encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: Analysis.stateWidth * Analysis.stateHeight) + } + } +} + +private let analysisShaders = """ +static float brightness(float3 c) { return dot(c, float3(0.299, 0.587, 0.114)); } +static float hash(float2 p) { return fract(sin(dot(p, float2(127.1, 311.7))) * 43758.5453); } + +// Camera -> small, blurred brightness image. Flow is measured on this. +fragment float4 lumaFragment(VertexOut v [[stage_in]], texture2d camera [[texture(0)]]) { + constexpr sampler s(filter::linear, address::clamp_to_edge); + float2 texel = 1.0 / float2(camera.get_width(), camera.get_height()); + float sum = 0.0; + for (int y = 0; y < 4; y++) { + for (int x = 0; x < 4; x++) { + sum += brightness(camera.sample(s, v.uv + (float2(x, y) - 1.5) * 2.0 * texel).rgb); + } + } + return float4(sum / 16.0); +} + +// Lucas-Kanade optical flow: for each pixel, the shift that best explains how its 7x7 +// neighbourhood changed between the previous and the current brightness image. +fragment float4 flowFragment(VertexOut v [[stage_in]], + texture2d now [[texture(0)]], + texture2d before [[texture(1)]], + texture2d flowBefore [[texture(2)]]) { + constexpr sampler s(filter::linear, address::clamp_to_edge); + float2 texel = 1.0 / float2(now.get_width(), now.get_height()); + float a = 1e-3, b = 0.0, c = 1e-3, p = 0.0, q = 0.0; // the 1e-3 keeps flat areas at zero flow + for (int y = -3; y <= 3; y++) { + for (int x = -3; x <= 3; x++) { + float2 uv = v.uv + float2(x, y) * texel; + float ix = (now.sample(s, uv + float2(texel.x, 0.0)).r - now.sample(s, uv - float2(texel.x, 0.0)).r) * 0.5; + float iy = (now.sample(s, uv + float2(0.0, texel.y)).r - now.sample(s, uv - float2(0.0, texel.y)).r) * 0.5; + float it = now.sample(s, uv).r - before.sample(s, uv).r; + a += ix * ix; b += ix * iy; c += iy * iy; + p += ix * it; q += iy * it; + } + } + float2 shift = -float2(c * p - b * q, a * q - b * p) / (a * c - b * b); // in small-image pixels + float len = length(shift); + if (len > 4.0) { shift *= 4.0 / len; } + float2 smoothed = mix(flowBefore.sample(s, v.uv).xy, shift * texel, 0.6); + return float4(smoothed, 0.0, 1.0); +} + +// One texel per particle: xy = position (uv), z = frames left to live, w = 1 if it was born from motion. +fragment float4 moveFragment(VertexOut v [[stage_in]], + texture2d state [[texture(0)]], + texture2d flow [[texture(1)]], + texture2d now [[texture(2)]], + texture2d before [[texture(3)]], + constant float4 &u [[buffer(0)]]) { // x = random seed, y = speed + constexpr sampler s(filter::linear, address::clamp_to_edge); + float4 particle = state.read(uint2(v.position.xy)); + float2 seed = v.position.xy * 0.37 + u.x; + float2 pos = particle.xy + flow.sample(s, particle.xy).xy * u.y; + pos += (float2(hash(seed), hash(seed + 7.3)) - 0.5) * 0.0015; + float life = particle.z - 1.0; + float born = particle.w; + + if (life <= 0.0 || pos.x < 0.0 || pos.x > 1.0 || pos.y < 0.0 || pos.y > 1.0) { + // respawn: try a few random spots and take the first one where something is moving + pos = float2(hash(seed + 1.1), hash(seed + 2.2)); + born = 0.0; + life = 2.0 + hash(seed + 5.5) * 4.0; // nothing moving: stay hidden and try again soon + for (int i = 0; i < 6; i++) { + float2 spot = float2(hash(seed + 3.3 + float(i)), hash(seed + 9.9 + float(i) * 1.7)); + if (abs(now.sample(s, spot).r - before.sample(s, spot).r) > 0.04) { + pos = spot; + born = 1.0; + life = 20.0 + hash(seed + 5.5) * 70.0; + break; + } + } + } + return float4(pos, life, born); +} + +// Particle trails: fade what was there, then add a dot per particle on top. +fragment float4 fadeFragment(VertexOut v [[stage_in]], texture2d trail [[texture(0)]], + constant float4 &u [[buffer(0)]]) { // x = fade + constexpr sampler s(filter::linear, address::clamp_to_edge); + return trail.sample(s, v.uv) * u.x; +} + +struct DotOut { + float4 position [[position]]; + float size [[point_size]]; +}; + +vertex DotOut dotVertex(uint id [[vertex_id]], texture2d state [[texture(0)]]) { + uint width = state.get_width(); + float4 particle = state.read(uint2(id % width, id / width)); + DotOut out; + out.position = float4(particle.x * 2.0 - 1.0, 1.0 - particle.y * 2.0, 0.0, 1.0); + if (particle.w < 0.5) { out.position.x = 4.0; } // hidden particles are moved off screen + out.size = 3.0; + return out; +} + +fragment float4 dotFragment(DotOut in [[stage_in]], float2 coord [[point_coord]]) { + return float4(smoothstep(0.5, 0.1, length(coord - 0.5)) * 0.9); +} +""" diff --git a/native/Sources/MotionLab/Camera.swift b/native/Sources/MotionLab/Camera.swift new file mode 100644 index 0000000..c176e74 --- /dev/null +++ b/native/Sources/MotionLab/Camera.swift @@ -0,0 +1,95 @@ +import AVFoundation +import CoreVideo +import Metal + +enum CameraError: Error, CustomStringConvertible { + case noPermission + case noCamera + + var description: String { + switch self { + case .noPermission: + return "No camera permission. Allow your terminal under System Settings > Privacy & Security > Camera, then restart it." + case .noCamera: + return "No camera found." + } + } +} + +/// Streams the built-in camera as Metal textures; `latest` is the newest frame and the one before it. +final class Camera: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate { + private let session = AVCaptureSession() + private var cache: CVMetalTextureCache? + private let lock = NSLock() + // each entry keeps its CVMetalTexture, which keeps the pixel buffer behind the Metal texture alive + private var current: (texture: MTLTexture, backing: CVMetalTexture)? + private var previous: (texture: MTLTexture, backing: CVMetalTexture)? + + private var serial = 0 // counts frames, so the renderer can tell a new frame from one it already saw + + var latest: (current: MTLTexture, last: MTLTexture, serial: Int)? { + lock.lock() + defer { lock.unlock() } + guard let current else { return nil } + return (current.texture, (previous ?? current).texture, serial) + } + + init(device: MTLDevice) throws { + super.init() + guard Camera.requestAccess() else { throw CameraError.noPermission } + // prefer the built-in camera over an iPhone that happens to be nearby + guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .unspecified) + ?? AVCaptureDevice.default(for: .video) + else { throw CameraError.noCamera } + + CVMetalTextureCacheCreate(nil, nil, device, nil, &cache) + let output = AVCaptureVideoDataOutput() + output.videoSettings = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferMetalCompatibilityKey as String: true, + ] + output.alwaysDiscardsLateVideoFrames = true + output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "motionlab.camera")) + + session.beginConfiguration() + session.addInput(try AVCaptureDeviceInput(device: camera)) + session.addOutput(output) + if session.canSetSessionPreset(.hd1280x720) { session.sessionPreset = .hd1280x720 } + print("camera: \(camera.localizedName)") + if let connection = output.connection(with: .video), connection.isVideoMirroringSupported { + connection.automaticallyAdjustsVideoMirroring = false + connection.isVideoMirrored = true // behave like a mirror + } + session.commitConfiguration() + session.startRunning() + } + + private static func requestAccess() -> Bool { + switch AVCaptureDevice.authorizationStatus(for: .video) { + case .authorized: + return true + case .notDetermined: + let done = DispatchSemaphore(value: 0) + var granted = false + AVCaptureDevice.requestAccess(for: .video) { granted = $0; done.signal() } + done.wait() + return granted + default: + return false + } + } + + func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { + guard let cache, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } + var created: CVMetalTexture? + CVMetalTextureCacheCreateTextureFromImage( + nil, cache, buffer, nil, .bgra8Unorm, + CVPixelBufferGetWidth(buffer), CVPixelBufferGetHeight(buffer), 0, &created) + guard let created, let frame = CVMetalTextureGetTexture(created) else { return } + lock.lock() + previous = current + current = (frame, created) + serial += 1 + lock.unlock() + } +} diff --git a/native/Sources/MotionLab/Output.swift b/native/Sources/MotionLab/Output.swift new file mode 100644 index 0000000..7e357ca --- /dev/null +++ b/native/Sources/MotionLab/Output.swift @@ -0,0 +1,198 @@ +import AVFoundation +import CoreMediaIO +import ImageIO +import UniformTypeIdentifiers + +func writePNG(bgra pixels: inout [UInt8], width: Int, height: Int, to url: URL) { + let bitmapInfo = CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue + guard let context = CGContext(data: &pixels, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4, + space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: bitmapInfo), + let image = context.makeImage(), + let destination = CGImageDestinationCreateWithURL(url as CFURL, UTType.png.identifier as CFString, 1, nil) + else { return } + CGImageDestinationAddImage(destination, image, nil) + CGImageDestinationFinalize(destination) +} + +/// Writes frames to an H.264 .mp4 file. +final class Recorder { + let url: URL + private let writer: AVAssetWriter + private let input: AVAssetWriterInput + private let adaptor: AVAssetWriterInputPixelBufferAdaptor + private let width: Int, height: Int + private var startTime: Double? + + init(url: URL, width: Int, height: Int) throws { + self.url = url + self.width = width + self.height = height + writer = try AVAssetWriter(outputURL: url, fileType: .mp4) + input = AVAssetWriterInput(mediaType: .video, outputSettings: [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: width, + AVVideoHeightKey: height, + ]) + input.expectsMediaDataInRealTime = true + adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: width, + kCVPixelBufferHeightKey as String: height, + ]) + writer.add(input) + guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) } + writer.startSession(atSourceTime: .zero) + } + + func append(bgra pixels: [UInt8], at time: Double) { + guard input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return } + var created: CVPixelBuffer? + CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created) + guard let buffer = created else { return } + + CVPixelBufferLockBaseAddress(buffer, []) + if let base = CVPixelBufferGetBaseAddress(buffer) { + let rowBytes = CVPixelBufferGetBytesPerRow(buffer) + pixels.withUnsafeBytes { source in + for y in 0.. CMIOObjectPropertyAddress { + CMIOObjectPropertyAddress(mSelector: CMIOObjectPropertySelector(selector), + mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal), + mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain)) + } + func objects(of object: CMIOObjectID, _ selector: Int) -> [CMIOObjectID] { + var property = address(selector) + var size: UInt32 = 0, used: UInt32 = 0 + CMIOObjectGetPropertyDataSize(object, &property, 0, nil, &size) + var ids = [CMIOObjectID](repeating: 0, count: Int(size) / MemoryLayout.size) + CMIOObjectGetPropertyData(object, &property, 0, nil, size, &used, &ids) + return ids + } + func uid(of device: CMIOObjectID) -> String? { + var property = address(kCMIODevicePropertyDeviceUID) + var value: Unmanaged? + var used: UInt32 = 0 + CMIOObjectGetPropertyData(device, &property, 0, nil, UInt32(MemoryLayout?>.size), &used, &value) + return value?.takeRetainedValue() as String? + } + + let devices = objects(of: CMIOObjectID(kCMIOObjectSystemObject), kCMIOHardwarePropertyDevices) + guard let device = devices.first(where: { uid(of: $0) == VirtualCamera.obsDeviceUID }) else { + throw VirtualCameraError(description: """ + OBS Virtual Camera not found. One-time setup: enable OBS under System Settings > General > \ + Login Items & Extensions > Camera Extensions, then in OBS click Start Virtual Camera, Stop Virtual Camera, and quit OBS. + """) + } + // the camera has two streams: [0] is what apps watch, [1] is where frames are fed in + let streams = objects(of: device, kCMIODevicePropertyStreams) + guard streams.count >= 2 else { throw VirtualCameraError(description: "OBS Virtual Camera has no input stream.") } + deviceID = device + streamID = streams[1] + + var createdQueue: Unmanaged? + CMIOStreamCopyBufferQueue(streamID, { _, _, _ in }, nil, &createdQueue) + var createdFormat: CMFormatDescription? + CMVideoFormatDescriptionCreate(allocator: nil, codecType: kCVPixelFormatType_422YpCbCr8, + width: Int32(width), height: Int32(height), extensions: nil, + formatDescriptionOut: &createdFormat) + var createdPool: CVPixelBufferPool? + CVPixelBufferPoolCreate(nil, nil, [ + kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_422YpCbCr8, + kCVPixelBufferWidthKey: width, + kCVPixelBufferHeightKey: height, + kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary, + ] as CFDictionary, &createdPool) + guard let createdQueue, let createdFormat, let createdPool else { + throw VirtualCameraError(description: "Could not set up the OBS Virtual Camera stream.") + } + queue = createdQueue.takeRetainedValue() + format = createdFormat + pool = createdPool + guard CMIODeviceStartStream(deviceID, streamID) == noErr else { + throw VirtualCameraError(description: "Could not start the OBS Virtual Camera stream (is OBS using it?).") + } + } + + deinit { + CMIODeviceStopStream(deviceID, streamID) + } + + /// Sends one frame. The picture is flipped back left-to-right: the preview is a mirror, + /// but the people on the call should see the real thing. + func send(bgra pixels: [UInt8], width: Int, height: Int) { + var created: CVPixelBuffer? + CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created) + guard let buffer = created else { return } + + CVPixelBufferLockBaseAddress(buffer, []) + if let base = CVPixelBufferGetBaseAddress(buffer)?.assumingMemoryBound(to: UInt8.self) { + let rowBytes = CVPixelBufferGetBytesPerRow(buffer) + pixels.withUnsafeBufferPointer { source in + // BGRA -> UYVY (BT.601 video range): two pixels share one U and one V + for y in 0..> 8) + 128) + row[pair * 4 + 1] = UInt8(clamping: ((66 * r0 + 129 * g0 + 25 * b0 + 128) >> 8) + 16) + row[pair * 4 + 2] = UInt8(clamping: ((112 * r0 - 94 * g0 - 18 * b0 + 128) >> 8) + 128) + row[pair * 4 + 3] = UInt8(clamping: ((66 * r1 + 129 * g1 + 25 * b1 + 128) >> 8) + 16) + } + } + } + } + CVPixelBufferUnlockBaseAddress(buffer, []) + + var timing = CMSampleTimingInfo( + duration: .invalid, + presentationTimeStamp: CMTime(value: CMTimeValue(clock_gettime_nsec_np(CLOCK_UPTIME_RAW)), timescale: 1_000_000_000), + decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + CMSampleBufferCreateForImageBuffer(allocator: nil, imageBuffer: buffer, dataReady: true, makeDataReadyCallback: nil, + refcon: nil, formatDescription: format, sampleTiming: &timing, sampleBufferOut: &sample) + if let sample { + // the queue takes ownership of the sample + if CMSimpleQueueEnqueue(queue, element: Unmanaged.passRetained(sample).toOpaque()) != noErr { + Unmanaged.passUnretained(sample).release() + } + } + } +} diff --git a/native/Sources/MotionLab/Panel.swift b/native/Sources/MotionLab/Panel.swift new file mode 100644 index 0000000..4b0ec86 --- /dev/null +++ b/native/Sources/MotionLab/Panel.swift @@ -0,0 +1,92 @@ +import AppKit + +private final class TopAlignedView: NSView { + override var isFlipped: Bool { true } // so the scroll view starts at the top +} + +/// Side panel: every effect with a checkbox, and the sliders of the ones that are switched on. +final class Panel: NSObject { + static let width: CGFloat = 230 + + let view = NSScrollView() + private let stack = NSStackView() + private let statusLabel = NSTextField(labelWithString: "") + private let renderer: Renderer + + init(renderer: Renderer) { + self.renderer = renderer + super.init() + stack.orientation = .vertical + stack.alignment = .leading + stack.spacing = 4 + stack.translatesAutoresizingMaskIntoConstraints = false + + let document = TopAlignedView() + document.translatesAutoresizingMaskIntoConstraints = false + document.addSubview(stack) + view.documentView = document + view.hasVerticalScroller = true + view.drawsBackground = false + NSLayoutConstraint.activate([ + view.widthAnchor.constraint(equalToConstant: Panel.width), + document.widthAnchor.constraint(equalTo: view.contentView.widthAnchor), + stack.topAnchor.constraint(equalTo: document.topAnchor, constant: 14), + stack.leadingAnchor.constraint(equalTo: document.leadingAnchor, constant: 14), + stack.trailingAnchor.constraint(equalTo: document.trailingAnchor, constant: -14), + stack.bottomAnchor.constraint(equalTo: document.bottomAnchor, constant: -14), + ]) + statusLabel.font = .boldSystemFont(ofSize: 11) + statusLabel.textColor = .systemRed + rebuild() + } + + private func label(_ text: String, dim: Bool = true) -> NSTextField { + let field = NSTextField(labelWithString: text) + field.font = .systemFont(ofSize: 11) + field.textColor = dim ? .secondaryLabelColor : .labelColor + return field + } + + func rebuild() { + stack.arrangedSubviews.forEach { $0.removeFromSuperview() } + statusLabel.stringValue = renderer.status + stack.addArrangedSubview(statusLabel) + + for (layerIndex, layer) in renderer.layers.enumerated() { + let title = layer.error == nil ? layer.name : "\(layer.name) (shader error)" + let checkbox = NSButton(checkboxWithTitle: title, target: self, action: #selector(toggled(_:))) + checkbox.state = layer.enabled ? .on : .off + checkbox.tag = layerIndex + checkbox.font = .systemFont(ofSize: 13, weight: .medium) + stack.addArrangedSubview(checkbox) + guard layer.enabled else { continue } + + for (paramIndex, param) in layer.params.enumerated() { + let slider = NSSlider(value: Double(param.value), minValue: Double(param.min), maxValue: Double(param.max), + target: self, action: #selector(sliderMoved(_:))) + slider.tag = layerIndex * maxParams + paramIndex + slider.controlSize = .small + stack.addArrangedSubview(label(param.name)) + stack.addArrangedSubview(slider) + slider.widthAnchor.constraint(equalTo: stack.widthAnchor).isActive = true + } + stack.setCustomSpacing(12, after: stack.arrangedSubviews.last!) + } + if renderer.layers.isEmpty { stack.addArrangedSubview(label("no .metal files found")) } + + stack.setCustomSpacing(16, after: stack.arrangedSubviews.last!) + stack.addArrangedSubview(label("s screenshot\nr record\nq quit")) + } + + func refreshStatus() { + statusLabel.stringValue = renderer.status + } + + @objc private func toggled(_ checkbox: NSButton) { + renderer.setEnabled(checkbox.tag, checkbox.state == .on) + } + + @objc private func sliderMoved(_ slider: NSSlider) { + renderer.setParam(layer: slider.tag / maxParams, param: slider.tag % maxParams, value: Float(slider.doubleValue)) + } +} diff --git a/native/Sources/MotionLab/Renderer.swift b/native/Sources/MotionLab/Renderer.swift new file mode 100644 index 0000000..be0818b --- /dev/null +++ b/native/Sources/MotionLab/Renderer.swift @@ -0,0 +1,287 @@ +import MetalKit + +/// One effect file in the chain. +final class Layer { + let url: URL + let name: String + var enabled = false + var order = 50 + var params: [EffectParam] = [] + var error: String? + fileprivate var pipeline: MTLRenderPipelineState? + fileprivate var modified: Date? + fileprivate var usesParticles = false + fileprivate var targets: [MTLTexture] = [] // [this frame, previous frame], swapped after every draw + + init(url: URL) { + self.url = url + self.name = url.deletingPathExtension().lastPathComponent + } +} + +private let passthrough = "float4 effect(float2 uv, Inputs in) { return in.image.sample(in.smp, uv); }" + +/// Runs every enabled effect in order on each new camera frame, shows the result in the window, +/// and hands it to whatever outputs are active (recording, virtual camera, screenshots). +final class Renderer: NSObject, MTKViewDelegate { + private let device: MTLDevice + private let queue: MTLCommandQueue + private let camera: Camera + private let effectsDirectory: URL + private let copyPipeline: MTLRenderPipelineState + private var analysis: Analysis? + private var plain: MTLTexture? // holds the camera picture when no effect is enabled + private var result: MTLTexture? // the finished frame + private var lastSerial = -1 + private let startTime = CACurrentMediaTime() + private var frameCount = 0 + private var pixels: [UInt8] = [] + + private(set) var layers: [Layer] = [] + /// Called when layers were added, removed, reloaded or toggled. + var onLayersChanged: (() -> Void)? + /// Called when recording or the virtual camera starts or stops. + var onStatusChanged: (() -> Void)? + + /// When set, the app exits after this many camera frames, first saving that frame as a PNG if a url is given + /// (used to check effects without watching). + var snapshot: (frames: Int, url: URL?)? + /// When set, recording into this file starts with the first frame. + var recordURL: URL? + /// When true, the OBS virtual camera is opened on the first frame, at the size of the output. + var wantsVirtualCamera = false + private var virtualCamera: VirtualCamera? { didSet { onStatusChanged?() } } + private var recorder: Recorder? { didSet { onStatusChanged?() } } + private var screenshotRequested = false + + var status: String { + [recorder != nil ? "REC" : nil, virtualCamera != nil ? "CAM" : nil].compactMap { $0 }.joined(separator: " ") + } + + init(device: MTLDevice, camera: Camera, effectsDirectory: URL) throws { + self.device = device + self.camera = camera + self.effectsDirectory = effectsDirectory + self.queue = device.makeCommandQueue()! + self.copyPipeline = try makePipeline(device, source: effectShader(body: passthrough, params: []), + fragment: "effectFragment", format: .bgra8Unorm) + super.init() + refreshFiles() + } + + // MARK: effect files + + /// Picks up new, deleted and edited effect files. On a compile error the last working version keeps running. + private func refreshFiles() { + let files = ((try? FileManager.default.contentsOfDirectory(at: effectsDirectory, includingPropertiesForKeys: nil)) ?? []) + .filter { $0.pathExtension == "metal" } + var changed = false + + let before = layers.count + layers.removeAll { layer in !files.contains(layer.url) } + changed = layers.count != before + for url in files where !layers.contains(where: { $0.url == url }) { + layers.append(Layer(url: url)) + changed = true + } + for layer in layers where reload(layer) { changed = true } + + if changed { + layers.sort { ($0.order, $0.name) < ($1.order, $1.name) } + onLayersChanged?() + } + } + + private func reload(_ layer: Layer) -> Bool { + let modified = (try? layer.url.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate + guard let modified, modified != layer.modified else { return false } + let firstLoad = layer.modified == nil + layer.modified = modified + do { + let source = try String(contentsOf: layer.url, encoding: .utf8) + var info = parseEffect(source) + layer.pipeline = try makePipeline(device, source: effectShader(body: source, params: info.params), + fragment: "effectFragment", format: .bgra8Unorm) + for i in info.params.indices { // keep slider positions across reloads + if let old = layer.params.first(where: { $0.name == info.params[i].name }) { + info.params[i].value = min(max(old.value, info.params[i].min), info.params[i].max) + } + } + layer.params = info.params + layer.order = info.order + layer.usesParticles = source.contains("in.particles") + layer.error = nil + if firstLoad { layer.enabled = info.startsOn } + print("[\(layer.name)] loaded") + } catch { + layer.error = error.localizedDescription + print("[\(layer.name)] shader error:\n\(error.localizedDescription)") + } + return true + } + + var failedLayers: [String] { layers.filter { $0.error != nil }.map(\.name) } + + /// Enables exactly the named layers. Returns the names that don't exist. + func enableOnly(_ names: [String]) -> [String] { + for layer in layers { layer.enabled = names.contains(layer.name) } + return names.filter { name in !layers.contains { $0.name == name } } + } + + func setEnabled(_ index: Int, _ enabled: Bool) { + guard layers.indices.contains(index) else { return } + layers[index].enabled = enabled + if !enabled { layers[index].targets = [] } // free its memory, and start clean next time + onLayersChanged?() + } + + func setParam(layer: Int, param: Int, value: Float) { + guard layers.indices.contains(layer), layers[layer].params.indices.contains(param) else { return } + let p = layers[layer].params[param] + layers[layer].params[param].value = min(max(value, p.min), p.max) + } + + // MARK: outputs + + func requestScreenshot() { screenshotRequested = true } + + func toggleRecording() { + if let recorder { + self.recorder = nil + recorder.finish() + print("saved \(recorder.url.path)") + } else if let result { + do { + recorder = try Recorder(url: capturesURL(extension: "mp4"), width: result.width, height: result.height) + print("recording \(recorder!.url.path)") + } catch { + print("could not start recording: \(error.localizedDescription)") + } + } + } + + func finish() { + if recorder != nil { toggleRecording() } + } + + private func capturesURL(extension ext: String) -> URL { + let directory = URL(fileURLWithPath: "captures") + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let stamp = DateFormatter() + stamp.dateFormat = "yyyyMMdd-HHmmss" + return directory.appendingPathComponent("motionlab-\(stamp.string(from: Date())).\(ext)") + } + + /// Runs once per camera frame, after the GPU work was submitted. + private func deliver(_ commands: MTLCommandBuffer) { + frameCount += 1 + let snapshotDue = snapshot.map { frameCount >= $0.frames } ?? false + guard let result else { return } + let width = result.width, height = result.height + if let url = recordURL { + recordURL = nil + do { recorder = try Recorder(url: url, width: width, height: height) } catch { print("could not start recording: \(error.localizedDescription)") } + } + if wantsVirtualCamera { + wantsVirtualCamera = false + do { + virtualCamera = try VirtualCamera(width: width, height: height) + print("virtual camera on: pick 'OBS Virtual Camera' as the camera in your call or streaming app") + } catch { + print("virtual camera off: \(error)") + } + } + guard recorder != nil || virtualCamera != nil || screenshotRequested || snapshotDue else { return } + + commands.waitUntilCompleted() + if pixels.count != width * height * 4 { pixels = [UInt8](repeating: 0, count: width * height * 4) } + result.getBytes(&pixels, bytesPerRow: width * 4, from: MTLRegionMake2D(0, 0, width, height), mipmapLevel: 0) + + recorder?.append(bgra: pixels, at: CACurrentMediaTime()) + virtualCamera?.send(bgra: pixels, width: width, height: height) + if screenshotRequested { + screenshotRequested = false + let url = capturesURL(extension: "png") + writePNG(bgra: &pixels, width: width, height: height, to: url) + print("saved \(url.path)") + } + if snapshotDue { + if let url = snapshot?.url { + writePNG(bgra: &pixels, width: width, height: height, to: url) + print("\(frameCount) frames -> \(url.path)") + } + finish() + exit(0) + } + } + + // MARK: drawing + + private func process(_ frames: (current: MTLTexture, last: MTLTexture, serial: Int), _ commands: MTLCommandBuffer) throws { + let width = frames.current.width, height = frames.current.height + if analysis == nil { analysis = try Analysis(device: device, queue: queue, width: width, height: height) } + guard let analysis else { return } + analysis.update(commands, camera: frames.current, particles: layers.contains { $0.enabled && $0.usesParticles }) + + let time = Float(CACurrentMediaTime() - startTime) + var image = frames.current + for layer in layers where layer.enabled { + guard let pipeline = layer.pipeline else { continue } + if layer.targets.first?.width != width || layer.targets.first?.height != height { + layer.targets = [makeTarget(device, width: width, height: height), makeTarget(device, width: width, height: height)] + for target in layer.targets { commands.pass(into: target, clear: true) } + } + // must match `struct Uniforms` in the shader: time, padding, resolution.xy, params[16] + var uniforms: [Float] = [time, 0, Float(width), Float(height)] + uniforms += layer.params.map(\.value) + [Float](repeating: 0, count: maxParams - layer.params.count) + let textures = [image, frames.current, frames.last, layer.targets[1], analysis.flow, analysis.particles] + commands.pass(into: layer.targets[0]) { $0.fullscreen(pipeline, textures: textures, uniforms: uniforms) } + image = layer.targets[0] + layer.targets.swapAt(0, 1) + } + + if image === frames.current { + if plain?.width != width || plain?.height != height { plain = makeTarget(device, width: width, height: height) } + commands.pass(into: plain!) { $0.fullscreen(copyPipeline, textures: [image], uniforms: [0, 0, 0, 0]) } + image = plain! + } + result = image + } + + /// The largest rectangle with the output's aspect ratio that fits the window, centred. + private func fitted(_ output: MTLTexture, in size: CGSize) -> MTLViewport { + let scale = min(size.width / Double(output.width), size.height / Double(output.height)) + let width = Double(output.width) * scale, height = Double(output.height) * scale + return MTLViewport(originX: (size.width - width) / 2, originY: (size.height - height) / 2, + width: width, height: height, znear: 0, zfar: 1) + } + + func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {} + + func draw(in view: MTKView) { + guard let frames = camera.latest, let commands = queue.makeCommandBuffer() else { return } + + // effects advance once per camera frame; in between, the window just shows the last result again + let isNewFrame = frames.serial != lastSerial + if isNewFrame { + lastSerial = frames.serial + if frameCount % 15 == 0 { refreshFiles() } + do { + try process(frames, commands) + } catch { + print("render setup failed: \(error.localizedDescription)") + exit(1) + } + } + + if let result, let drawable = view.currentDrawable, let windowPass = view.currentRenderPassDescriptor, + let encoder = commands.makeRenderCommandEncoder(descriptor: windowPass) { + encoder.setViewport(fitted(result, in: view.drawableSize)) + encoder.fullscreen(copyPipeline, textures: [result], uniforms: [0, 0, 0, 0]) + encoder.endEncoding() + commands.present(drawable) + } + commands.commit() + if isNewFrame { deliver(commands) } + } +} diff --git a/native/Sources/MotionLab/Shaders.swift b/native/Sources/MotionLab/Shaders.swift new file mode 100644 index 0000000..53e1a8a --- /dev/null +++ b/native/Sources/MotionLab/Shaders.swift @@ -0,0 +1,161 @@ +import Metal + +/// A slider an effect file declares with a comment line: `// slider ` +struct EffectParam { + let name: String + let min: Float + let max: Float + var value: Float +} + +/// Everything an effect file says about itself in `//` comment lines: +/// +/// // slider a slider, read in the shader as in.p. +/// // order position in the chain, low runs first (default 50) +/// // on enabled when the app starts +struct EffectInfo { + var params: [EffectParam] = [] + var order = 50 + var startsOn = false +} + +let maxParams = 16 + +func parseEffect(_ source: String) -> EffectInfo { + var info = EffectInfo() + for line in source.split(separator: "\n") { + let parts = line.split(separator: " ") + guard parts.count >= 2, parts[0] == "//" else { continue } + switch parts[1] { + case "slider" where parts.count == 6: + if let min = Float(parts[3]), let max = Float(parts[4]), let value = Float(parts[5]) { + info.params.append(EffectParam(name: String(parts[2]), min: min, max: max, value: value)) + } + case "order" where parts.count == 3: + info.order = Int(parts[2]) ?? info.order + case "on" where parts.count == 2: + info.startsOn = true + default: + break + } + } + info.params = Array(info.params.prefix(maxParams)) + return info +} + +/// Shared by every shader in the app: a triangle that covers the whole target. +let shaderPrelude = """ +#include +using namespace metal; + +struct VertexOut { + float4 position [[position]]; + float2 uv; +}; + +vertex VertexOut fullscreenVertex(uint id [[vertex_id]]) { + float2 p = float2((id << 1) & 2, id & 2); + VertexOut out; + out.position = float4(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0, 0.0, 1.0); + out.uv = p; + return out; +} + +""" + +/// Wraps an effect file into a complete Metal shader. An effect file only defines: +/// +/// float4 effect(float2 uv, Inputs in) { ... } +/// +/// `uv` runs from (0,0) top-left to (1,1) bottom-right. +func effectShader(body: String, params: [EffectParam]) -> String { + let fields = params.isEmpty ? "float unused;" : params.map { "float \($0.name);" }.joined(separator: " ") + let values = params.isEmpty ? "0.0" : params.indices.map { "u.params[\($0)]" }.joined(separator: ", ") + return shaderPrelude + """ + struct Uniforms { + float time; + float2 resolution; + float params[\(maxParams)]; + }; + + struct Params { \(fields) }; + + struct Inputs { + texture2d image; // the picture so far: output of the effect above, or the camera for the first one + texture2d camera; // the untouched camera frame + texture2d last; // the camera frame before this one (compare with camera to find motion) + texture2d prev; // this effect's own output from the previous frame (for trails) + texture2d flow; // .xy = which way and how far things moved since the last frame, in uv units + texture2d particles; // .r = brightness of particles that drift along with the motion + sampler smp; + float time; // seconds since start + float2 resolution; // output size in pixels + Params p; // this effect's sliders + }; + + #line 1 + \(body) + + fragment float4 effectFragment(VertexOut v [[stage_in]], + texture2d image [[texture(0)]], + texture2d camera [[texture(1)]], + texture2d last [[texture(2)]], + texture2d prev [[texture(3)]], + texture2d flow [[texture(4)]], + texture2d particles [[texture(5)]], + constant Uniforms &u [[buffer(0)]]) { + constexpr sampler smp(filter::linear, address::clamp_to_edge); + Params p = { \(values) }; + Inputs in = { image, camera, last, prev, flow, particles, smp, u.time, u.resolution, p }; + return effect(v.uv, in); + } + """ +} + +func makePipeline(_ device: MTLDevice, source: String, vertex: String = "fullscreenVertex", fragment: String, + format: MTLPixelFormat, additive: Bool = false) throws -> MTLRenderPipelineState { + let library = try device.makeLibrary(source: source, options: nil) + let descriptor = MTLRenderPipelineDescriptor() + descriptor.vertexFunction = library.makeFunction(name: vertex) + descriptor.fragmentFunction = library.makeFunction(name: fragment) + let attachment = descriptor.colorAttachments[0]! + attachment.pixelFormat = format + if additive { + attachment.isBlendingEnabled = true + attachment.sourceRGBBlendFactor = .one + attachment.destinationRGBBlendFactor = .one + attachment.sourceAlphaBlendFactor = .one + attachment.destinationAlphaBlendFactor = .one + } + return try device.makeRenderPipelineState(descriptor: descriptor) +} + +func makeTarget(_ device: MTLDevice, width: Int, height: Int, format: MTLPixelFormat = .bgra8Unorm) -> MTLTexture { + let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width, height: height, mipmapped: false) + descriptor.usage = [.renderTarget, .shaderRead] + return device.makeTexture(descriptor: descriptor)! +} + +extension MTLCommandBuffer { + /// One render pass into `target`. With `clear`, the target starts black; otherwise its old contents are ignored. + func pass(into target: MTLTexture, clear: Bool = false, _ body: (MTLRenderCommandEncoder) -> Void = { _ in }) { + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = target + descriptor.colorAttachments[0].loadAction = clear ? .clear : .dontCare + descriptor.colorAttachments[0].storeAction = .store + guard let encoder = makeRenderCommandEncoder(descriptor: descriptor) else { return } + body(encoder) + encoder.endEncoding() + } +} + +extension MTLRenderCommandEncoder { + func fullscreen(_ pipeline: MTLRenderPipelineState, textures: [MTLTexture], uniforms: [Float] = []) { + setRenderPipelineState(pipeline) + setFragmentTextures(textures, range: 0...stride, index: 0) + } + drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + } +} diff --git a/native/Sources/MotionLab/main.swift b/native/Sources/MotionLab/main.swift new file mode 100644 index 0000000..eaa7c1c --- /dev/null +++ b/native/Sources/MotionLab/main.swift @@ -0,0 +1,110 @@ +// MotionLab: camera -> chain of Metal shader effects -> window, recording, virtual camera. +// +// swift run -c release MotionLab [options] +// +// --effects DIR folder of .metal effect files (default ./effects) +// --only a,b,c start with exactly these effects enabled +// --virtualcam also send the output to the OBS virtual camera (for Meet, Zoom, ...) +// --snapshot OUT.png save one frame after --frames N camera frames (default 90) and exit +// --record OUT.mp4 record --frames N camera frames (default 90) and exit +// +// In the window: s saves a screenshot, r starts/stops recording (both into ./captures), q or Esc quits. +// Effect files are reloaded when saved. + +import AppKit +import MetalKit + +func option(_ name: String) -> String? { + let arguments = CommandLine.arguments + guard let i = arguments.firstIndex(of: name), i + 1 < arguments.count else { return nil } + return arguments[i + 1] +} + +func fail(_ message: String) -> Never { + FileHandle.standardError.write((message + "\n").data(using: .utf8)!) + exit(1) +} + +final class AppDelegate: NSObject, NSApplicationDelegate { + var onQuit: (() -> Void)? + func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true } + func applicationWillTerminate(_ notification: Notification) { onQuit?() } +} + +setvbuf(stdout, nil, _IOLBF, 0) // show "loaded" / shader errors immediately, even when piped + +guard let device = MTLCreateSystemDefaultDevice() else { fail("Metal is not available on this machine.") } +let effectsDirectory = URL(fileURLWithPath: option("--effects") ?? "effects") + +let renderer: Renderer +do { + renderer = try Renderer(device: device, camera: Camera(device: device), effectsDirectory: effectsDirectory) +} catch { + fail("\(error)") +} +if renderer.layers.isEmpty { print("No .metal files in \(effectsDirectory.path) - showing the plain camera.") } +if let names = option("--only") { + let unknown = renderer.enableOnly(names.split(separator: ",").map(String.init)) + if !unknown.isEmpty { + fail("Unknown effect(s): \(unknown.joined(separator: ", ")). Available: \(renderer.layers.map(\.name).joined(separator: ", "))") + } +} +if option("--snapshot") != nil || option("--record") != nil { + if !renderer.failedLayers.isEmpty { fail("shader compile failed: \(renderer.failedLayers.joined(separator: ", "))") } + renderer.snapshot = (Int(option("--frames") ?? "90") ?? 90, option("--snapshot").map { URL(fileURLWithPath: $0) }) + renderer.recordURL = option("--record").map { URL(fileURLWithPath: $0) } +} +renderer.wantsVirtualCamera = CommandLine.arguments.contains("--virtualcam") + +let app = NSApplication.shared +let delegate = AppDelegate() +delegate.onQuit = { renderer.finish() } +app.delegate = delegate +app.setActivationPolicy(.regular) + +let size = NSSize(width: 960 + Panel.width, height: 540) +let window = NSWindow(contentRect: NSRect(origin: .zero, size: size), + styleMask: [.titled, .closable, .miniaturizable, .resizable], + backing: .buffered, defer: false) +let view = MTKView(frame: .zero, device: device) +view.colorPixelFormat = .bgra8Unorm +view.delegate = renderer + +let panel = Panel(renderer: renderer) +renderer.onLayersChanged = { panel.rebuild() } +renderer.onStatusChanged = { panel.refreshStatus() } + +NSEvent.addLocalMonitorForEvents(matching: .keyDown) { event in + if event.modifierFlags.contains(.command) { return event } + switch event.charactersIgnoringModifiers { + case "s": renderer.requestScreenshot() + case "r": renderer.toggleRecording() + case "q", "\u{1B}": app.terminate(nil) + default: return event + } + return nil +} + +let content = NSView(frame: NSRect(origin: .zero, size: size)) +for subview in [view, panel.view] { + subview.translatesAutoresizingMaskIntoConstraints = false + content.addSubview(subview) +} +NSLayoutConstraint.activate([ + view.leadingAnchor.constraint(equalTo: content.leadingAnchor), + view.topAnchor.constraint(equalTo: content.topAnchor), + view.bottomAnchor.constraint(equalTo: content.bottomAnchor), + view.trailingAnchor.constraint(equalTo: panel.view.leadingAnchor), + panel.view.trailingAnchor.constraint(equalTo: content.trailingAnchor), + panel.view.topAnchor.constraint(equalTo: content.topAnchor), + panel.view.bottomAnchor.constraint(equalTo: content.bottomAnchor), +]) + +window.title = "MotionLab" +window.contentMinSize = NSSize(width: 480 + Panel.width, height: 270) +window.contentView = content +window.setContentSize(size) // auto layout would otherwise pick its own window size +window.center() +window.makeKeyAndOrderFront(nil) +app.activate(ignoringOtherApps: true) +app.run() diff --git a/native/effects/colorize.metal b/native/effects/colorize.metal new file mode 100644 index 0000000..a913e49 --- /dev/null +++ b/native/effects/colorize.metal @@ -0,0 +1,12 @@ +// Recolours the picture by brightness with a smooth colour ramp. +// order 10 +// slider amount 0.0 1.0 0.8 +// slider palette 0.0 1.0 0.0 +// slider contrast 0.5 3.0 1.2 +float4 effect(float2 uv, Inputs in) { + float3 c = in.image.sample(in.smp, uv).rgb; + float luma = pow(dot(c, float3(0.299, 0.587, 0.114)), in.p.contrast); + float3 ramp = 0.5 + 0.5 * cos(6.2832 * (luma * 0.8 + in.p.palette + float3(0.55, 0.4, 0.2))); + float3 mapped = ramp * (0.15 + 0.85 * luma); + return float4(mix(c, mapped, in.p.amount), 1.0); +} diff --git a/native/effects/halftone.metal b/native/effects/halftone.metal new file mode 100644 index 0000000..1052374 --- /dev/null +++ b/native/effects/halftone.metal @@ -0,0 +1,18 @@ +// Halftone dots sized by brightness; dots turn hot orange wherever something moves. +// order 70 +// slider cell 4.0 40.0 12.0 +// slider heat 0.0 20.0 8.0 +float4 effect(float2 uv, Inputs in) { + float2 grid = in.resolution / in.p.cell; + float2 centre = (floor(uv * grid) + 0.5) / grid; + float3 c = in.image.sample(in.smp, centre).rgb; + float luma = dot(c, float3(0.299, 0.587, 0.114)); + + float radius = 0.5 * sqrt(luma); + float dist = length(fract(uv * grid) - 0.5); + float mask = 1.0 - smoothstep(radius - 0.05, radius + 0.05, dist); + + float moved = length(in.camera.sample(in.smp, centre).rgb - in.last.sample(in.smp, centre).rgb); + float3 hot = mix(c, float3(1.0, 0.35, 0.1), clamp(moved * in.p.heat, 0.0, 1.0)); + return float4(hot * mask, 1.0); +} diff --git a/native/effects/kaleidoscope.metal b/native/effects/kaleidoscope.metal new file mode 100644 index 0000000..92fb1c0 --- /dev/null +++ b/native/effects/kaleidoscope.metal @@ -0,0 +1,13 @@ +// Folds the picture into mirrored wedges around the centre. +// order 90 +// slider segments 2.0 16.0 6.0 +// slider spin -2.0 2.0 0.2 +float4 effect(float2 uv, Inputs in) { + float2 shape = float2(in.resolution.x / in.resolution.y, 1.0); + float2 c = (uv - 0.5) * shape; + float wedge = 6.2832 / floor(in.p.segments); + float a = atan2(c.y, c.x) + in.time * in.p.spin; + a = abs(a - wedge * floor(a / wedge) - wedge * 0.5) - 1.5708; // fold into one mirrored wedge, pointing up + float2 from = float2(cos(a), sin(a)) * length(c) / shape + 0.5; + return float4(in.image.sample(in.smp, from).rgb, 1.0); +} diff --git a/native/effects/liquid.metal b/native/effects/liquid.metal new file mode 100644 index 0000000..073f0ea --- /dev/null +++ b/native/effects/liquid.metal @@ -0,0 +1,13 @@ +// Liquid trails: the previous output is dragged along the motion, swirled a little, +// and slowly fades back to the live picture. +// order 45 +// slider persistence 0.0 0.99 0.92 +// slider drag 0.0 6.0 2.5 +// slider swirl 0.0 0.01 0.002 +float4 effect(float2 uv, Inputs in) { + float2 centred = uv - 0.5; + float2 from = uv - in.flow.sample(in.smp, uv).xy * in.p.drag + float2(-centred.y, centred.x) * in.p.swirl; + float3 trail = in.prev.sample(in.smp, from).rgb; + float3 live = in.image.sample(in.smp, uv).rgb; + return float4(mix(live, trail, in.p.persistence), 1.0); +} diff --git a/native/effects/motion-glow.metal b/native/effects/motion-glow.metal new file mode 100644 index 0000000..d333b82 --- /dev/null +++ b/native/effects/motion-glow.metal @@ -0,0 +1,20 @@ +// Whatever moves lights up and the glow lingers. The heat is carried from frame to frame +// in the alpha channel of this effect's own previous output. +// order 50 +// slider sensitivity 1.0 30.0 10.0 +// slider linger 0.5 0.99 0.93 +// slider dim 0.0 1.0 0.45 +// slider hue 0.0 1.0 0.07 +// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue) +static float3 hueColor(float h) { + return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0); +} + +float4 effect(float2 uv, Inputs in) { + float moved = length(in.camera.sample(in.smp, uv).rgb - in.last.sample(in.smp, uv).rgb); + float fresh = smoothstep(0.3, 1.0, moved * in.p.sensitivity); + float heat = max(in.prev.sample(in.smp, uv).a * in.p.linger, fresh); + + float3 glow = hueColor(in.p.hue); + return float4(in.image.sample(in.smp, uv).rgb * in.p.dim + glow * heat, heat); +} diff --git a/native/effects/neon-edges.metal b/native/effects/neon-edges.metal new file mode 100644 index 0000000..9b985ff --- /dev/null +++ b/native/effects/neon-edges.metal @@ -0,0 +1,38 @@ +// Glowing outlines with a slowly cycling colour. +// order 20 +// slider detail 1.0 12.0 5.0 +// slider glow 0.0 6.0 2.5 +// slider source 0.0 1.0 0.35 +// slider hue 0.0 1.0 0.55 +// slider cycle 0.0 1.0 0.3 +// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue) +static float3 hueColor(float h) { + return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0); +} + +static float luma(Inputs in, float2 uv) { + return dot(in.image.sample(in.smp, uv).rgb, float3(0.299, 0.587, 0.114)); +} + +// how strongly brightness changes around this point +static float edge(Inputs in, float2 uv) { + float2 t = 1.5 / in.resolution; + float dx = luma(in, uv + float2(t.x, 0.0)) - luma(in, uv - float2(t.x, 0.0)); + float dy = luma(in, uv + float2(0.0, t.y)) - luma(in, uv - float2(0.0, t.y)); + return length(float2(dx, dy)); +} + +float4 effect(float2 uv, Inputs in) { + float sharp = smoothstep(0.1, 0.5, edge(in, uv) * in.p.detail); + float halo = 0.0; // edges found in a ring around this pixel = a cheap glow + for (int i = 0; i < 12; i++) { + float a = 6.2832 * float(i) / 12.0; + float2 dir = float2(cos(a), sin(a)) / in.resolution; + halo += edge(in, uv + dir * 4.0) + edge(in, uv + dir * 9.0); + } + halo = halo / 24.0 * in.p.detail * in.p.glow; + + float3 tint = hueColor(in.p.hue + in.time * in.p.cycle * 0.2); + float3 base = in.image.sample(in.smp, uv).rgb * in.p.source; + return float4(base + tint * (sharp + halo), 1.0); +} diff --git a/native/effects/particles.metal b/native/effects/particles.metal new file mode 100644 index 0000000..7ad565f --- /dev/null +++ b/native/effects/particles.metal @@ -0,0 +1,22 @@ +// Particles spawn where you move and drift along with the motion. +// order 30 +// on +// slider brightness 0.0 3.0 1.4 +// slider glow 0.0 4.0 1.5 +// slider hue 0.0 1.0 0.08 +// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue) +static float3 hueColor(float h) { + return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0); +} + +float4 effect(float2 uv, Inputs in) { + float core = in.particles.sample(in.smp, uv).r; + float halo = 0.0; + for (int i = 0; i < 8; i++) { + float a = 6.2832 * float(i) / 8.0; + halo += in.particles.sample(in.smp, uv + float2(cos(a), sin(a)) * 5.0 / in.resolution).r; + } + float3 tint = hueColor(in.p.hue); + float3 base = in.image.sample(in.smp, uv).rgb; + return float4(base + tint * (core * in.p.brightness + halo / 8.0 * in.p.glow), 1.0); +} diff --git a/native/effects/trails.metal b/native/effects/trails.metal new file mode 100644 index 0000000..956c4a5 --- /dev/null +++ b/native/effects/trails.metal @@ -0,0 +1,16 @@ +// Feedback trails: the previous output is zoomed, turned and faded, so bright things leave streaks. +// "push" smears the trails in the direction things are moving. +// order 40 +// on +// slider fade 0.5 0.995 0.92 +// slider zoom -0.04 0.04 0.012 +// slider spin -0.05 0.05 0.005 +// slider push 0.0 4.0 0.0 +float4 effect(float2 uv, Inputs in) { + float2 c = uv - 0.5; + float2 turned = float2(c.x * cos(in.p.spin) - c.y * sin(in.p.spin), c.x * sin(in.p.spin) + c.y * cos(in.p.spin)); + float2 from = turned * (1.0 - in.p.zoom) + 0.5 - in.flow.sample(in.smp, uv).xy * in.p.push; + float3 trail = in.prev.sample(in.smp, from).rgb * in.p.fade; + float3 live = in.image.sample(in.smp, uv).rgb; + return float4(max(live, trail), 1.0); +} diff --git a/native/effects/wave-shift.metal b/native/effects/wave-shift.metal new file mode 100644 index 0000000..42bb3c4 --- /dev/null +++ b/native/effects/wave-shift.metal @@ -0,0 +1,13 @@ +// RGB split that ripples down the image and gets pushed around by your movement. +// order 60 +// slider amount 0.0 0.05 0.012 +// slider waves 1.0 80.0 24.0 +// slider speed 0.0 10.0 3.0 +// slider react 0.0 3.0 1.0 +float4 effect(float2 uv, Inputs in) { + float2 d = float2(in.p.amount * sin(uv.y * in.p.waves + in.time * in.p.speed), 0.0); + d += in.flow.sample(in.smp, uv).xy * in.p.react; + return float4(in.image.sample(in.smp, uv + d).r, + in.image.sample(in.smp, uv).g, + in.image.sample(in.smp, uv - d).b, 1.0); +} diff --git a/panel.py b/panel.py new file mode 100644 index 0000000..6290a5c --- /dev/null +++ b/panel.py @@ -0,0 +1,61 @@ +"""Control panel drawn straight onto the output canvas: click a layer to toggle it, drag its sliders.""" +import cv2 + +BG = (26, 24, 22) +TRACK = (52, 48, 46) +FG = (228, 228, 228) +DIM = (125, 125, 125) +ACCENT = (255, 176, 64) +FONT = cv2.FONT_HERSHEY_SIMPLEX + + +class Panel: + WIDTH = 260 + + def __init__(self, layers): + self.layers = layers + self.hits = [] # (x0, y0, x1, y1, target, bar_x0, bar_x1), rebuilt on every draw + self.drag = None + + def draw(self, canvas, x0, status): + h = canvas.shape[0] + canvas[:, x0:] = BG + self.hits = [] + x, right = x0 + 14, x0 + self.WIDTH - 14 + cv2.putText(canvas, "MOTIONLAB", (x, 26), FONT, 0.55, FG, 1, cv2.LINE_AA) + cv2.putText(canvas, status, (x + 124, 26), FONT, 0.4, DIM, 1, cv2.LINE_AA) + + y = 44 + for layer in self.layers: + on = layer.enabled + cv2.rectangle(canvas, (x, y), (x + 12, y + 12), ACCENT if on else DIM, -1 if on else 1) + cv2.putText(canvas, layer.name, (x + 22, y + 11), FONT, 0.48, FG if on else DIM, 1, cv2.LINE_AA) + self.hits.append((x0, y - 5, x0 + self.WIDTH, y + 17, layer, 0, 0)) + y += 22 + if not on: + continue + bar = x + 100 + for name, p in layer.params.items(): + cv2.putText(canvas, name, (x + 4, y + 8), FONT, 0.36, DIM, 1, cv2.LINE_AA) + cv2.rectangle(canvas, (bar, y), (right, y + 8), TRACK, -1) + cv2.rectangle(canvas, (bar, y), (bar + int((right - bar) * p.norm), y + 8), ACCENT, -1) + self.hits.append((bar - 8, y - 4, right + 8, y + 12, p, bar, right)) + y += 16 + y += 4 + + cv2.putText(canvas, "[h] panel [s] shot [r] rec [q] quit", (x, h - 12), FONT, 0.36, DIM, 1, cv2.LINE_AA) + + def on_mouse(self, event, mx, my, flags, _): + if event == cv2.EVENT_LBUTTONDOWN: + for x0, y0, x1, y1, target, bar_x0, bar_x1 in self.hits: + if x0 <= mx < x1 and y0 <= my < y1: + if bar_x1: + self.drag = (target, bar_x0, bar_x1) + else: + target.enabled = not target.enabled + break + elif event == cv2.EVENT_LBUTTONUP: + self.drag = None + if self.drag and event in (cv2.EVENT_LBUTTONDOWN, cv2.EVENT_MOUSEMOVE): + param, bar_x0, bar_x1 = self.drag + param.norm = (mx - bar_x0) / (bar_x1 - bar_x0) diff --git a/shaders.py b/shaders.py new file mode 100644 index 0000000..52b7b0f --- /dev/null +++ b/shaders.py @@ -0,0 +1,172 @@ +"""Shader layers: every effects/*.glsl file becomes a layer, reloaded whenever the file is saved. + +A shader file only contains its own sliders and a main(). The host supplies the rest: + + in vec2 uv; pixel position, (0,0) top-left .. (1,1) bottom-right + out vec4 color; what to write for this pixel + uniform sampler2D camera; the incoming frame (output of the layer above), RGB + uniform sampler2D prev; this layer's own output from the previous frame + uniform sampler2D flow; .xy = how far this pixel moved since last frame, in uv units + uniform sampler2D motion; .r = how much this pixel changed since last frame, 0..1 + uniform float time; seconds since start + uniform float energy; overall amount of motion in the frame, 0..1 + uniform vec2 resolution; output size in pixels + +Sliders are plain float uniforms with a range comment: + + uniform float amount; // 0.0 .. 0.05 = 0.01 +""" +import os +import re + +import moderngl +import numpy as np + +from effects import Layer, Param + +VERTEX = """#version 410 +out vec2 uv; +void main() { + uv = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); + gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); +} +""" + +# "#line 1" makes compiler errors report line numbers of the user's file +HEADER = """#version 410 +uniform sampler2D camera; +uniform sampler2D prev; +uniform sampler2D flow; +uniform sampler2D motion; +uniform float time; +uniform float energy; +uniform vec2 resolution; +in vec2 uv; +out vec4 color; +#line 1 +""" + +NUM = r"(-?[\d.]+)" +PARAM_RE = re.compile(rf"uniform\s+float\s+(\w+)\s*;(?:[ \t]*//\s*{NUM}\s*\.\.\s*{NUM}\s*=\s*{NUM})?") + + +class ShaderLayer(Layer): + enabled = False + + def __init__(self, gl, path): + super().__init__() + self.gl, self.path = gl, path + self.name = os.path.splitext(os.path.basename(path))[0] + self.program = None + self.error = None + self.mtime = None + self.targets = None + self.reload() + + def reload(self): + """Recompile if the file changed. On a compile error the last working version keeps running.""" + try: + mtime = os.stat(self.path).st_mtime + except FileNotFoundError: + return + if mtime == self.mtime: + return + self.mtime = mtime + with open(self.path) as f: + source = f.read() + try: + program = self.gl.program(vertex_shader=VERTEX, fragment_shader=HEADER + source) + except moderngl.Error as e: + self.error = str(e) + print(f"[{self.name}] shader error:\n{e}", flush=True) + return + self.error = None + self.program = program + self.vao = self.gl.vertex_array(program, []) + old = self.params + self.params = {name: Param(float(default or 0.5), float(lo or 0), float(hi or 1)) + for name, lo, hi, default in PARAM_RE.findall(source)} + for name, p in self.params.items(): # keep slider positions across reloads + if name in old: + p.value = min(max(old[name].value, p.lo), p.hi) + print(f"[{self.name}] loaded", flush=True) + + def _target(self, size): + texture = self.gl.texture(size, 4) + texture.repeat_x = texture.repeat_y = False + return texture, self.gl.framebuffer([texture]) + + def _texture(self, attr, size, components, dtype="f1"): + if not hasattr(self, attr): + texture = self.gl.texture(size, components, dtype=dtype) + texture.repeat_x = texture.repeat_y = False + setattr(self, attr, texture) + return getattr(self, attr) + + def _set(self, name, value): + uniform = self.program.get(name, None) # absent if the shader doesn't use it + if uniform is not None: + uniform.value = value + return uniform is not None + + def apply(self, frame, ctx): + if self.program is None: + return frame + h, w = frame.shape[:2] + if self.targets is None: + # two render targets, swapped every frame, so the shader can read its previous output + self.targets = [self._target((w, h)), self._target((w, h))] + (_, framebuffer), (prev, _) = self.targets + + camera = self._texture("camera", (w, h), 3) + camera.swizzle = "BGR1" # frames are BGR; shaders see RGB + camera.write(np.ascontiguousarray(frame)) + camera.use(0) + self._set("camera", 0) + prev.use(1) + self._set("prev", 1) + if self._set("flow", 2): + fh, fw = ctx.flow.shape[:2] + flow = self._texture("flow_tex", (fw, fh), 2, "f4") + flow.write((ctx.flow / (fw, fh)).astype("f4")) + flow.use(2) + if self._set("motion", 3): + mh, mw = ctx.motion.shape + motion = self._texture("motion_tex", (mw, mh), 1) + motion.write(ctx.motion) + motion.use(3) + self._set("time", ctx.t) + self._set("energy", ctx.energy) + self._set("resolution", (w, h)) + for name, p in self.params.items(): + self._set(name, p.value) + + framebuffer.use() + self.vao.render(moderngl.TRIANGLES, vertices=3) + out = np.frombuffer(framebuffer.read(components=3), np.uint8).reshape(h, w, 3) + self.targets.reverse() + return np.ascontiguousarray(out[:, :, ::-1]) + + +class ShaderBank: + """Watches a folder of .glsl files and keeps one ShaderLayer per file in the shared layer list.""" + + def __init__(self, directory, layers): + self.directory, self.layers = directory, layers + self.known = {} + self.gl = moderngl.create_context(standalone=True, require=410) + self.refresh() + + def refresh(self): + if os.path.isdir(self.directory): + for filename in sorted(os.listdir(self.directory)): + path = os.path.join(self.directory, filename) + if filename.endswith(".glsl") and path not in self.known: + self.known[path] = ShaderLayer(self.gl, path) + self.layers.append(self.known[path]) + for layer in self.known.values(): + layer.reload() + + @property + def errors(self): + return [layer.name for layer in self.known.values() if layer.error]