Python/OpenCV proof of concept plus first native Swift port
Snapshot before the Python POC is removed. The POC (main.py, effects.py, panel.py, shaders.py, effects/*.glsl, flake.nix) is the reference for the native app under native/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
6591138bbf
27 changed files with 2076 additions and 0 deletions
5
.gitignore
vendored
Normal file
5
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
.build/
|
||||||
|
captures/
|
||||||
|
__pycache__/
|
||||||
|
.DS_Store
|
||||||
|
headless.png
|
||||||
227
effects.py
Normal file
227
effects.py
Normal file
|
|
@ -0,0 +1,227 @@
|
||||||
|
"""Effect layers.
|
||||||
|
|
||||||
|
Each layer takes the current BGR frame plus a shared Context (motion analysis
|
||||||
|
of the raw camera frame) and returns a new BGR frame. Layers run top to bottom
|
||||||
|
in the order returned by default_layers(), like a chain of TOPs.
|
||||||
|
"""
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
rng = np.random.default_rng()
|
||||||
|
|
||||||
|
|
||||||
|
class Param:
|
||||||
|
def __init__(self, value, lo=0.0, hi=1.0, integer=False):
|
||||||
|
self.value, self.lo, self.hi, self.integer = value, lo, hi, integer
|
||||||
|
|
||||||
|
@property
|
||||||
|
def norm(self):
|
||||||
|
return (self.value - self.lo) / (self.hi - self.lo)
|
||||||
|
|
||||||
|
@norm.setter
|
||||||
|
def norm(self, n):
|
||||||
|
v = self.lo + min(max(n, 0.0), 1.0) * (self.hi - self.lo)
|
||||||
|
self.value = round(v) if self.integer else v
|
||||||
|
|
||||||
|
|
||||||
|
class Layer:
|
||||||
|
name = "layer"
|
||||||
|
enabled = True
|
||||||
|
|
||||||
|
def __init__(self, **params):
|
||||||
|
self.params = params
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
return self.params[key].value
|
||||||
|
|
||||||
|
def apply(self, frame, ctx):
|
||||||
|
return frame
|
||||||
|
|
||||||
|
|
||||||
|
class Context:
|
||||||
|
"""Per-frame motion analysis, computed at 1/4 resolution and shared by all layers."""
|
||||||
|
|
||||||
|
SCALE = 4
|
||||||
|
|
||||||
|
def __init__(self, size):
|
||||||
|
self.small = (size[0] // self.SCALE, size[1] // self.SCALE)
|
||||||
|
self.prev = None
|
||||||
|
self.t = 0.0
|
||||||
|
self.dt = 1 / 30
|
||||||
|
|
||||||
|
def update(self, frame, dt):
|
||||||
|
self.dt = dt
|
||||||
|
self.t += dt
|
||||||
|
gray = cv2.cvtColor(cv2.resize(frame, self.small, interpolation=cv2.INTER_AREA), cv2.COLOR_BGR2GRAY)
|
||||||
|
if self.prev is None:
|
||||||
|
self.prev = gray
|
||||||
|
self.motion = cv2.absdiff(gray, self.prev) # where things changed
|
||||||
|
self.energy = float(self.motion.mean()) / 255 # how much changed overall
|
||||||
|
self._pair = (self.prev, gray)
|
||||||
|
self._flow = None
|
||||||
|
self.prev = gray
|
||||||
|
|
||||||
|
@property
|
||||||
|
def flow(self):
|
||||||
|
"""Dense optical flow (small-res pixels/frame). Lazy: only paid for if a layer asks."""
|
||||||
|
if self._flow is None:
|
||||||
|
self._flow = cv2.calcOpticalFlowFarneback(*self._pair, None, 0.5, 2, 15, 2, 5, 1.1, 0)
|
||||||
|
return self._flow
|
||||||
|
|
||||||
|
|
||||||
|
def hue_bgr(hue):
|
||||||
|
"""Hue in 0..1 (wraps) -> saturated BGR colour with components in 0..1."""
|
||||||
|
hsv = np.uint8([[[int(hue % 1.0 * 180), 255, 255]]])
|
||||||
|
return cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)[0, 0] / 255.0
|
||||||
|
|
||||||
|
|
||||||
|
def tint(gray, color):
|
||||||
|
return cv2.merge([cv2.convertScaleAbs(gray, alpha=float(c)) for c in color])
|
||||||
|
|
||||||
|
|
||||||
|
class Colorize(Layer):
|
||||||
|
name = "Colorize"
|
||||||
|
enabled = False
|
||||||
|
PALETTES = [cv2.COLORMAP_INFERNO, cv2.COLORMAP_OCEAN, cv2.COLORMAP_TWILIGHT_SHIFTED, cv2.COLORMAP_TURBO]
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(mix=Param(0.8), palette=Param(0, 0, len(self.PALETTES) - 1, integer=True))
|
||||||
|
|
||||||
|
def apply(self, frame, ctx):
|
||||||
|
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||||||
|
mapped = cv2.applyColorMap(gray, self.PALETTES[self["palette"]])
|
||||||
|
return cv2.addWeighted(frame, 1 - self["mix"], mapped, self["mix"], 0)
|
||||||
|
|
||||||
|
|
||||||
|
class Edges(Layer):
|
||||||
|
name = "Neon Edges"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(detail=Param(0.6), glow=Param(0.35), source=Param(0.35),
|
||||||
|
hue=Param(0.55), cycle=Param(0.3))
|
||||||
|
|
||||||
|
def apply(self, frame, ctx):
|
||||||
|
gray = cv2.GaussianBlur(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), (5, 5), 0)
|
||||||
|
lo = 20 + 180 * (1 - self["detail"])
|
||||||
|
edges = cv2.Canny(gray, lo, lo * 2)
|
||||||
|
sigma = self["glow"] * 12
|
||||||
|
if sigma > 0.5:
|
||||||
|
edges = cv2.addWeighted(edges, 1, cv2.GaussianBlur(edges, (0, 0), sigma), 2.5, 0)
|
||||||
|
color = hue_bgr(self["hue"] + ctx.t * self["cycle"] * 0.2)
|
||||||
|
return cv2.add(cv2.convertScaleAbs(frame, alpha=self["source"]), tint(edges, color))
|
||||||
|
|
||||||
|
|
||||||
|
class Particles(Layer):
|
||||||
|
"""Particles spawn where there is motion and get carried along by the optical flow."""
|
||||||
|
|
||||||
|
name = "Flow Particles"
|
||||||
|
MAX = 6000
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(count=Param(3000, 200, self.MAX, integer=True), speed=Param(1.5, 0, 4),
|
||||||
|
trail=Param(0.88, 0, 0.98), hue=Param(0.08))
|
||||||
|
self.buf = None
|
||||||
|
|
||||||
|
def apply(self, frame, ctx):
|
||||||
|
h, w = frame.shape[:2]
|
||||||
|
if self.buf is None:
|
||||||
|
self.buf = np.zeros((h, w), np.uint8)
|
||||||
|
self.pos = (rng.random((self.MAX, 2)) * (w, h)).astype(np.float32)
|
||||||
|
self.life = rng.integers(20, 90, self.MAX)
|
||||||
|
|
||||||
|
n = self["count"]
|
||||||
|
pos, life = self.pos[:n], self.life[:n]
|
||||||
|
flow = ctx.flow
|
||||||
|
fh, fw = flow.shape[:2]
|
||||||
|
scale = w / fw
|
||||||
|
ix = np.clip((pos[:, 0] / scale).astype(int), 0, fw - 1)
|
||||||
|
iy = np.clip((pos[:, 1] / scale).astype(int), 0, fh - 1)
|
||||||
|
new = pos + flow[iy, ix] * scale * self["speed"] + rng.normal(0, 0.5, (n, 2)).astype(np.float32)
|
||||||
|
|
||||||
|
life -= 1
|
||||||
|
dead = (life <= 0) | (new[:, 0] < 0) | (new[:, 0] >= w) | (new[:, 1] < 0) | (new[:, 1] >= h)
|
||||||
|
|
||||||
|
self.buf = cv2.convertScaleAbs(self.buf, alpha=self["trail"])
|
||||||
|
segments = np.stack([pos, new], axis=1).astype(np.int32)[~dead]
|
||||||
|
cv2.polylines(self.buf, list(segments), False, 255, 1, cv2.LINE_AA)
|
||||||
|
|
||||||
|
k = int(dead.sum())
|
||||||
|
if k:
|
||||||
|
hot = np.flatnonzero(ctx.motion > 24)
|
||||||
|
if hot.size > 50:
|
||||||
|
ys, xs = np.divmod(rng.choice(hot, k), fw)
|
||||||
|
new[dead] = (np.stack([xs, ys], axis=1) + rng.random((k, 2))) * scale
|
||||||
|
else:
|
||||||
|
new[dead] = rng.random((k, 2)) * (w, h)
|
||||||
|
life[dead] = rng.integers(20, 90, k)
|
||||||
|
self.pos[:n] = new
|
||||||
|
|
||||||
|
glow = cv2.add(self.buf, cv2.GaussianBlur(self.buf, (0, 0), 3))
|
||||||
|
return cv2.add(frame, tint(glow, hue_bgr(self["hue"])))
|
||||||
|
|
||||||
|
|
||||||
|
class Feedback(Layer):
|
||||||
|
"""Classic feedback loop: the previous output is transformed, faded and blended back in."""
|
||||||
|
|
||||||
|
name = "Feedback"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(decay=Param(0.92, 0, 0.995), zoom=Param(0.012, -0.05, 0.05),
|
||||||
|
rotate=Param(0.3, -3, 3), **{"flow push": Param(0.0, 0, 4)})
|
||||||
|
self.prev = None
|
||||||
|
|
||||||
|
def apply(self, frame, ctx):
|
||||||
|
h, w = frame.shape[:2]
|
||||||
|
if self.prev is None:
|
||||||
|
self.prev = frame
|
||||||
|
self.grid = np.stack(np.meshgrid(np.arange(w, dtype=np.float32),
|
||||||
|
np.arange(h, dtype=np.float32)), axis=-1)
|
||||||
|
m = cv2.getRotationMatrix2D((w / 2, h / 2), self["rotate"], 1 + self["zoom"])
|
||||||
|
fb = cv2.warpAffine(self.prev, m, (w, h), borderMode=cv2.BORDER_REFLECT)
|
||||||
|
if self["flow push"] > 0:
|
||||||
|
# smear the feedback along the direction things are moving
|
||||||
|
flow = cv2.resize(ctx.flow, (w, h)) * (self["flow push"] * Context.SCALE)
|
||||||
|
fb = cv2.remap(fb, self.grid - flow, None, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)
|
||||||
|
out = cv2.max(frame, cv2.convertScaleAbs(fb, alpha=self["decay"]))
|
||||||
|
self.prev = out
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class RGBShift(Layer):
|
||||||
|
name = "RGB Shift"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(amount=Param(4, 0, 40), **{"motion react": Param(0.5)})
|
||||||
|
|
||||||
|
def apply(self, frame, ctx):
|
||||||
|
d = int(self["amount"] + self["motion react"] * ctx.energy * 400)
|
||||||
|
if d == 0:
|
||||||
|
return frame
|
||||||
|
b, g, r = cv2.split(frame)
|
||||||
|
return cv2.merge([np.roll(b, d, axis=1), g, np.roll(r, -d, axis=1)])
|
||||||
|
|
||||||
|
|
||||||
|
class Kaleido(Layer):
|
||||||
|
name = "Kaleidoscope"
|
||||||
|
enabled = False
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(segments=Param(6, 2, 16, integer=True), spin=Param(0.2, -2, 2))
|
||||||
|
self.radius = None
|
||||||
|
self.angle = 0.0
|
||||||
|
|
||||||
|
def apply(self, frame, ctx):
|
||||||
|
h, w = frame.shape[:2]
|
||||||
|
if self.radius is None:
|
||||||
|
ys, xs = np.indices((h, w), dtype=np.float32)
|
||||||
|
self.radius, self.theta = cv2.cartToPolar(xs - w / 2, ys - h / 2)
|
||||||
|
self.angle += self["spin"] * ctx.dt
|
||||||
|
wedge = 2 * np.pi / self["segments"]
|
||||||
|
# fold every angle into one mirrored wedge, then sample the source from that wedge
|
||||||
|
a = np.abs(np.mod(self.theta + np.float32(self.angle), np.float32(wedge)) - np.float32(wedge / 2))
|
||||||
|
x, y = cv2.polarToCart(self.radius, a - np.float32(np.pi / 2))
|
||||||
|
return cv2.remap(frame, x + w / 2, y + h / 2, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)
|
||||||
|
|
||||||
|
|
||||||
|
def default_layers():
|
||||||
|
return [Colorize(), Edges(), Particles(), Feedback(), RGBShift(), Kaleido()]
|
||||||
17
effects/halftone.glsl
Normal file
17
effects/halftone.glsl
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
// Halftone dots sized by brightness; dots turn hot orange wherever something moves.
|
||||||
|
uniform float cell; // 4.0 .. 40.0 = 10.0
|
||||||
|
uniform float heat; // 0.0 .. 6.0 = 3.0
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
vec2 grid = resolution / cell;
|
||||||
|
vec2 centre = (floor(uv * grid) + 0.5) / grid;
|
||||||
|
vec3 c = texture(camera, centre).rgb;
|
||||||
|
float luma = dot(c, vec3(0.299, 0.587, 0.114));
|
||||||
|
|
||||||
|
float radius = 0.5 * sqrt(luma);
|
||||||
|
float dist = length(fract(uv * grid) - 0.5);
|
||||||
|
float mask = 1.0 - smoothstep(radius - 0.05, radius + 0.05, dist);
|
||||||
|
|
||||||
|
float moving = clamp(texture(motion, centre).r * heat, 0.0, 1.0);
|
||||||
|
color = vec4(mix(c, vec3(1.0, 0.35, 0.1), moving) * mask, 1.0);
|
||||||
|
}
|
||||||
13
effects/liquid.glsl
Normal file
13
effects/liquid.glsl
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
// Liquid trails: the previous output is dragged along the motion, swirled a little,
|
||||||
|
// and slowly fades back to the live camera.
|
||||||
|
uniform float persistence; // 0.0 .. 0.99 = 0.92
|
||||||
|
uniform float drag; // 0.0 .. 6.0 = 2.5
|
||||||
|
uniform float swirl; // 0.0 .. 0.01 = 0.002
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
vec2 centred = uv - 0.5;
|
||||||
|
vec2 from = uv - texture(flow, uv).xy * drag + vec2(-centred.y, centred.x) * swirl;
|
||||||
|
vec3 trail = texture(prev, from).rgb;
|
||||||
|
vec3 live = texture(camera, uv).rgb;
|
||||||
|
color = vec4(mix(live, trail, persistence), 1.0);
|
||||||
|
}
|
||||||
12
effects/wave-shift.glsl
Normal file
12
effects/wave-shift.glsl
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
// RGB split that ripples down the image and gets pushed around by your movement.
|
||||||
|
uniform float amount; // 0.0 .. 0.05 = 0.012
|
||||||
|
uniform float waves; // 1.0 .. 80.0 = 24.0
|
||||||
|
uniform float react; // 0.0 .. 3.0 = 1.0
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
vec2 d = vec2(amount * sin(uv.y * waves + time * 3.0), 0.0);
|
||||||
|
d += texture(flow, uv).xy * react;
|
||||||
|
color = vec4(texture(camera, uv + d).r,
|
||||||
|
texture(camera, uv).g,
|
||||||
|
texture(camera, uv - d).b, 1.0);
|
||||||
|
}
|
||||||
27
flake.lock
generated
Normal file
27
flake.lock
generated
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1791097562,
|
||||||
|
"narHash": "sha256-K9O9lG3UZNFCnwrz0ViWgP3Kgqukg1DvvzsrkFJfcvU=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "a322c65bf7233456788a9ebf03bc067db7dc090a",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixpkgs-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
53
flake.nix
Normal file
53
flake.nix
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
{
|
||||||
|
description = "motionlab - realtime camera + motion graphics playground";
|
||||||
|
|
||||||
|
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||||
|
|
||||||
|
outputs = { self, nixpkgs }:
|
||||||
|
let
|
||||||
|
systems = [ "aarch64-darwin" "x86_64-darwin" "x86_64-linux" "aarch64-linux" ];
|
||||||
|
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system});
|
||||||
|
|
||||||
|
# not in nixpkgs; sends frames to the OBS virtual camera (macOS) or v4l2loopback (Linux)
|
||||||
|
pyvirtualcamFor = ps: ps.buildPythonPackage rec {
|
||||||
|
pname = "pyvirtualcam";
|
||||||
|
version = "0.15.0";
|
||||||
|
pyproject = true;
|
||||||
|
src = nixpkgs.legacyPackages.${ps.python.stdenv.hostPlatform.system}.fetchFromGitHub {
|
||||||
|
owner = "letmaik";
|
||||||
|
repo = "pyvirtualcam";
|
||||||
|
tag = "v${version}";
|
||||||
|
fetchSubmodules = true;
|
||||||
|
hash = "sha256-JtFchhuvVYYhrYOOcsHPC79/Iezyo5QppV4tY9nrjgQ=";
|
||||||
|
};
|
||||||
|
build-system = [ ps.setuptools ps.pybind11 ];
|
||||||
|
dependencies = [ ps.numpy ];
|
||||||
|
doCheck = false;
|
||||||
|
pythonImportsCheck = [ "pyvirtualcam" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
pythonFor = pkgs: pkgs.python3.withPackages (ps: [
|
||||||
|
ps.numpy
|
||||||
|
ps.moderngl
|
||||||
|
(pyvirtualcamFor ps)
|
||||||
|
# macOS gets a Cocoa window out of the box; Linux needs GTK for cv2.imshow
|
||||||
|
(if pkgs.stdenv.hostPlatform.isLinux
|
||||||
|
then ps.opencv4.override { enableGtk3 = true; }
|
||||||
|
else ps.opencv4)
|
||||||
|
]);
|
||||||
|
in
|
||||||
|
{
|
||||||
|
devShells = forAllSystems (pkgs: {
|
||||||
|
default = pkgs.mkShell { packages = [ (pythonFor pkgs) ]; };
|
||||||
|
});
|
||||||
|
|
||||||
|
apps = forAllSystems (pkgs: {
|
||||||
|
default = {
|
||||||
|
type = "app";
|
||||||
|
program = toString (pkgs.writeShellScript "motionlab" ''
|
||||||
|
exec ${pythonFor pkgs}/bin/python ${self}/main.py "$@"
|
||||||
|
'');
|
||||||
|
};
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
191
main.py
Normal file
191
main.py
Normal file
|
|
@ -0,0 +1,191 @@
|
||||||
|
"""motionlab - a tiny TouchDesigner-style playground: live camera in, motion-reactive graphics out."""
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from effects import Context, default_layers, hue_bgr
|
||||||
|
from panel import Panel
|
||||||
|
from shaders import ShaderBank
|
||||||
|
|
||||||
|
SIZE = (960, 540)
|
||||||
|
VCAM_SIZE = (1280, 720)
|
||||||
|
WINDOW = "motionlab"
|
||||||
|
|
||||||
|
|
||||||
|
class Synthetic:
|
||||||
|
"""Drifting blobs, so the app (and --headless) works without a camera."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.i = 0
|
||||||
|
|
||||||
|
def read(self):
|
||||||
|
w, h = SIZE
|
||||||
|
t = self.i / 30
|
||||||
|
self.i += 1
|
||||||
|
img = np.full((h, w, 3), (30, 20, 20), np.uint8)
|
||||||
|
for k in range(5):
|
||||||
|
x = int(w * (0.5 + 0.4 * np.sin(t * (0.7 + 0.2 * k) + k)))
|
||||||
|
y = int(h * (0.5 + 0.4 * np.cos(t * (0.5 + 0.3 * k) + 2 * k)))
|
||||||
|
cv2.circle(img, (x, y), 40 + 12 * k, hue_bgr(k / 5) * 255, -1)
|
||||||
|
return True, cv2.GaussianBlur(img, (0, 0), 4)
|
||||||
|
|
||||||
|
def release(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def open_source(args):
|
||||||
|
if args.demo or args.headless:
|
||||||
|
return Synthetic()
|
||||||
|
backend = cv2.CAP_AVFOUNDATION if sys.platform == "darwin" else cv2.CAP_ANY
|
||||||
|
cap = cv2.VideoCapture(args.camera, backend)
|
||||||
|
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
|
||||||
|
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
|
||||||
|
# without camera permission macOS can hand back all-black frames instead of failing
|
||||||
|
for _ in range(10):
|
||||||
|
ok, frame = cap.read()
|
||||||
|
if ok and frame.any():
|
||||||
|
return cap
|
||||||
|
cap.release()
|
||||||
|
print("Could not read from the camera - falling back to the synthetic demo source.\n"
|
||||||
|
"On macOS: allow your terminal under System Settings > Privacy & Security > Camera, then rerun.",
|
||||||
|
file=sys.stderr)
|
||||||
|
return Synthetic()
|
||||||
|
|
||||||
|
|
||||||
|
def open_virtualcam():
|
||||||
|
"""Virtual camera other apps (Meet, Zoom, ...) can pick as their webcam."""
|
||||||
|
import pyvirtualcam
|
||||||
|
try:
|
||||||
|
cam = pyvirtualcam.Camera(*VCAM_SIZE, 30, fmt=pyvirtualcam.PixelFormat.BGR)
|
||||||
|
except RuntimeError as e:
|
||||||
|
sys.exit(f"Could not start the virtual camera: {e}")
|
||||||
|
print(f"virtual camera on: pick '{cam.device}' as the camera in your call app")
|
||||||
|
return cam
|
||||||
|
|
||||||
|
|
||||||
|
def fit(frame):
|
||||||
|
"""Centre-crop to the output aspect ratio, then resize."""
|
||||||
|
w, h = SIZE
|
||||||
|
fh, fw = frame.shape[:2]
|
||||||
|
cw = min(fw, fh * w // h)
|
||||||
|
ch = min(fh, fw * h // w)
|
||||||
|
x, y = (fw - cw) // 2, (fh - ch) // 2
|
||||||
|
return cv2.resize(frame[y:y + ch, x:x + cw], SIZE, interpolation=cv2.INTER_AREA)
|
||||||
|
|
||||||
|
|
||||||
|
def capture_path(ext):
|
||||||
|
os.makedirs("captures", exist_ok=True)
|
||||||
|
return os.path.join("captures", time.strftime(f"motionlab-%Y%m%d-%H%M%S.{ext}"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__)
|
||||||
|
ap.add_argument("--camera", type=int, default=0, help="camera index (default 0)")
|
||||||
|
ap.add_argument("--demo", action="store_true", help="use the synthetic source instead of the camera")
|
||||||
|
ap.add_argument("--virtualcam", action="store_true", help="also send the output to a virtual camera for video calls")
|
||||||
|
ap.add_argument("--headless", type=int, metavar="N", help="render N synthetic frames without a window, then exit")
|
||||||
|
ap.add_argument("--out", default="headless.png", help="where --headless writes its last frame")
|
||||||
|
ap.add_argument("--effects", default="effects", help="folder of .glsl shader effects (default ./effects)")
|
||||||
|
ap.add_argument("--only", metavar="NAMES", help="enable only these layers, comma separated (e.g. --only liquid,halftone)")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
w, h = SIZE
|
||||||
|
src = open_source(args)
|
||||||
|
layers = default_layers()
|
||||||
|
# prefer ./effects so edits are picked up live; fall back to the copy shipped next to this file
|
||||||
|
effects_dir = args.effects if os.path.isdir(args.effects) else os.path.join(os.path.dirname(__file__), "effects")
|
||||||
|
bank = ShaderBank(effects_dir, layers)
|
||||||
|
if args.only:
|
||||||
|
wanted = {name.strip().lower() for name in args.only.split(",")}
|
||||||
|
unknown = wanted - {layer.name.lower() for layer in layers}
|
||||||
|
if unknown:
|
||||||
|
sys.exit(f"unknown layer(s): {', '.join(sorted(unknown))}. Available: {', '.join(l.name for l in layers)}")
|
||||||
|
for layer in layers:
|
||||||
|
layer.enabled = layer.name.lower() in wanted
|
||||||
|
if args.headless and bank.errors:
|
||||||
|
sys.exit(f"shader compile failed: {', '.join(bank.errors)}")
|
||||||
|
ctx = Context(SIZE)
|
||||||
|
panel = Panel(layers)
|
||||||
|
canvas = np.empty((h, w + Panel.WIDTH, 3), np.uint8)
|
||||||
|
show_panel, writer, fps = True, None, 30.0
|
||||||
|
vcam = open_virtualcam() if args.virtualcam else None
|
||||||
|
|
||||||
|
if not args.headless:
|
||||||
|
cv2.namedWindow(WINDOW, cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO)
|
||||||
|
cv2.resizeWindow(WINDOW, w + Panel.WIDTH, h)
|
||||||
|
cv2.setMouseCallback(WINDOW, panel.on_mouse)
|
||||||
|
|
||||||
|
start = last = time.perf_counter()
|
||||||
|
frames = 0
|
||||||
|
while True:
|
||||||
|
ok, frame = src.read()
|
||||||
|
if not ok:
|
||||||
|
break
|
||||||
|
now = time.perf_counter()
|
||||||
|
dt = 1 / 30 if args.headless else max(now - last, 1e-3)
|
||||||
|
last = now
|
||||||
|
fps += (1 / dt - fps) * 0.1
|
||||||
|
|
||||||
|
frame = cv2.flip(fit(frame), 1) # mirror, so it behaves like a mirror
|
||||||
|
if frames % 15 == 0:
|
||||||
|
bank.refresh() # pick up new and edited shader files
|
||||||
|
ctx.update(frame, dt)
|
||||||
|
out = frame
|
||||||
|
for layer in layers:
|
||||||
|
if layer.enabled:
|
||||||
|
out = layer.apply(out, ctx)
|
||||||
|
|
||||||
|
if writer:
|
||||||
|
writer.write(out)
|
||||||
|
if vcam:
|
||||||
|
# un-mirror: the preview is a mirror, but the people on the call should see the real thing
|
||||||
|
vcam.send(cv2.resize(cv2.flip(out, 1), VCAM_SIZE))
|
||||||
|
view = out
|
||||||
|
if show_panel:
|
||||||
|
canvas[:, :w] = out
|
||||||
|
panel.draw(canvas, w, f"{fps:3.0f} fps" + (" REC" if writer else "") + (" CAM" if vcam else ""))
|
||||||
|
view = canvas
|
||||||
|
else:
|
||||||
|
panel.hits = []
|
||||||
|
|
||||||
|
frames += 1
|
||||||
|
if args.headless:
|
||||||
|
if frames >= args.headless:
|
||||||
|
cv2.imwrite(args.out, view)
|
||||||
|
print(f"{frames} frames, {(time.perf_counter() - start) / frames * 1000:.1f} ms/frame -> {args.out}")
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
|
||||||
|
cv2.imshow(WINDOW, view)
|
||||||
|
key = cv2.waitKey(1) & 0xFF
|
||||||
|
if key in (ord("q"), 27) or cv2.getWindowProperty(WINDOW, cv2.WND_PROP_VISIBLE) < 1:
|
||||||
|
break
|
||||||
|
elif key == ord("h"):
|
||||||
|
show_panel = not show_panel
|
||||||
|
elif key == ord("s"):
|
||||||
|
path = capture_path("png")
|
||||||
|
cv2.imwrite(path, out)
|
||||||
|
print("saved", path)
|
||||||
|
elif key == ord("r"):
|
||||||
|
if writer:
|
||||||
|
writer.release()
|
||||||
|
writer = None
|
||||||
|
else:
|
||||||
|
path = capture_path("mp4")
|
||||||
|
writer = cv2.VideoWriter(path, cv2.VideoWriter_fourcc(*"mp4v"), round(fps), SIZE)
|
||||||
|
print("recording", path)
|
||||||
|
|
||||||
|
if writer:
|
||||||
|
writer.release()
|
||||||
|
if vcam:
|
||||||
|
vcam.close()
|
||||||
|
src.release()
|
||||||
|
cv2.destroyAllWindows()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
10
native/Package.swift
Normal file
10
native/Package.swift
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
// swift-tools-version:5.9
|
||||||
|
import PackageDescription
|
||||||
|
|
||||||
|
let package = Package(
|
||||||
|
name: "MotionLab",
|
||||||
|
platforms: [.macOS(.v13)],
|
||||||
|
targets: [
|
||||||
|
.executableTarget(name: "MotionLab", path: "Sources/MotionLab")
|
||||||
|
]
|
||||||
|
)
|
||||||
180
native/Sources/MotionLab/Analysis.swift
Normal file
180
native/Sources/MotionLab/Analysis.swift
Normal file
|
|
@ -0,0 +1,180 @@
|
||||||
|
import Metal
|
||||||
|
|
||||||
|
/// Host-side motion analysis, shared by all effects. Every camera frame it produces:
|
||||||
|
///
|
||||||
|
/// flow which way things moved (small 2-channel texture, uv units per frame)
|
||||||
|
/// particles a picture of particles that spawn where something moves and drift along the flow
|
||||||
|
///
|
||||||
|
/// Effects read these as `in.flow` and `in.particles`.
|
||||||
|
final class Analysis {
|
||||||
|
private static let flowWidth = 160, flowHeight = 90
|
||||||
|
private static let stateWidth = 128, stateHeight = 64 // one texel per particle
|
||||||
|
private static let particleSpeed: Float = 1.5
|
||||||
|
private static let trailFade: Float = 0.88
|
||||||
|
|
||||||
|
// each pair is [this frame, previous frame], swapped at the start of every update
|
||||||
|
private var luma: [MTLTexture]
|
||||||
|
private var flowTextures: [MTLTexture]
|
||||||
|
private var state: [MTLTexture]
|
||||||
|
private var trail: [MTLTexture]
|
||||||
|
|
||||||
|
private let lumaPipeline: MTLRenderPipelineState
|
||||||
|
private let flowPipeline: MTLRenderPipelineState
|
||||||
|
private let movePipeline: MTLRenderPipelineState
|
||||||
|
private let fadePipeline: MTLRenderPipelineState
|
||||||
|
private let dotPipeline: MTLRenderPipelineState
|
||||||
|
|
||||||
|
var flow: MTLTexture { flowTextures[0] }
|
||||||
|
var particles: MTLTexture { trail[0] }
|
||||||
|
|
||||||
|
init(device: MTLDevice, queue: MTLCommandQueue, width: Int, height: Int) throws {
|
||||||
|
func pair(_ w: Int, _ h: Int, _ format: MTLPixelFormat) -> [MTLTexture] {
|
||||||
|
[makeTarget(device, width: w, height: h, format: format), makeTarget(device, width: w, height: h, format: format)]
|
||||||
|
}
|
||||||
|
luma = pair(Analysis.flowWidth, Analysis.flowHeight, .r16Float)
|
||||||
|
flowTextures = pair(Analysis.flowWidth, Analysis.flowHeight, .rg16Float)
|
||||||
|
state = pair(Analysis.stateWidth, Analysis.stateHeight, .rgba32Float)
|
||||||
|
trail = pair(width, height, .r8Unorm)
|
||||||
|
|
||||||
|
let source = shaderPrelude + analysisShaders
|
||||||
|
lumaPipeline = try makePipeline(device, source: source, fragment: "lumaFragment", format: .r16Float)
|
||||||
|
flowPipeline = try makePipeline(device, source: source, fragment: "flowFragment", format: .rg16Float)
|
||||||
|
movePipeline = try makePipeline(device, source: source, fragment: "moveFragment", format: .rgba32Float)
|
||||||
|
fadePipeline = try makePipeline(device, source: source, fragment: "fadeFragment", format: .r8Unorm)
|
||||||
|
dotPipeline = try makePipeline(device, source: source, vertex: "dotVertex", fragment: "dotFragment",
|
||||||
|
format: .r8Unorm, additive: true)
|
||||||
|
|
||||||
|
// new textures hold garbage; start everything from zero
|
||||||
|
if let commands = queue.makeCommandBuffer() {
|
||||||
|
for texture in luma + flowTextures + state + trail { commands.pass(into: texture, clear: true) }
|
||||||
|
commands.commit()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func update(_ commands: MTLCommandBuffer, camera: MTLTexture, particles wantsParticles: Bool) {
|
||||||
|
luma.swapAt(0, 1)
|
||||||
|
commands.pass(into: luma[0]) { $0.fullscreen(lumaPipeline, textures: [camera]) }
|
||||||
|
|
||||||
|
flowTextures.swapAt(0, 1)
|
||||||
|
commands.pass(into: flowTextures[0]) {
|
||||||
|
$0.fullscreen(flowPipeline, textures: [luma[0], luma[1], flowTextures[1]])
|
||||||
|
}
|
||||||
|
|
||||||
|
guard wantsParticles else { return }
|
||||||
|
state.swapAt(0, 1)
|
||||||
|
commands.pass(into: state[0]) {
|
||||||
|
$0.fullscreen(movePipeline, textures: [state[1], flowTextures[0], luma[0], luma[1]],
|
||||||
|
uniforms: [Float.random(in: 0..<1000), Analysis.particleSpeed, 0, 0])
|
||||||
|
}
|
||||||
|
trail.swapAt(0, 1)
|
||||||
|
commands.pass(into: trail[0]) { encoder in
|
||||||
|
encoder.fullscreen(fadePipeline, textures: [trail[1]], uniforms: [Analysis.trailFade, 0, 0, 0])
|
||||||
|
encoder.setRenderPipelineState(dotPipeline)
|
||||||
|
encoder.setVertexTexture(state[0], index: 0)
|
||||||
|
encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: Analysis.stateWidth * Analysis.stateHeight)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private let analysisShaders = """
|
||||||
|
static float brightness(float3 c) { return dot(c, float3(0.299, 0.587, 0.114)); }
|
||||||
|
static float hash(float2 p) { return fract(sin(dot(p, float2(127.1, 311.7))) * 43758.5453); }
|
||||||
|
|
||||||
|
// Camera -> small, blurred brightness image. Flow is measured on this.
|
||||||
|
fragment float4 lumaFragment(VertexOut v [[stage_in]], texture2d<float> camera [[texture(0)]]) {
|
||||||
|
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||||
|
float2 texel = 1.0 / float2(camera.get_width(), camera.get_height());
|
||||||
|
float sum = 0.0;
|
||||||
|
for (int y = 0; y < 4; y++) {
|
||||||
|
for (int x = 0; x < 4; x++) {
|
||||||
|
sum += brightness(camera.sample(s, v.uv + (float2(x, y) - 1.5) * 2.0 * texel).rgb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return float4(sum / 16.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lucas-Kanade optical flow: for each pixel, the shift that best explains how its 7x7
|
||||||
|
// neighbourhood changed between the previous and the current brightness image.
|
||||||
|
fragment float4 flowFragment(VertexOut v [[stage_in]],
|
||||||
|
texture2d<float> now [[texture(0)]],
|
||||||
|
texture2d<float> before [[texture(1)]],
|
||||||
|
texture2d<float> flowBefore [[texture(2)]]) {
|
||||||
|
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||||
|
float2 texel = 1.0 / float2(now.get_width(), now.get_height());
|
||||||
|
float a = 1e-3, b = 0.0, c = 1e-3, p = 0.0, q = 0.0; // the 1e-3 keeps flat areas at zero flow
|
||||||
|
for (int y = -3; y <= 3; y++) {
|
||||||
|
for (int x = -3; x <= 3; x++) {
|
||||||
|
float2 uv = v.uv + float2(x, y) * texel;
|
||||||
|
float ix = (now.sample(s, uv + float2(texel.x, 0.0)).r - now.sample(s, uv - float2(texel.x, 0.0)).r) * 0.5;
|
||||||
|
float iy = (now.sample(s, uv + float2(0.0, texel.y)).r - now.sample(s, uv - float2(0.0, texel.y)).r) * 0.5;
|
||||||
|
float it = now.sample(s, uv).r - before.sample(s, uv).r;
|
||||||
|
a += ix * ix; b += ix * iy; c += iy * iy;
|
||||||
|
p += ix * it; q += iy * it;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
float2 shift = -float2(c * p - b * q, a * q - b * p) / (a * c - b * b); // in small-image pixels
|
||||||
|
float len = length(shift);
|
||||||
|
if (len > 4.0) { shift *= 4.0 / len; }
|
||||||
|
float2 smoothed = mix(flowBefore.sample(s, v.uv).xy, shift * texel, 0.6);
|
||||||
|
return float4(smoothed, 0.0, 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// One texel per particle: xy = position (uv), z = frames left to live, w = 1 if it was born from motion.
|
||||||
|
fragment float4 moveFragment(VertexOut v [[stage_in]],
|
||||||
|
texture2d<float> state [[texture(0)]],
|
||||||
|
texture2d<float> flow [[texture(1)]],
|
||||||
|
texture2d<float> now [[texture(2)]],
|
||||||
|
texture2d<float> before [[texture(3)]],
|
||||||
|
constant float4 &u [[buffer(0)]]) { // x = random seed, y = speed
|
||||||
|
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||||
|
float4 particle = state.read(uint2(v.position.xy));
|
||||||
|
float2 seed = v.position.xy * 0.37 + u.x;
|
||||||
|
float2 pos = particle.xy + flow.sample(s, particle.xy).xy * u.y;
|
||||||
|
pos += (float2(hash(seed), hash(seed + 7.3)) - 0.5) * 0.0015;
|
||||||
|
float life = particle.z - 1.0;
|
||||||
|
float born = particle.w;
|
||||||
|
|
||||||
|
if (life <= 0.0 || pos.x < 0.0 || pos.x > 1.0 || pos.y < 0.0 || pos.y > 1.0) {
|
||||||
|
// respawn: try a few random spots and take the first one where something is moving
|
||||||
|
pos = float2(hash(seed + 1.1), hash(seed + 2.2));
|
||||||
|
born = 0.0;
|
||||||
|
life = 2.0 + hash(seed + 5.5) * 4.0; // nothing moving: stay hidden and try again soon
|
||||||
|
for (int i = 0; i < 6; i++) {
|
||||||
|
float2 spot = float2(hash(seed + 3.3 + float(i)), hash(seed + 9.9 + float(i) * 1.7));
|
||||||
|
if (abs(now.sample(s, spot).r - before.sample(s, spot).r) > 0.04) {
|
||||||
|
pos = spot;
|
||||||
|
born = 1.0;
|
||||||
|
life = 20.0 + hash(seed + 5.5) * 70.0;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return float4(pos, life, born);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Particle trails: fade what was there, then add a dot per particle on top.
|
||||||
|
fragment float4 fadeFragment(VertexOut v [[stage_in]], texture2d<float> trail [[texture(0)]],
|
||||||
|
constant float4 &u [[buffer(0)]]) { // x = fade
|
||||||
|
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||||
|
return trail.sample(s, v.uv) * u.x;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct DotOut {
|
||||||
|
float4 position [[position]];
|
||||||
|
float size [[point_size]];
|
||||||
|
};
|
||||||
|
|
||||||
|
vertex DotOut dotVertex(uint id [[vertex_id]], texture2d<float> state [[texture(0)]]) {
|
||||||
|
uint width = state.get_width();
|
||||||
|
float4 particle = state.read(uint2(id % width, id / width));
|
||||||
|
DotOut out;
|
||||||
|
out.position = float4(particle.x * 2.0 - 1.0, 1.0 - particle.y * 2.0, 0.0, 1.0);
|
||||||
|
if (particle.w < 0.5) { out.position.x = 4.0; } // hidden particles are moved off screen
|
||||||
|
out.size = 3.0;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
fragment float4 dotFragment(DotOut in [[stage_in]], float2 coord [[point_coord]]) {
|
||||||
|
return float4(smoothstep(0.5, 0.1, length(coord - 0.5)) * 0.9);
|
||||||
|
}
|
||||||
|
"""
|
||||||
95
native/Sources/MotionLab/Camera.swift
Normal file
95
native/Sources/MotionLab/Camera.swift
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
import AVFoundation
|
||||||
|
import CoreVideo
|
||||||
|
import Metal
|
||||||
|
|
||||||
|
enum CameraError: Error, CustomStringConvertible {
|
||||||
|
case noPermission
|
||||||
|
case noCamera
|
||||||
|
|
||||||
|
var description: String {
|
||||||
|
switch self {
|
||||||
|
case .noPermission:
|
||||||
|
return "No camera permission. Allow your terminal under System Settings > Privacy & Security > Camera, then restart it."
|
||||||
|
case .noCamera:
|
||||||
|
return "No camera found."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Streams the built-in camera as Metal textures; `latest` is the newest frame and the one before it.
|
||||||
|
final class Camera: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
||||||
|
private let session = AVCaptureSession()
|
||||||
|
private var cache: CVMetalTextureCache?
|
||||||
|
private let lock = NSLock()
|
||||||
|
// each entry keeps its CVMetalTexture, which keeps the pixel buffer behind the Metal texture alive
|
||||||
|
private var current: (texture: MTLTexture, backing: CVMetalTexture)?
|
||||||
|
private var previous: (texture: MTLTexture, backing: CVMetalTexture)?
|
||||||
|
|
||||||
|
private var serial = 0 // counts frames, so the renderer can tell a new frame from one it already saw
|
||||||
|
|
||||||
|
var latest: (current: MTLTexture, last: MTLTexture, serial: Int)? {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
guard let current else { return nil }
|
||||||
|
return (current.texture, (previous ?? current).texture, serial)
|
||||||
|
}
|
||||||
|
|
||||||
|
init(device: MTLDevice) throws {
|
||||||
|
super.init()
|
||||||
|
guard Camera.requestAccess() else { throw CameraError.noPermission }
|
||||||
|
// prefer the built-in camera over an iPhone that happens to be nearby
|
||||||
|
guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .unspecified)
|
||||||
|
?? AVCaptureDevice.default(for: .video)
|
||||||
|
else { throw CameraError.noCamera }
|
||||||
|
|
||||||
|
CVMetalTextureCacheCreate(nil, nil, device, nil, &cache)
|
||||||
|
let output = AVCaptureVideoDataOutput()
|
||||||
|
output.videoSettings = [
|
||||||
|
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||||
|
kCVPixelBufferMetalCompatibilityKey as String: true,
|
||||||
|
]
|
||||||
|
output.alwaysDiscardsLateVideoFrames = true
|
||||||
|
output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "motionlab.camera"))
|
||||||
|
|
||||||
|
session.beginConfiguration()
|
||||||
|
session.addInput(try AVCaptureDeviceInput(device: camera))
|
||||||
|
session.addOutput(output)
|
||||||
|
if session.canSetSessionPreset(.hd1280x720) { session.sessionPreset = .hd1280x720 }
|
||||||
|
print("camera: \(camera.localizedName)")
|
||||||
|
if let connection = output.connection(with: .video), connection.isVideoMirroringSupported {
|
||||||
|
connection.automaticallyAdjustsVideoMirroring = false
|
||||||
|
connection.isVideoMirrored = true // behave like a mirror
|
||||||
|
}
|
||||||
|
session.commitConfiguration()
|
||||||
|
session.startRunning()
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func requestAccess() -> Bool {
|
||||||
|
switch AVCaptureDevice.authorizationStatus(for: .video) {
|
||||||
|
case .authorized:
|
||||||
|
return true
|
||||||
|
case .notDetermined:
|
||||||
|
let done = DispatchSemaphore(value: 0)
|
||||||
|
var granted = false
|
||||||
|
AVCaptureDevice.requestAccess(for: .video) { granted = $0; done.signal() }
|
||||||
|
done.wait()
|
||||||
|
return granted
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
|
||||||
|
guard let cache, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
||||||
|
var created: CVMetalTexture?
|
||||||
|
CVMetalTextureCacheCreateTextureFromImage(
|
||||||
|
nil, cache, buffer, nil, .bgra8Unorm,
|
||||||
|
CVPixelBufferGetWidth(buffer), CVPixelBufferGetHeight(buffer), 0, &created)
|
||||||
|
guard let created, let frame = CVMetalTextureGetTexture(created) else { return }
|
||||||
|
lock.lock()
|
||||||
|
previous = current
|
||||||
|
current = (frame, created)
|
||||||
|
serial += 1
|
||||||
|
lock.unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
198
native/Sources/MotionLab/Output.swift
Normal file
198
native/Sources/MotionLab/Output.swift
Normal file
|
|
@ -0,0 +1,198 @@
|
||||||
|
import AVFoundation
|
||||||
|
import CoreMediaIO
|
||||||
|
import ImageIO
|
||||||
|
import UniformTypeIdentifiers
|
||||||
|
|
||||||
|
func writePNG(bgra pixels: inout [UInt8], width: Int, height: Int, to url: URL) {
|
||||||
|
let bitmapInfo = CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue
|
||||||
|
guard let context = CGContext(data: &pixels, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4,
|
||||||
|
space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: bitmapInfo),
|
||||||
|
let image = context.makeImage(),
|
||||||
|
let destination = CGImageDestinationCreateWithURL(url as CFURL, UTType.png.identifier as CFString, 1, nil)
|
||||||
|
else { return }
|
||||||
|
CGImageDestinationAddImage(destination, image, nil)
|
||||||
|
CGImageDestinationFinalize(destination)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes frames to an H.264 .mp4 file.
|
||||||
|
final class Recorder {
|
||||||
|
let url: URL
|
||||||
|
private let writer: AVAssetWriter
|
||||||
|
private let input: AVAssetWriterInput
|
||||||
|
private let adaptor: AVAssetWriterInputPixelBufferAdaptor
|
||||||
|
private let width: Int, height: Int
|
||||||
|
private var startTime: Double?
|
||||||
|
|
||||||
|
init(url: URL, width: Int, height: Int) throws {
|
||||||
|
self.url = url
|
||||||
|
self.width = width
|
||||||
|
self.height = height
|
||||||
|
writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
|
||||||
|
input = AVAssetWriterInput(mediaType: .video, outputSettings: [
|
||||||
|
AVVideoCodecKey: AVVideoCodecType.h264,
|
||||||
|
AVVideoWidthKey: width,
|
||||||
|
AVVideoHeightKey: height,
|
||||||
|
])
|
||||||
|
input.expectsMediaDataInRealTime = true
|
||||||
|
adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [
|
||||||
|
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||||
|
kCVPixelBufferWidthKey as String: width,
|
||||||
|
kCVPixelBufferHeightKey as String: height,
|
||||||
|
])
|
||||||
|
writer.add(input)
|
||||||
|
guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) }
|
||||||
|
writer.startSession(atSourceTime: .zero)
|
||||||
|
}
|
||||||
|
|
||||||
|
func append(bgra pixels: [UInt8], at time: Double) {
|
||||||
|
guard input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return }
|
||||||
|
var created: CVPixelBuffer?
|
||||||
|
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created)
|
||||||
|
guard let buffer = created else { return }
|
||||||
|
|
||||||
|
CVPixelBufferLockBaseAddress(buffer, [])
|
||||||
|
if let base = CVPixelBufferGetBaseAddress(buffer) {
|
||||||
|
let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
|
||||||
|
pixels.withUnsafeBytes { source in
|
||||||
|
for y in 0..<height {
|
||||||
|
memcpy(base + y * rowBytes, source.baseAddress! + y * width * 4, width * 4)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CVPixelBufferUnlockBaseAddress(buffer, [])
|
||||||
|
|
||||||
|
if startTime == nil { startTime = time }
|
||||||
|
adaptor.append(buffer, withPresentationTime: CMTime(seconds: time - startTime!, preferredTimescale: 600))
|
||||||
|
}
|
||||||
|
|
||||||
|
func finish() {
|
||||||
|
input.markAsFinished()
|
||||||
|
let done = DispatchSemaphore(value: 0)
|
||||||
|
writer.finishWriting { done.signal() }
|
||||||
|
done.wait()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct VirtualCameraError: Error, CustomStringConvertible {
|
||||||
|
let description: String
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Feeds frames into OBS's virtual camera, so Meet, Zoom and anything else that takes a webcam can use them.
|
||||||
|
/// OBS itself must not be running its own virtual camera at the same time.
|
||||||
|
final class VirtualCamera {
|
||||||
|
// fixed identifier of the camera OBS installs (from obs-studio's mac-virtualcam plugin)
|
||||||
|
private static let obsDeviceUID = "7626645E-4425-469E-9D8B-97E0FA59AC75"
|
||||||
|
|
||||||
|
private let deviceID: CMIODeviceID
|
||||||
|
private let streamID: CMIOStreamID
|
||||||
|
private let queue: CMSimpleQueue
|
||||||
|
private let format: CMFormatDescription
|
||||||
|
private let pool: CVPixelBufferPool
|
||||||
|
|
||||||
|
init(width: Int, height: Int) throws {
|
||||||
|
func address(_ selector: Int) -> CMIOObjectPropertyAddress {
|
||||||
|
CMIOObjectPropertyAddress(mSelector: CMIOObjectPropertySelector(selector),
|
||||||
|
mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
|
||||||
|
mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain))
|
||||||
|
}
|
||||||
|
func objects(of object: CMIOObjectID, _ selector: Int) -> [CMIOObjectID] {
|
||||||
|
var property = address(selector)
|
||||||
|
var size: UInt32 = 0, used: UInt32 = 0
|
||||||
|
CMIOObjectGetPropertyDataSize(object, &property, 0, nil, &size)
|
||||||
|
var ids = [CMIOObjectID](repeating: 0, count: Int(size) / MemoryLayout<CMIOObjectID>.size)
|
||||||
|
CMIOObjectGetPropertyData(object, &property, 0, nil, size, &used, &ids)
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
func uid(of device: CMIOObjectID) -> String? {
|
||||||
|
var property = address(kCMIODevicePropertyDeviceUID)
|
||||||
|
var value: Unmanaged<CFString>?
|
||||||
|
var used: UInt32 = 0
|
||||||
|
CMIOObjectGetPropertyData(device, &property, 0, nil, UInt32(MemoryLayout<Unmanaged<CFString>?>.size), &used, &value)
|
||||||
|
return value?.takeRetainedValue() as String?
|
||||||
|
}
|
||||||
|
|
||||||
|
let devices = objects(of: CMIOObjectID(kCMIOObjectSystemObject), kCMIOHardwarePropertyDevices)
|
||||||
|
guard let device = devices.first(where: { uid(of: $0) == VirtualCamera.obsDeviceUID }) else {
|
||||||
|
throw VirtualCameraError(description: """
|
||||||
|
OBS Virtual Camera not found. One-time setup: enable OBS under System Settings > General > \
|
||||||
|
Login Items & Extensions > Camera Extensions, then in OBS click Start Virtual Camera, Stop Virtual Camera, and quit OBS.
|
||||||
|
""")
|
||||||
|
}
|
||||||
|
// the camera has two streams: [0] is what apps watch, [1] is where frames are fed in
|
||||||
|
let streams = objects(of: device, kCMIODevicePropertyStreams)
|
||||||
|
guard streams.count >= 2 else { throw VirtualCameraError(description: "OBS Virtual Camera has no input stream.") }
|
||||||
|
deviceID = device
|
||||||
|
streamID = streams[1]
|
||||||
|
|
||||||
|
var createdQueue: Unmanaged<CMSimpleQueue>?
|
||||||
|
CMIOStreamCopyBufferQueue(streamID, { _, _, _ in }, nil, &createdQueue)
|
||||||
|
var createdFormat: CMFormatDescription?
|
||||||
|
CMVideoFormatDescriptionCreate(allocator: nil, codecType: kCVPixelFormatType_422YpCbCr8,
|
||||||
|
width: Int32(width), height: Int32(height), extensions: nil,
|
||||||
|
formatDescriptionOut: &createdFormat)
|
||||||
|
var createdPool: CVPixelBufferPool?
|
||||||
|
CVPixelBufferPoolCreate(nil, nil, [
|
||||||
|
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_422YpCbCr8,
|
||||||
|
kCVPixelBufferWidthKey: width,
|
||||||
|
kCVPixelBufferHeightKey: height,
|
||||||
|
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary,
|
||||||
|
] as CFDictionary, &createdPool)
|
||||||
|
guard let createdQueue, let createdFormat, let createdPool else {
|
||||||
|
throw VirtualCameraError(description: "Could not set up the OBS Virtual Camera stream.")
|
||||||
|
}
|
||||||
|
queue = createdQueue.takeRetainedValue()
|
||||||
|
format = createdFormat
|
||||||
|
pool = createdPool
|
||||||
|
guard CMIODeviceStartStream(deviceID, streamID) == noErr else {
|
||||||
|
throw VirtualCameraError(description: "Could not start the OBS Virtual Camera stream (is OBS using it?).")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
deinit {
|
||||||
|
CMIODeviceStopStream(deviceID, streamID)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sends one frame. The picture is flipped back left-to-right: the preview is a mirror,
|
||||||
|
/// but the people on the call should see the real thing.
|
||||||
|
func send(bgra pixels: [UInt8], width: Int, height: Int) {
|
||||||
|
var created: CVPixelBuffer?
|
||||||
|
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created)
|
||||||
|
guard let buffer = created else { return }
|
||||||
|
|
||||||
|
CVPixelBufferLockBaseAddress(buffer, [])
|
||||||
|
if let base = CVPixelBufferGetBaseAddress(buffer)?.assumingMemoryBound(to: UInt8.self) {
|
||||||
|
let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
|
||||||
|
pixels.withUnsafeBufferPointer { source in
|
||||||
|
// BGRA -> UYVY (BT.601 video range): two pixels share one U and one V
|
||||||
|
for y in 0..<height {
|
||||||
|
let row = base + y * rowBytes
|
||||||
|
for pair in 0..<(width / 2) {
|
||||||
|
let first = (y * width + (width - 1 - pair * 2)) * 4
|
||||||
|
let second = first - 4
|
||||||
|
let b0 = Int(source[first]), g0 = Int(source[first + 1]), r0 = Int(source[first + 2])
|
||||||
|
let b1 = Int(source[second]), g1 = Int(source[second + 1]), r1 = Int(source[second + 2])
|
||||||
|
row[pair * 4] = UInt8(clamping: ((-38 * r0 - 74 * g0 + 112 * b0 + 128) >> 8) + 128)
|
||||||
|
row[pair * 4 + 1] = UInt8(clamping: ((66 * r0 + 129 * g0 + 25 * b0 + 128) >> 8) + 16)
|
||||||
|
row[pair * 4 + 2] = UInt8(clamping: ((112 * r0 - 94 * g0 - 18 * b0 + 128) >> 8) + 128)
|
||||||
|
row[pair * 4 + 3] = UInt8(clamping: ((66 * r1 + 129 * g1 + 25 * b1 + 128) >> 8) + 16)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CVPixelBufferUnlockBaseAddress(buffer, [])
|
||||||
|
|
||||||
|
var timing = CMSampleTimingInfo(
|
||||||
|
duration: .invalid,
|
||||||
|
presentationTimeStamp: CMTime(value: CMTimeValue(clock_gettime_nsec_np(CLOCK_UPTIME_RAW)), timescale: 1_000_000_000),
|
||||||
|
decodeTimeStamp: .invalid)
|
||||||
|
var sample: CMSampleBuffer?
|
||||||
|
CMSampleBufferCreateForImageBuffer(allocator: nil, imageBuffer: buffer, dataReady: true, makeDataReadyCallback: nil,
|
||||||
|
refcon: nil, formatDescription: format, sampleTiming: &timing, sampleBufferOut: &sample)
|
||||||
|
if let sample {
|
||||||
|
// the queue takes ownership of the sample
|
||||||
|
if CMSimpleQueueEnqueue(queue, element: Unmanaged.passRetained(sample).toOpaque()) != noErr {
|
||||||
|
Unmanaged.passUnretained(sample).release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
92
native/Sources/MotionLab/Panel.swift
Normal file
92
native/Sources/MotionLab/Panel.swift
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
import AppKit
|
||||||
|
|
||||||
|
private final class TopAlignedView: NSView {
|
||||||
|
override var isFlipped: Bool { true } // so the scroll view starts at the top
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Side panel: every effect with a checkbox, and the sliders of the ones that are switched on.
|
||||||
|
final class Panel: NSObject {
|
||||||
|
static let width: CGFloat = 230
|
||||||
|
|
||||||
|
let view = NSScrollView()
|
||||||
|
private let stack = NSStackView()
|
||||||
|
private let statusLabel = NSTextField(labelWithString: "")
|
||||||
|
private let renderer: Renderer
|
||||||
|
|
||||||
|
init(renderer: Renderer) {
|
||||||
|
self.renderer = renderer
|
||||||
|
super.init()
|
||||||
|
stack.orientation = .vertical
|
||||||
|
stack.alignment = .leading
|
||||||
|
stack.spacing = 4
|
||||||
|
stack.translatesAutoresizingMaskIntoConstraints = false
|
||||||
|
|
||||||
|
let document = TopAlignedView()
|
||||||
|
document.translatesAutoresizingMaskIntoConstraints = false
|
||||||
|
document.addSubview(stack)
|
||||||
|
view.documentView = document
|
||||||
|
view.hasVerticalScroller = true
|
||||||
|
view.drawsBackground = false
|
||||||
|
NSLayoutConstraint.activate([
|
||||||
|
view.widthAnchor.constraint(equalToConstant: Panel.width),
|
||||||
|
document.widthAnchor.constraint(equalTo: view.contentView.widthAnchor),
|
||||||
|
stack.topAnchor.constraint(equalTo: document.topAnchor, constant: 14),
|
||||||
|
stack.leadingAnchor.constraint(equalTo: document.leadingAnchor, constant: 14),
|
||||||
|
stack.trailingAnchor.constraint(equalTo: document.trailingAnchor, constant: -14),
|
||||||
|
stack.bottomAnchor.constraint(equalTo: document.bottomAnchor, constant: -14),
|
||||||
|
])
|
||||||
|
statusLabel.font = .boldSystemFont(ofSize: 11)
|
||||||
|
statusLabel.textColor = .systemRed
|
||||||
|
rebuild()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func label(_ text: String, dim: Bool = true) -> NSTextField {
|
||||||
|
let field = NSTextField(labelWithString: text)
|
||||||
|
field.font = .systemFont(ofSize: 11)
|
||||||
|
field.textColor = dim ? .secondaryLabelColor : .labelColor
|
||||||
|
return field
|
||||||
|
}
|
||||||
|
|
||||||
|
func rebuild() {
|
||||||
|
stack.arrangedSubviews.forEach { $0.removeFromSuperview() }
|
||||||
|
statusLabel.stringValue = renderer.status
|
||||||
|
stack.addArrangedSubview(statusLabel)
|
||||||
|
|
||||||
|
for (layerIndex, layer) in renderer.layers.enumerated() {
|
||||||
|
let title = layer.error == nil ? layer.name : "\(layer.name) (shader error)"
|
||||||
|
let checkbox = NSButton(checkboxWithTitle: title, target: self, action: #selector(toggled(_:)))
|
||||||
|
checkbox.state = layer.enabled ? .on : .off
|
||||||
|
checkbox.tag = layerIndex
|
||||||
|
checkbox.font = .systemFont(ofSize: 13, weight: .medium)
|
||||||
|
stack.addArrangedSubview(checkbox)
|
||||||
|
guard layer.enabled else { continue }
|
||||||
|
|
||||||
|
for (paramIndex, param) in layer.params.enumerated() {
|
||||||
|
let slider = NSSlider(value: Double(param.value), minValue: Double(param.min), maxValue: Double(param.max),
|
||||||
|
target: self, action: #selector(sliderMoved(_:)))
|
||||||
|
slider.tag = layerIndex * maxParams + paramIndex
|
||||||
|
slider.controlSize = .small
|
||||||
|
stack.addArrangedSubview(label(param.name))
|
||||||
|
stack.addArrangedSubview(slider)
|
||||||
|
slider.widthAnchor.constraint(equalTo: stack.widthAnchor).isActive = true
|
||||||
|
}
|
||||||
|
stack.setCustomSpacing(12, after: stack.arrangedSubviews.last!)
|
||||||
|
}
|
||||||
|
if renderer.layers.isEmpty { stack.addArrangedSubview(label("no .metal files found")) }
|
||||||
|
|
||||||
|
stack.setCustomSpacing(16, after: stack.arrangedSubviews.last!)
|
||||||
|
stack.addArrangedSubview(label("s screenshot\nr record\nq quit"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func refreshStatus() {
|
||||||
|
statusLabel.stringValue = renderer.status
|
||||||
|
}
|
||||||
|
|
||||||
|
@objc private func toggled(_ checkbox: NSButton) {
|
||||||
|
renderer.setEnabled(checkbox.tag, checkbox.state == .on)
|
||||||
|
}
|
||||||
|
|
||||||
|
@objc private func sliderMoved(_ slider: NSSlider) {
|
||||||
|
renderer.setParam(layer: slider.tag / maxParams, param: slider.tag % maxParams, value: Float(slider.doubleValue))
|
||||||
|
}
|
||||||
|
}
|
||||||
287
native/Sources/MotionLab/Renderer.swift
Normal file
287
native/Sources/MotionLab/Renderer.swift
Normal file
|
|
@ -0,0 +1,287 @@
|
||||||
|
import MetalKit
|
||||||
|
|
||||||
|
/// One effect file in the chain.
|
||||||
|
final class Layer {
|
||||||
|
let url: URL
|
||||||
|
let name: String
|
||||||
|
var enabled = false
|
||||||
|
var order = 50
|
||||||
|
var params: [EffectParam] = []
|
||||||
|
var error: String?
|
||||||
|
fileprivate var pipeline: MTLRenderPipelineState?
|
||||||
|
fileprivate var modified: Date?
|
||||||
|
fileprivate var usesParticles = false
|
||||||
|
fileprivate var targets: [MTLTexture] = [] // [this frame, previous frame], swapped after every draw
|
||||||
|
|
||||||
|
init(url: URL) {
|
||||||
|
self.url = url
|
||||||
|
self.name = url.deletingPathExtension().lastPathComponent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private let passthrough = "float4 effect(float2 uv, Inputs in) { return in.image.sample(in.smp, uv); }"
|
||||||
|
|
||||||
|
/// Runs every enabled effect in order on each new camera frame, shows the result in the window,
|
||||||
|
/// and hands it to whatever outputs are active (recording, virtual camera, screenshots).
|
||||||
|
final class Renderer: NSObject, MTKViewDelegate {
|
||||||
|
private let device: MTLDevice
|
||||||
|
private let queue: MTLCommandQueue
|
||||||
|
private let camera: Camera
|
||||||
|
private let effectsDirectory: URL
|
||||||
|
private let copyPipeline: MTLRenderPipelineState
|
||||||
|
private var analysis: Analysis?
|
||||||
|
private var plain: MTLTexture? // holds the camera picture when no effect is enabled
|
||||||
|
private var result: MTLTexture? // the finished frame
|
||||||
|
private var lastSerial = -1
|
||||||
|
private let startTime = CACurrentMediaTime()
|
||||||
|
private var frameCount = 0
|
||||||
|
private var pixels: [UInt8] = []
|
||||||
|
|
||||||
|
private(set) var layers: [Layer] = []
|
||||||
|
/// Called when layers were added, removed, reloaded or toggled.
|
||||||
|
var onLayersChanged: (() -> Void)?
|
||||||
|
/// Called when recording or the virtual camera starts or stops.
|
||||||
|
var onStatusChanged: (() -> Void)?
|
||||||
|
|
||||||
|
/// When set, the app exits after this many camera frames, first saving that frame as a PNG if a url is given
|
||||||
|
/// (used to check effects without watching).
|
||||||
|
var snapshot: (frames: Int, url: URL?)?
|
||||||
|
/// When set, recording into this file starts with the first frame.
|
||||||
|
var recordURL: URL?
|
||||||
|
/// When true, the OBS virtual camera is opened on the first frame, at the size of the output.
|
||||||
|
var wantsVirtualCamera = false
|
||||||
|
private var virtualCamera: VirtualCamera? { didSet { onStatusChanged?() } }
|
||||||
|
private var recorder: Recorder? { didSet { onStatusChanged?() } }
|
||||||
|
private var screenshotRequested = false
|
||||||
|
|
||||||
|
var status: String {
|
||||||
|
[recorder != nil ? "REC" : nil, virtualCamera != nil ? "CAM" : nil].compactMap { $0 }.joined(separator: " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
init(device: MTLDevice, camera: Camera, effectsDirectory: URL) throws {
|
||||||
|
self.device = device
|
||||||
|
self.camera = camera
|
||||||
|
self.effectsDirectory = effectsDirectory
|
||||||
|
self.queue = device.makeCommandQueue()!
|
||||||
|
self.copyPipeline = try makePipeline(device, source: effectShader(body: passthrough, params: []),
|
||||||
|
fragment: "effectFragment", format: .bgra8Unorm)
|
||||||
|
super.init()
|
||||||
|
refreshFiles()
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: effect files
|
||||||
|
|
||||||
|
/// Picks up new, deleted and edited effect files. On a compile error the last working version keeps running.
|
||||||
|
private func refreshFiles() {
|
||||||
|
let files = ((try? FileManager.default.contentsOfDirectory(at: effectsDirectory, includingPropertiesForKeys: nil)) ?? [])
|
||||||
|
.filter { $0.pathExtension == "metal" }
|
||||||
|
var changed = false
|
||||||
|
|
||||||
|
let before = layers.count
|
||||||
|
layers.removeAll { layer in !files.contains(layer.url) }
|
||||||
|
changed = layers.count != before
|
||||||
|
for url in files where !layers.contains(where: { $0.url == url }) {
|
||||||
|
layers.append(Layer(url: url))
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
for layer in layers where reload(layer) { changed = true }
|
||||||
|
|
||||||
|
if changed {
|
||||||
|
layers.sort { ($0.order, $0.name) < ($1.order, $1.name) }
|
||||||
|
onLayersChanged?()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func reload(_ layer: Layer) -> Bool {
|
||||||
|
let modified = (try? layer.url.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate
|
||||||
|
guard let modified, modified != layer.modified else { return false }
|
||||||
|
let firstLoad = layer.modified == nil
|
||||||
|
layer.modified = modified
|
||||||
|
do {
|
||||||
|
let source = try String(contentsOf: layer.url, encoding: .utf8)
|
||||||
|
var info = parseEffect(source)
|
||||||
|
layer.pipeline = try makePipeline(device, source: effectShader(body: source, params: info.params),
|
||||||
|
fragment: "effectFragment", format: .bgra8Unorm)
|
||||||
|
for i in info.params.indices { // keep slider positions across reloads
|
||||||
|
if let old = layer.params.first(where: { $0.name == info.params[i].name }) {
|
||||||
|
info.params[i].value = min(max(old.value, info.params[i].min), info.params[i].max)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
layer.params = info.params
|
||||||
|
layer.order = info.order
|
||||||
|
layer.usesParticles = source.contains("in.particles")
|
||||||
|
layer.error = nil
|
||||||
|
if firstLoad { layer.enabled = info.startsOn }
|
||||||
|
print("[\(layer.name)] loaded")
|
||||||
|
} catch {
|
||||||
|
layer.error = error.localizedDescription
|
||||||
|
print("[\(layer.name)] shader error:\n\(error.localizedDescription)")
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
var failedLayers: [String] { layers.filter { $0.error != nil }.map(\.name) }
|
||||||
|
|
||||||
|
/// Enables exactly the named layers. Returns the names that don't exist.
|
||||||
|
func enableOnly(_ names: [String]) -> [String] {
|
||||||
|
for layer in layers { layer.enabled = names.contains(layer.name) }
|
||||||
|
return names.filter { name in !layers.contains { $0.name == name } }
|
||||||
|
}
|
||||||
|
|
||||||
|
func setEnabled(_ index: Int, _ enabled: Bool) {
|
||||||
|
guard layers.indices.contains(index) else { return }
|
||||||
|
layers[index].enabled = enabled
|
||||||
|
if !enabled { layers[index].targets = [] } // free its memory, and start clean next time
|
||||||
|
onLayersChanged?()
|
||||||
|
}
|
||||||
|
|
||||||
|
func setParam(layer: Int, param: Int, value: Float) {
|
||||||
|
guard layers.indices.contains(layer), layers[layer].params.indices.contains(param) else { return }
|
||||||
|
let p = layers[layer].params[param]
|
||||||
|
layers[layer].params[param].value = min(max(value, p.min), p.max)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: outputs
|
||||||
|
|
||||||
|
func requestScreenshot() { screenshotRequested = true }
|
||||||
|
|
||||||
|
func toggleRecording() {
|
||||||
|
if let recorder {
|
||||||
|
self.recorder = nil
|
||||||
|
recorder.finish()
|
||||||
|
print("saved \(recorder.url.path)")
|
||||||
|
} else if let result {
|
||||||
|
do {
|
||||||
|
recorder = try Recorder(url: capturesURL(extension: "mp4"), width: result.width, height: result.height)
|
||||||
|
print("recording \(recorder!.url.path)")
|
||||||
|
} catch {
|
||||||
|
print("could not start recording: \(error.localizedDescription)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func finish() {
|
||||||
|
if recorder != nil { toggleRecording() }
|
||||||
|
}
|
||||||
|
|
||||||
|
private func capturesURL(extension ext: String) -> URL {
|
||||||
|
let directory = URL(fileURLWithPath: "captures")
|
||||||
|
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
||||||
|
let stamp = DateFormatter()
|
||||||
|
stamp.dateFormat = "yyyyMMdd-HHmmss"
|
||||||
|
return directory.appendingPathComponent("motionlab-\(stamp.string(from: Date())).\(ext)")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs once per camera frame, after the GPU work was submitted.
|
||||||
|
private func deliver(_ commands: MTLCommandBuffer) {
|
||||||
|
frameCount += 1
|
||||||
|
let snapshotDue = snapshot.map { frameCount >= $0.frames } ?? false
|
||||||
|
guard let result else { return }
|
||||||
|
let width = result.width, height = result.height
|
||||||
|
if let url = recordURL {
|
||||||
|
recordURL = nil
|
||||||
|
do { recorder = try Recorder(url: url, width: width, height: height) } catch { print("could not start recording: \(error.localizedDescription)") }
|
||||||
|
}
|
||||||
|
if wantsVirtualCamera {
|
||||||
|
wantsVirtualCamera = false
|
||||||
|
do {
|
||||||
|
virtualCamera = try VirtualCamera(width: width, height: height)
|
||||||
|
print("virtual camera on: pick 'OBS Virtual Camera' as the camera in your call or streaming app")
|
||||||
|
} catch {
|
||||||
|
print("virtual camera off: \(error)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
guard recorder != nil || virtualCamera != nil || screenshotRequested || snapshotDue else { return }
|
||||||
|
|
||||||
|
commands.waitUntilCompleted()
|
||||||
|
if pixels.count != width * height * 4 { pixels = [UInt8](repeating: 0, count: width * height * 4) }
|
||||||
|
result.getBytes(&pixels, bytesPerRow: width * 4, from: MTLRegionMake2D(0, 0, width, height), mipmapLevel: 0)
|
||||||
|
|
||||||
|
recorder?.append(bgra: pixels, at: CACurrentMediaTime())
|
||||||
|
virtualCamera?.send(bgra: pixels, width: width, height: height)
|
||||||
|
if screenshotRequested {
|
||||||
|
screenshotRequested = false
|
||||||
|
let url = capturesURL(extension: "png")
|
||||||
|
writePNG(bgra: &pixels, width: width, height: height, to: url)
|
||||||
|
print("saved \(url.path)")
|
||||||
|
}
|
||||||
|
if snapshotDue {
|
||||||
|
if let url = snapshot?.url {
|
||||||
|
writePNG(bgra: &pixels, width: width, height: height, to: url)
|
||||||
|
print("\(frameCount) frames -> \(url.path)")
|
||||||
|
}
|
||||||
|
finish()
|
||||||
|
exit(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: drawing
|
||||||
|
|
||||||
|
private func process(_ frames: (current: MTLTexture, last: MTLTexture, serial: Int), _ commands: MTLCommandBuffer) throws {
|
||||||
|
let width = frames.current.width, height = frames.current.height
|
||||||
|
if analysis == nil { analysis = try Analysis(device: device, queue: queue, width: width, height: height) }
|
||||||
|
guard let analysis else { return }
|
||||||
|
analysis.update(commands, camera: frames.current, particles: layers.contains { $0.enabled && $0.usesParticles })
|
||||||
|
|
||||||
|
let time = Float(CACurrentMediaTime() - startTime)
|
||||||
|
var image = frames.current
|
||||||
|
for layer in layers where layer.enabled {
|
||||||
|
guard let pipeline = layer.pipeline else { continue }
|
||||||
|
if layer.targets.first?.width != width || layer.targets.first?.height != height {
|
||||||
|
layer.targets = [makeTarget(device, width: width, height: height), makeTarget(device, width: width, height: height)]
|
||||||
|
for target in layer.targets { commands.pass(into: target, clear: true) }
|
||||||
|
}
|
||||||
|
// must match `struct Uniforms` in the shader: time, padding, resolution.xy, params[16]
|
||||||
|
var uniforms: [Float] = [time, 0, Float(width), Float(height)]
|
||||||
|
uniforms += layer.params.map(\.value) + [Float](repeating: 0, count: maxParams - layer.params.count)
|
||||||
|
let textures = [image, frames.current, frames.last, layer.targets[1], analysis.flow, analysis.particles]
|
||||||
|
commands.pass(into: layer.targets[0]) { $0.fullscreen(pipeline, textures: textures, uniforms: uniforms) }
|
||||||
|
image = layer.targets[0]
|
||||||
|
layer.targets.swapAt(0, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if image === frames.current {
|
||||||
|
if plain?.width != width || plain?.height != height { plain = makeTarget(device, width: width, height: height) }
|
||||||
|
commands.pass(into: plain!) { $0.fullscreen(copyPipeline, textures: [image], uniforms: [0, 0, 0, 0]) }
|
||||||
|
image = plain!
|
||||||
|
}
|
||||||
|
result = image
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The largest rectangle with the output's aspect ratio that fits the window, centred.
|
||||||
|
private func fitted(_ output: MTLTexture, in size: CGSize) -> MTLViewport {
|
||||||
|
let scale = min(size.width / Double(output.width), size.height / Double(output.height))
|
||||||
|
let width = Double(output.width) * scale, height = Double(output.height) * scale
|
||||||
|
return MTLViewport(originX: (size.width - width) / 2, originY: (size.height - height) / 2,
|
||||||
|
width: width, height: height, znear: 0, zfar: 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {}
|
||||||
|
|
||||||
|
func draw(in view: MTKView) {
|
||||||
|
guard let frames = camera.latest, let commands = queue.makeCommandBuffer() else { return }
|
||||||
|
|
||||||
|
// effects advance once per camera frame; in between, the window just shows the last result again
|
||||||
|
let isNewFrame = frames.serial != lastSerial
|
||||||
|
if isNewFrame {
|
||||||
|
lastSerial = frames.serial
|
||||||
|
if frameCount % 15 == 0 { refreshFiles() }
|
||||||
|
do {
|
||||||
|
try process(frames, commands)
|
||||||
|
} catch {
|
||||||
|
print("render setup failed: \(error.localizedDescription)")
|
||||||
|
exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let result, let drawable = view.currentDrawable, let windowPass = view.currentRenderPassDescriptor,
|
||||||
|
let encoder = commands.makeRenderCommandEncoder(descriptor: windowPass) {
|
||||||
|
encoder.setViewport(fitted(result, in: view.drawableSize))
|
||||||
|
encoder.fullscreen(copyPipeline, textures: [result], uniforms: [0, 0, 0, 0])
|
||||||
|
encoder.endEncoding()
|
||||||
|
commands.present(drawable)
|
||||||
|
}
|
||||||
|
commands.commit()
|
||||||
|
if isNewFrame { deliver(commands) }
|
||||||
|
}
|
||||||
|
}
|
||||||
161
native/Sources/MotionLab/Shaders.swift
Normal file
161
native/Sources/MotionLab/Shaders.swift
Normal file
|
|
@ -0,0 +1,161 @@
|
||||||
|
import Metal
|
||||||
|
|
||||||
|
/// A slider an effect file declares with a comment line: `// slider <name> <min> <max> <default>`
|
||||||
|
struct EffectParam {
|
||||||
|
let name: String
|
||||||
|
let min: Float
|
||||||
|
let max: Float
|
||||||
|
var value: Float
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Everything an effect file says about itself in `//` comment lines:
|
||||||
|
///
|
||||||
|
/// // slider <name> <min> <max> <default> a slider, read in the shader as in.p.<name>
|
||||||
|
/// // order <number> position in the chain, low runs first (default 50)
|
||||||
|
/// // on enabled when the app starts
|
||||||
|
struct EffectInfo {
|
||||||
|
var params: [EffectParam] = []
|
||||||
|
var order = 50
|
||||||
|
var startsOn = false
|
||||||
|
}
|
||||||
|
|
||||||
|
let maxParams = 16
|
||||||
|
|
||||||
|
func parseEffect(_ source: String) -> EffectInfo {
|
||||||
|
var info = EffectInfo()
|
||||||
|
for line in source.split(separator: "\n") {
|
||||||
|
let parts = line.split(separator: " ")
|
||||||
|
guard parts.count >= 2, parts[0] == "//" else { continue }
|
||||||
|
switch parts[1] {
|
||||||
|
case "slider" where parts.count == 6:
|
||||||
|
if let min = Float(parts[3]), let max = Float(parts[4]), let value = Float(parts[5]) {
|
||||||
|
info.params.append(EffectParam(name: String(parts[2]), min: min, max: max, value: value))
|
||||||
|
}
|
||||||
|
case "order" where parts.count == 3:
|
||||||
|
info.order = Int(parts[2]) ?? info.order
|
||||||
|
case "on" where parts.count == 2:
|
||||||
|
info.startsOn = true
|
||||||
|
default:
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
info.params = Array(info.params.prefix(maxParams))
|
||||||
|
return info
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared by every shader in the app: a triangle that covers the whole target.
|
||||||
|
let shaderPrelude = """
|
||||||
|
#include <metal_stdlib>
|
||||||
|
using namespace metal;
|
||||||
|
|
||||||
|
struct VertexOut {
|
||||||
|
float4 position [[position]];
|
||||||
|
float2 uv;
|
||||||
|
};
|
||||||
|
|
||||||
|
vertex VertexOut fullscreenVertex(uint id [[vertex_id]]) {
|
||||||
|
float2 p = float2((id << 1) & 2, id & 2);
|
||||||
|
VertexOut out;
|
||||||
|
out.position = float4(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0, 0.0, 1.0);
|
||||||
|
out.uv = p;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
/// Wraps an effect file into a complete Metal shader. An effect file only defines:
|
||||||
|
///
|
||||||
|
/// float4 effect(float2 uv, Inputs in) { ... }
|
||||||
|
///
|
||||||
|
/// `uv` runs from (0,0) top-left to (1,1) bottom-right.
|
||||||
|
func effectShader(body: String, params: [EffectParam]) -> String {
|
||||||
|
let fields = params.isEmpty ? "float unused;" : params.map { "float \($0.name);" }.joined(separator: " ")
|
||||||
|
let values = params.isEmpty ? "0.0" : params.indices.map { "u.params[\($0)]" }.joined(separator: ", ")
|
||||||
|
return shaderPrelude + """
|
||||||
|
struct Uniforms {
|
||||||
|
float time;
|
||||||
|
float2 resolution;
|
||||||
|
float params[\(maxParams)];
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Params { \(fields) };
|
||||||
|
|
||||||
|
struct Inputs {
|
||||||
|
texture2d<float> image; // the picture so far: output of the effect above, or the camera for the first one
|
||||||
|
texture2d<float> camera; // the untouched camera frame
|
||||||
|
texture2d<float> last; // the camera frame before this one (compare with camera to find motion)
|
||||||
|
texture2d<float> prev; // this effect's own output from the previous frame (for trails)
|
||||||
|
texture2d<float> flow; // .xy = which way and how far things moved since the last frame, in uv units
|
||||||
|
texture2d<float> particles; // .r = brightness of particles that drift along with the motion
|
||||||
|
sampler smp;
|
||||||
|
float time; // seconds since start
|
||||||
|
float2 resolution; // output size in pixels
|
||||||
|
Params p; // this effect's sliders
|
||||||
|
};
|
||||||
|
|
||||||
|
#line 1
|
||||||
|
\(body)
|
||||||
|
|
||||||
|
fragment float4 effectFragment(VertexOut v [[stage_in]],
|
||||||
|
texture2d<float> image [[texture(0)]],
|
||||||
|
texture2d<float> camera [[texture(1)]],
|
||||||
|
texture2d<float> last [[texture(2)]],
|
||||||
|
texture2d<float> prev [[texture(3)]],
|
||||||
|
texture2d<float> flow [[texture(4)]],
|
||||||
|
texture2d<float> particles [[texture(5)]],
|
||||||
|
constant Uniforms &u [[buffer(0)]]) {
|
||||||
|
constexpr sampler smp(filter::linear, address::clamp_to_edge);
|
||||||
|
Params p = { \(values) };
|
||||||
|
Inputs in = { image, camera, last, prev, flow, particles, smp, u.time, u.resolution, p };
|
||||||
|
return effect(v.uv, in);
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
}
|
||||||
|
|
||||||
|
func makePipeline(_ device: MTLDevice, source: String, vertex: String = "fullscreenVertex", fragment: String,
|
||||||
|
format: MTLPixelFormat, additive: Bool = false) throws -> MTLRenderPipelineState {
|
||||||
|
let library = try device.makeLibrary(source: source, options: nil)
|
||||||
|
let descriptor = MTLRenderPipelineDescriptor()
|
||||||
|
descriptor.vertexFunction = library.makeFunction(name: vertex)
|
||||||
|
descriptor.fragmentFunction = library.makeFunction(name: fragment)
|
||||||
|
let attachment = descriptor.colorAttachments[0]!
|
||||||
|
attachment.pixelFormat = format
|
||||||
|
if additive {
|
||||||
|
attachment.isBlendingEnabled = true
|
||||||
|
attachment.sourceRGBBlendFactor = .one
|
||||||
|
attachment.destinationRGBBlendFactor = .one
|
||||||
|
attachment.sourceAlphaBlendFactor = .one
|
||||||
|
attachment.destinationAlphaBlendFactor = .one
|
||||||
|
}
|
||||||
|
return try device.makeRenderPipelineState(descriptor: descriptor)
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeTarget(_ device: MTLDevice, width: Int, height: Int, format: MTLPixelFormat = .bgra8Unorm) -> MTLTexture {
|
||||||
|
let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width, height: height, mipmapped: false)
|
||||||
|
descriptor.usage = [.renderTarget, .shaderRead]
|
||||||
|
return device.makeTexture(descriptor: descriptor)!
|
||||||
|
}
|
||||||
|
|
||||||
|
extension MTLCommandBuffer {
|
||||||
|
/// One render pass into `target`. With `clear`, the target starts black; otherwise its old contents are ignored.
|
||||||
|
func pass(into target: MTLTexture, clear: Bool = false, _ body: (MTLRenderCommandEncoder) -> Void = { _ in }) {
|
||||||
|
let descriptor = MTLRenderPassDescriptor()
|
||||||
|
descriptor.colorAttachments[0].texture = target
|
||||||
|
descriptor.colorAttachments[0].loadAction = clear ? .clear : .dontCare
|
||||||
|
descriptor.colorAttachments[0].storeAction = .store
|
||||||
|
guard let encoder = makeRenderCommandEncoder(descriptor: descriptor) else { return }
|
||||||
|
body(encoder)
|
||||||
|
encoder.endEncoding()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
extension MTLRenderCommandEncoder {
|
||||||
|
func fullscreen(_ pipeline: MTLRenderPipelineState, textures: [MTLTexture], uniforms: [Float] = []) {
|
||||||
|
setRenderPipelineState(pipeline)
|
||||||
|
setFragmentTextures(textures, range: 0..<textures.count)
|
||||||
|
if !uniforms.isEmpty {
|
||||||
|
setFragmentBytes(uniforms, length: uniforms.count * MemoryLayout<Float>.stride, index: 0)
|
||||||
|
}
|
||||||
|
drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3)
|
||||||
|
}
|
||||||
|
}
|
||||||
110
native/Sources/MotionLab/main.swift
Normal file
110
native/Sources/MotionLab/main.swift
Normal file
|
|
@ -0,0 +1,110 @@
|
||||||
|
// MotionLab: camera -> chain of Metal shader effects -> window, recording, virtual camera.
|
||||||
|
//
|
||||||
|
// swift run -c release MotionLab [options]
|
||||||
|
//
|
||||||
|
// --effects DIR folder of .metal effect files (default ./effects)
|
||||||
|
// --only a,b,c start with exactly these effects enabled
|
||||||
|
// --virtualcam also send the output to the OBS virtual camera (for Meet, Zoom, ...)
|
||||||
|
// --snapshot OUT.png save one frame after --frames N camera frames (default 90) and exit
|
||||||
|
// --record OUT.mp4 record --frames N camera frames (default 90) and exit
|
||||||
|
//
|
||||||
|
// In the window: s saves a screenshot, r starts/stops recording (both into ./captures), q or Esc quits.
|
||||||
|
// Effect files are reloaded when saved.
|
||||||
|
|
||||||
|
import AppKit
|
||||||
|
import MetalKit
|
||||||
|
|
||||||
|
func option(_ name: String) -> String? {
|
||||||
|
let arguments = CommandLine.arguments
|
||||||
|
guard let i = arguments.firstIndex(of: name), i + 1 < arguments.count else { return nil }
|
||||||
|
return arguments[i + 1]
|
||||||
|
}
|
||||||
|
|
||||||
|
func fail(_ message: String) -> Never {
|
||||||
|
FileHandle.standardError.write((message + "\n").data(using: .utf8)!)
|
||||||
|
exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
final class AppDelegate: NSObject, NSApplicationDelegate {
|
||||||
|
var onQuit: (() -> Void)?
|
||||||
|
func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
|
||||||
|
func applicationWillTerminate(_ notification: Notification) { onQuit?() }
|
||||||
|
}
|
||||||
|
|
||||||
|
setvbuf(stdout, nil, _IOLBF, 0) // show "loaded" / shader errors immediately, even when piped
|
||||||
|
|
||||||
|
guard let device = MTLCreateSystemDefaultDevice() else { fail("Metal is not available on this machine.") }
|
||||||
|
let effectsDirectory = URL(fileURLWithPath: option("--effects") ?? "effects")
|
||||||
|
|
||||||
|
let renderer: Renderer
|
||||||
|
do {
|
||||||
|
renderer = try Renderer(device: device, camera: Camera(device: device), effectsDirectory: effectsDirectory)
|
||||||
|
} catch {
|
||||||
|
fail("\(error)")
|
||||||
|
}
|
||||||
|
if renderer.layers.isEmpty { print("No .metal files in \(effectsDirectory.path) - showing the plain camera.") }
|
||||||
|
if let names = option("--only") {
|
||||||
|
let unknown = renderer.enableOnly(names.split(separator: ",").map(String.init))
|
||||||
|
if !unknown.isEmpty {
|
||||||
|
fail("Unknown effect(s): \(unknown.joined(separator: ", ")). Available: \(renderer.layers.map(\.name).joined(separator: ", "))")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if option("--snapshot") != nil || option("--record") != nil {
|
||||||
|
if !renderer.failedLayers.isEmpty { fail("shader compile failed: \(renderer.failedLayers.joined(separator: ", "))") }
|
||||||
|
renderer.snapshot = (Int(option("--frames") ?? "90") ?? 90, option("--snapshot").map { URL(fileURLWithPath: $0) })
|
||||||
|
renderer.recordURL = option("--record").map { URL(fileURLWithPath: $0) }
|
||||||
|
}
|
||||||
|
renderer.wantsVirtualCamera = CommandLine.arguments.contains("--virtualcam")
|
||||||
|
|
||||||
|
let app = NSApplication.shared
|
||||||
|
let delegate = AppDelegate()
|
||||||
|
delegate.onQuit = { renderer.finish() }
|
||||||
|
app.delegate = delegate
|
||||||
|
app.setActivationPolicy(.regular)
|
||||||
|
|
||||||
|
let size = NSSize(width: 960 + Panel.width, height: 540)
|
||||||
|
let window = NSWindow(contentRect: NSRect(origin: .zero, size: size),
|
||||||
|
styleMask: [.titled, .closable, .miniaturizable, .resizable],
|
||||||
|
backing: .buffered, defer: false)
|
||||||
|
let view = MTKView(frame: .zero, device: device)
|
||||||
|
view.colorPixelFormat = .bgra8Unorm
|
||||||
|
view.delegate = renderer
|
||||||
|
|
||||||
|
let panel = Panel(renderer: renderer)
|
||||||
|
renderer.onLayersChanged = { panel.rebuild() }
|
||||||
|
renderer.onStatusChanged = { panel.refreshStatus() }
|
||||||
|
|
||||||
|
NSEvent.addLocalMonitorForEvents(matching: .keyDown) { event in
|
||||||
|
if event.modifierFlags.contains(.command) { return event }
|
||||||
|
switch event.charactersIgnoringModifiers {
|
||||||
|
case "s": renderer.requestScreenshot()
|
||||||
|
case "r": renderer.toggleRecording()
|
||||||
|
case "q", "\u{1B}": app.terminate(nil)
|
||||||
|
default: return event
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let content = NSView(frame: NSRect(origin: .zero, size: size))
|
||||||
|
for subview in [view, panel.view] {
|
||||||
|
subview.translatesAutoresizingMaskIntoConstraints = false
|
||||||
|
content.addSubview(subview)
|
||||||
|
}
|
||||||
|
NSLayoutConstraint.activate([
|
||||||
|
view.leadingAnchor.constraint(equalTo: content.leadingAnchor),
|
||||||
|
view.topAnchor.constraint(equalTo: content.topAnchor),
|
||||||
|
view.bottomAnchor.constraint(equalTo: content.bottomAnchor),
|
||||||
|
view.trailingAnchor.constraint(equalTo: panel.view.leadingAnchor),
|
||||||
|
panel.view.trailingAnchor.constraint(equalTo: content.trailingAnchor),
|
||||||
|
panel.view.topAnchor.constraint(equalTo: content.topAnchor),
|
||||||
|
panel.view.bottomAnchor.constraint(equalTo: content.bottomAnchor),
|
||||||
|
])
|
||||||
|
|
||||||
|
window.title = "MotionLab"
|
||||||
|
window.contentMinSize = NSSize(width: 480 + Panel.width, height: 270)
|
||||||
|
window.contentView = content
|
||||||
|
window.setContentSize(size) // auto layout would otherwise pick its own window size
|
||||||
|
window.center()
|
||||||
|
window.makeKeyAndOrderFront(nil)
|
||||||
|
app.activate(ignoringOtherApps: true)
|
||||||
|
app.run()
|
||||||
12
native/effects/colorize.metal
Normal file
12
native/effects/colorize.metal
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
// Recolours the picture by brightness with a smooth colour ramp.
|
||||||
|
// order 10
|
||||||
|
// slider amount 0.0 1.0 0.8
|
||||||
|
// slider palette 0.0 1.0 0.0
|
||||||
|
// slider contrast 0.5 3.0 1.2
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float3 c = in.image.sample(in.smp, uv).rgb;
|
||||||
|
float luma = pow(dot(c, float3(0.299, 0.587, 0.114)), in.p.contrast);
|
||||||
|
float3 ramp = 0.5 + 0.5 * cos(6.2832 * (luma * 0.8 + in.p.palette + float3(0.55, 0.4, 0.2)));
|
||||||
|
float3 mapped = ramp * (0.15 + 0.85 * luma);
|
||||||
|
return float4(mix(c, mapped, in.p.amount), 1.0);
|
||||||
|
}
|
||||||
18
native/effects/halftone.metal
Normal file
18
native/effects/halftone.metal
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
// Halftone dots sized by brightness; dots turn hot orange wherever something moves.
|
||||||
|
// order 70
|
||||||
|
// slider cell 4.0 40.0 12.0
|
||||||
|
// slider heat 0.0 20.0 8.0
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float2 grid = in.resolution / in.p.cell;
|
||||||
|
float2 centre = (floor(uv * grid) + 0.5) / grid;
|
||||||
|
float3 c = in.image.sample(in.smp, centre).rgb;
|
||||||
|
float luma = dot(c, float3(0.299, 0.587, 0.114));
|
||||||
|
|
||||||
|
float radius = 0.5 * sqrt(luma);
|
||||||
|
float dist = length(fract(uv * grid) - 0.5);
|
||||||
|
float mask = 1.0 - smoothstep(radius - 0.05, radius + 0.05, dist);
|
||||||
|
|
||||||
|
float moved = length(in.camera.sample(in.smp, centre).rgb - in.last.sample(in.smp, centre).rgb);
|
||||||
|
float3 hot = mix(c, float3(1.0, 0.35, 0.1), clamp(moved * in.p.heat, 0.0, 1.0));
|
||||||
|
return float4(hot * mask, 1.0);
|
||||||
|
}
|
||||||
13
native/effects/kaleidoscope.metal
Normal file
13
native/effects/kaleidoscope.metal
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
// Folds the picture into mirrored wedges around the centre.
|
||||||
|
// order 90
|
||||||
|
// slider segments 2.0 16.0 6.0
|
||||||
|
// slider spin -2.0 2.0 0.2
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float2 shape = float2(in.resolution.x / in.resolution.y, 1.0);
|
||||||
|
float2 c = (uv - 0.5) * shape;
|
||||||
|
float wedge = 6.2832 / floor(in.p.segments);
|
||||||
|
float a = atan2(c.y, c.x) + in.time * in.p.spin;
|
||||||
|
a = abs(a - wedge * floor(a / wedge) - wedge * 0.5) - 1.5708; // fold into one mirrored wedge, pointing up
|
||||||
|
float2 from = float2(cos(a), sin(a)) * length(c) / shape + 0.5;
|
||||||
|
return float4(in.image.sample(in.smp, from).rgb, 1.0);
|
||||||
|
}
|
||||||
13
native/effects/liquid.metal
Normal file
13
native/effects/liquid.metal
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
// Liquid trails: the previous output is dragged along the motion, swirled a little,
|
||||||
|
// and slowly fades back to the live picture.
|
||||||
|
// order 45
|
||||||
|
// slider persistence 0.0 0.99 0.92
|
||||||
|
// slider drag 0.0 6.0 2.5
|
||||||
|
// slider swirl 0.0 0.01 0.002
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float2 centred = uv - 0.5;
|
||||||
|
float2 from = uv - in.flow.sample(in.smp, uv).xy * in.p.drag + float2(-centred.y, centred.x) * in.p.swirl;
|
||||||
|
float3 trail = in.prev.sample(in.smp, from).rgb;
|
||||||
|
float3 live = in.image.sample(in.smp, uv).rgb;
|
||||||
|
return float4(mix(live, trail, in.p.persistence), 1.0);
|
||||||
|
}
|
||||||
20
native/effects/motion-glow.metal
Normal file
20
native/effects/motion-glow.metal
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
// Whatever moves lights up and the glow lingers. The heat is carried from frame to frame
|
||||||
|
// in the alpha channel of this effect's own previous output.
|
||||||
|
// order 50
|
||||||
|
// slider sensitivity 1.0 30.0 10.0
|
||||||
|
// slider linger 0.5 0.99 0.93
|
||||||
|
// slider dim 0.0 1.0 0.45
|
||||||
|
// slider hue 0.0 1.0 0.07
|
||||||
|
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
|
||||||
|
static float3 hueColor(float h) {
|
||||||
|
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float moved = length(in.camera.sample(in.smp, uv).rgb - in.last.sample(in.smp, uv).rgb);
|
||||||
|
float fresh = smoothstep(0.3, 1.0, moved * in.p.sensitivity);
|
||||||
|
float heat = max(in.prev.sample(in.smp, uv).a * in.p.linger, fresh);
|
||||||
|
|
||||||
|
float3 glow = hueColor(in.p.hue);
|
||||||
|
return float4(in.image.sample(in.smp, uv).rgb * in.p.dim + glow * heat, heat);
|
||||||
|
}
|
||||||
38
native/effects/neon-edges.metal
Normal file
38
native/effects/neon-edges.metal
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
// Glowing outlines with a slowly cycling colour.
|
||||||
|
// order 20
|
||||||
|
// slider detail 1.0 12.0 5.0
|
||||||
|
// slider glow 0.0 6.0 2.5
|
||||||
|
// slider source 0.0 1.0 0.35
|
||||||
|
// slider hue 0.0 1.0 0.55
|
||||||
|
// slider cycle 0.0 1.0 0.3
|
||||||
|
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
|
||||||
|
static float3 hueColor(float h) {
|
||||||
|
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static float luma(Inputs in, float2 uv) {
|
||||||
|
return dot(in.image.sample(in.smp, uv).rgb, float3(0.299, 0.587, 0.114));
|
||||||
|
}
|
||||||
|
|
||||||
|
// how strongly brightness changes around this point
|
||||||
|
static float edge(Inputs in, float2 uv) {
|
||||||
|
float2 t = 1.5 / in.resolution;
|
||||||
|
float dx = luma(in, uv + float2(t.x, 0.0)) - luma(in, uv - float2(t.x, 0.0));
|
||||||
|
float dy = luma(in, uv + float2(0.0, t.y)) - luma(in, uv - float2(0.0, t.y));
|
||||||
|
return length(float2(dx, dy));
|
||||||
|
}
|
||||||
|
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float sharp = smoothstep(0.1, 0.5, edge(in, uv) * in.p.detail);
|
||||||
|
float halo = 0.0; // edges found in a ring around this pixel = a cheap glow
|
||||||
|
for (int i = 0; i < 12; i++) {
|
||||||
|
float a = 6.2832 * float(i) / 12.0;
|
||||||
|
float2 dir = float2(cos(a), sin(a)) / in.resolution;
|
||||||
|
halo += edge(in, uv + dir * 4.0) + edge(in, uv + dir * 9.0);
|
||||||
|
}
|
||||||
|
halo = halo / 24.0 * in.p.detail * in.p.glow;
|
||||||
|
|
||||||
|
float3 tint = hueColor(in.p.hue + in.time * in.p.cycle * 0.2);
|
||||||
|
float3 base = in.image.sample(in.smp, uv).rgb * in.p.source;
|
||||||
|
return float4(base + tint * (sharp + halo), 1.0);
|
||||||
|
}
|
||||||
22
native/effects/particles.metal
Normal file
22
native/effects/particles.metal
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
// Particles spawn where you move and drift along with the motion.
|
||||||
|
// order 30
|
||||||
|
// on
|
||||||
|
// slider brightness 0.0 3.0 1.4
|
||||||
|
// slider glow 0.0 4.0 1.5
|
||||||
|
// slider hue 0.0 1.0 0.08
|
||||||
|
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
|
||||||
|
static float3 hueColor(float h) {
|
||||||
|
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float core = in.particles.sample(in.smp, uv).r;
|
||||||
|
float halo = 0.0;
|
||||||
|
for (int i = 0; i < 8; i++) {
|
||||||
|
float a = 6.2832 * float(i) / 8.0;
|
||||||
|
halo += in.particles.sample(in.smp, uv + float2(cos(a), sin(a)) * 5.0 / in.resolution).r;
|
||||||
|
}
|
||||||
|
float3 tint = hueColor(in.p.hue);
|
||||||
|
float3 base = in.image.sample(in.smp, uv).rgb;
|
||||||
|
return float4(base + tint * (core * in.p.brightness + halo / 8.0 * in.p.glow), 1.0);
|
||||||
|
}
|
||||||
16
native/effects/trails.metal
Normal file
16
native/effects/trails.metal
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
// Feedback trails: the previous output is zoomed, turned and faded, so bright things leave streaks.
|
||||||
|
// "push" smears the trails in the direction things are moving.
|
||||||
|
// order 40
|
||||||
|
// on
|
||||||
|
// slider fade 0.5 0.995 0.92
|
||||||
|
// slider zoom -0.04 0.04 0.012
|
||||||
|
// slider spin -0.05 0.05 0.005
|
||||||
|
// slider push 0.0 4.0 0.0
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float2 c = uv - 0.5;
|
||||||
|
float2 turned = float2(c.x * cos(in.p.spin) - c.y * sin(in.p.spin), c.x * sin(in.p.spin) + c.y * cos(in.p.spin));
|
||||||
|
float2 from = turned * (1.0 - in.p.zoom) + 0.5 - in.flow.sample(in.smp, uv).xy * in.p.push;
|
||||||
|
float3 trail = in.prev.sample(in.smp, from).rgb * in.p.fade;
|
||||||
|
float3 live = in.image.sample(in.smp, uv).rgb;
|
||||||
|
return float4(max(live, trail), 1.0);
|
||||||
|
}
|
||||||
13
native/effects/wave-shift.metal
Normal file
13
native/effects/wave-shift.metal
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
// RGB split that ripples down the image and gets pushed around by your movement.
|
||||||
|
// order 60
|
||||||
|
// slider amount 0.0 0.05 0.012
|
||||||
|
// slider waves 1.0 80.0 24.0
|
||||||
|
// slider speed 0.0 10.0 3.0
|
||||||
|
// slider react 0.0 3.0 1.0
|
||||||
|
float4 effect(float2 uv, Inputs in) {
|
||||||
|
float2 d = float2(in.p.amount * sin(uv.y * in.p.waves + in.time * in.p.speed), 0.0);
|
||||||
|
d += in.flow.sample(in.smp, uv).xy * in.p.react;
|
||||||
|
return float4(in.image.sample(in.smp, uv + d).r,
|
||||||
|
in.image.sample(in.smp, uv).g,
|
||||||
|
in.image.sample(in.smp, uv - d).b, 1.0);
|
||||||
|
}
|
||||||
61
panel.py
Normal file
61
panel.py
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
"""Control panel drawn straight onto the output canvas: click a layer to toggle it, drag its sliders."""
|
||||||
|
import cv2
|
||||||
|
|
||||||
|
BG = (26, 24, 22)
|
||||||
|
TRACK = (52, 48, 46)
|
||||||
|
FG = (228, 228, 228)
|
||||||
|
DIM = (125, 125, 125)
|
||||||
|
ACCENT = (255, 176, 64)
|
||||||
|
FONT = cv2.FONT_HERSHEY_SIMPLEX
|
||||||
|
|
||||||
|
|
||||||
|
class Panel:
|
||||||
|
WIDTH = 260
|
||||||
|
|
||||||
|
def __init__(self, layers):
|
||||||
|
self.layers = layers
|
||||||
|
self.hits = [] # (x0, y0, x1, y1, target, bar_x0, bar_x1), rebuilt on every draw
|
||||||
|
self.drag = None
|
||||||
|
|
||||||
|
def draw(self, canvas, x0, status):
|
||||||
|
h = canvas.shape[0]
|
||||||
|
canvas[:, x0:] = BG
|
||||||
|
self.hits = []
|
||||||
|
x, right = x0 + 14, x0 + self.WIDTH - 14
|
||||||
|
cv2.putText(canvas, "MOTIONLAB", (x, 26), FONT, 0.55, FG, 1, cv2.LINE_AA)
|
||||||
|
cv2.putText(canvas, status, (x + 124, 26), FONT, 0.4, DIM, 1, cv2.LINE_AA)
|
||||||
|
|
||||||
|
y = 44
|
||||||
|
for layer in self.layers:
|
||||||
|
on = layer.enabled
|
||||||
|
cv2.rectangle(canvas, (x, y), (x + 12, y + 12), ACCENT if on else DIM, -1 if on else 1)
|
||||||
|
cv2.putText(canvas, layer.name, (x + 22, y + 11), FONT, 0.48, FG if on else DIM, 1, cv2.LINE_AA)
|
||||||
|
self.hits.append((x0, y - 5, x0 + self.WIDTH, y + 17, layer, 0, 0))
|
||||||
|
y += 22
|
||||||
|
if not on:
|
||||||
|
continue
|
||||||
|
bar = x + 100
|
||||||
|
for name, p in layer.params.items():
|
||||||
|
cv2.putText(canvas, name, (x + 4, y + 8), FONT, 0.36, DIM, 1, cv2.LINE_AA)
|
||||||
|
cv2.rectangle(canvas, (bar, y), (right, y + 8), TRACK, -1)
|
||||||
|
cv2.rectangle(canvas, (bar, y), (bar + int((right - bar) * p.norm), y + 8), ACCENT, -1)
|
||||||
|
self.hits.append((bar - 8, y - 4, right + 8, y + 12, p, bar, right))
|
||||||
|
y += 16
|
||||||
|
y += 4
|
||||||
|
|
||||||
|
cv2.putText(canvas, "[h] panel [s] shot [r] rec [q] quit", (x, h - 12), FONT, 0.36, DIM, 1, cv2.LINE_AA)
|
||||||
|
|
||||||
|
def on_mouse(self, event, mx, my, flags, _):
|
||||||
|
if event == cv2.EVENT_LBUTTONDOWN:
|
||||||
|
for x0, y0, x1, y1, target, bar_x0, bar_x1 in self.hits:
|
||||||
|
if x0 <= mx < x1 and y0 <= my < y1:
|
||||||
|
if bar_x1:
|
||||||
|
self.drag = (target, bar_x0, bar_x1)
|
||||||
|
else:
|
||||||
|
target.enabled = not target.enabled
|
||||||
|
break
|
||||||
|
elif event == cv2.EVENT_LBUTTONUP:
|
||||||
|
self.drag = None
|
||||||
|
if self.drag and event in (cv2.EVENT_LBUTTONDOWN, cv2.EVENT_MOUSEMOVE):
|
||||||
|
param, bar_x0, bar_x1 = self.drag
|
||||||
|
param.norm = (mx - bar_x0) / (bar_x1 - bar_x0)
|
||||||
172
shaders.py
Normal file
172
shaders.py
Normal file
|
|
@ -0,0 +1,172 @@
|
||||||
|
"""Shader layers: every effects/*.glsl file becomes a layer, reloaded whenever the file is saved.
|
||||||
|
|
||||||
|
A shader file only contains its own sliders and a main(). The host supplies the rest:
|
||||||
|
|
||||||
|
in vec2 uv; pixel position, (0,0) top-left .. (1,1) bottom-right
|
||||||
|
out vec4 color; what to write for this pixel
|
||||||
|
uniform sampler2D camera; the incoming frame (output of the layer above), RGB
|
||||||
|
uniform sampler2D prev; this layer's own output from the previous frame
|
||||||
|
uniform sampler2D flow; .xy = how far this pixel moved since last frame, in uv units
|
||||||
|
uniform sampler2D motion; .r = how much this pixel changed since last frame, 0..1
|
||||||
|
uniform float time; seconds since start
|
||||||
|
uniform float energy; overall amount of motion in the frame, 0..1
|
||||||
|
uniform vec2 resolution; output size in pixels
|
||||||
|
|
||||||
|
Sliders are plain float uniforms with a range comment:
|
||||||
|
|
||||||
|
uniform float amount; // 0.0 .. 0.05 = 0.01
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
|
import moderngl
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from effects import Layer, Param
|
||||||
|
|
||||||
|
VERTEX = """#version 410
|
||||||
|
out vec2 uv;
|
||||||
|
void main() {
|
||||||
|
uv = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2);
|
||||||
|
gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0);
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
# "#line 1" makes compiler errors report line numbers of the user's file
|
||||||
|
HEADER = """#version 410
|
||||||
|
uniform sampler2D camera;
|
||||||
|
uniform sampler2D prev;
|
||||||
|
uniform sampler2D flow;
|
||||||
|
uniform sampler2D motion;
|
||||||
|
uniform float time;
|
||||||
|
uniform float energy;
|
||||||
|
uniform vec2 resolution;
|
||||||
|
in vec2 uv;
|
||||||
|
out vec4 color;
|
||||||
|
#line 1
|
||||||
|
"""
|
||||||
|
|
||||||
|
NUM = r"(-?[\d.]+)"
|
||||||
|
PARAM_RE = re.compile(rf"uniform\s+float\s+(\w+)\s*;(?:[ \t]*//\s*{NUM}\s*\.\.\s*{NUM}\s*=\s*{NUM})?")
|
||||||
|
|
||||||
|
|
||||||
|
class ShaderLayer(Layer):
|
||||||
|
enabled = False
|
||||||
|
|
||||||
|
def __init__(self, gl, path):
|
||||||
|
super().__init__()
|
||||||
|
self.gl, self.path = gl, path
|
||||||
|
self.name = os.path.splitext(os.path.basename(path))[0]
|
||||||
|
self.program = None
|
||||||
|
self.error = None
|
||||||
|
self.mtime = None
|
||||||
|
self.targets = None
|
||||||
|
self.reload()
|
||||||
|
|
||||||
|
def reload(self):
|
||||||
|
"""Recompile if the file changed. On a compile error the last working version keeps running."""
|
||||||
|
try:
|
||||||
|
mtime = os.stat(self.path).st_mtime
|
||||||
|
except FileNotFoundError:
|
||||||
|
return
|
||||||
|
if mtime == self.mtime:
|
||||||
|
return
|
||||||
|
self.mtime = mtime
|
||||||
|
with open(self.path) as f:
|
||||||
|
source = f.read()
|
||||||
|
try:
|
||||||
|
program = self.gl.program(vertex_shader=VERTEX, fragment_shader=HEADER + source)
|
||||||
|
except moderngl.Error as e:
|
||||||
|
self.error = str(e)
|
||||||
|
print(f"[{self.name}] shader error:\n{e}", flush=True)
|
||||||
|
return
|
||||||
|
self.error = None
|
||||||
|
self.program = program
|
||||||
|
self.vao = self.gl.vertex_array(program, [])
|
||||||
|
old = self.params
|
||||||
|
self.params = {name: Param(float(default or 0.5), float(lo or 0), float(hi or 1))
|
||||||
|
for name, lo, hi, default in PARAM_RE.findall(source)}
|
||||||
|
for name, p in self.params.items(): # keep slider positions across reloads
|
||||||
|
if name in old:
|
||||||
|
p.value = min(max(old[name].value, p.lo), p.hi)
|
||||||
|
print(f"[{self.name}] loaded", flush=True)
|
||||||
|
|
||||||
|
def _target(self, size):
|
||||||
|
texture = self.gl.texture(size, 4)
|
||||||
|
texture.repeat_x = texture.repeat_y = False
|
||||||
|
return texture, self.gl.framebuffer([texture])
|
||||||
|
|
||||||
|
def _texture(self, attr, size, components, dtype="f1"):
|
||||||
|
if not hasattr(self, attr):
|
||||||
|
texture = self.gl.texture(size, components, dtype=dtype)
|
||||||
|
texture.repeat_x = texture.repeat_y = False
|
||||||
|
setattr(self, attr, texture)
|
||||||
|
return getattr(self, attr)
|
||||||
|
|
||||||
|
def _set(self, name, value):
|
||||||
|
uniform = self.program.get(name, None) # absent if the shader doesn't use it
|
||||||
|
if uniform is not None:
|
||||||
|
uniform.value = value
|
||||||
|
return uniform is not None
|
||||||
|
|
||||||
|
def apply(self, frame, ctx):
|
||||||
|
if self.program is None:
|
||||||
|
return frame
|
||||||
|
h, w = frame.shape[:2]
|
||||||
|
if self.targets is None:
|
||||||
|
# two render targets, swapped every frame, so the shader can read its previous output
|
||||||
|
self.targets = [self._target((w, h)), self._target((w, h))]
|
||||||
|
(_, framebuffer), (prev, _) = self.targets
|
||||||
|
|
||||||
|
camera = self._texture("camera", (w, h), 3)
|
||||||
|
camera.swizzle = "BGR1" # frames are BGR; shaders see RGB
|
||||||
|
camera.write(np.ascontiguousarray(frame))
|
||||||
|
camera.use(0)
|
||||||
|
self._set("camera", 0)
|
||||||
|
prev.use(1)
|
||||||
|
self._set("prev", 1)
|
||||||
|
if self._set("flow", 2):
|
||||||
|
fh, fw = ctx.flow.shape[:2]
|
||||||
|
flow = self._texture("flow_tex", (fw, fh), 2, "f4")
|
||||||
|
flow.write((ctx.flow / (fw, fh)).astype("f4"))
|
||||||
|
flow.use(2)
|
||||||
|
if self._set("motion", 3):
|
||||||
|
mh, mw = ctx.motion.shape
|
||||||
|
motion = self._texture("motion_tex", (mw, mh), 1)
|
||||||
|
motion.write(ctx.motion)
|
||||||
|
motion.use(3)
|
||||||
|
self._set("time", ctx.t)
|
||||||
|
self._set("energy", ctx.energy)
|
||||||
|
self._set("resolution", (w, h))
|
||||||
|
for name, p in self.params.items():
|
||||||
|
self._set(name, p.value)
|
||||||
|
|
||||||
|
framebuffer.use()
|
||||||
|
self.vao.render(moderngl.TRIANGLES, vertices=3)
|
||||||
|
out = np.frombuffer(framebuffer.read(components=3), np.uint8).reshape(h, w, 3)
|
||||||
|
self.targets.reverse()
|
||||||
|
return np.ascontiguousarray(out[:, :, ::-1])
|
||||||
|
|
||||||
|
|
||||||
|
class ShaderBank:
|
||||||
|
"""Watches a folder of .glsl files and keeps one ShaderLayer per file in the shared layer list."""
|
||||||
|
|
||||||
|
def __init__(self, directory, layers):
|
||||||
|
self.directory, self.layers = directory, layers
|
||||||
|
self.known = {}
|
||||||
|
self.gl = moderngl.create_context(standalone=True, require=410)
|
||||||
|
self.refresh()
|
||||||
|
|
||||||
|
def refresh(self):
|
||||||
|
if os.path.isdir(self.directory):
|
||||||
|
for filename in sorted(os.listdir(self.directory)):
|
||||||
|
path = os.path.join(self.directory, filename)
|
||||||
|
if filename.endswith(".glsl") and path not in self.known:
|
||||||
|
self.known[path] = ShaderLayer(self.gl, path)
|
||||||
|
self.layers.append(self.known[path])
|
||||||
|
for layer in self.known.values():
|
||||||
|
layer.reload()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def errors(self):
|
||||||
|
return [layer.name for layer in self.known.values() if layer.error]
|
||||||
Loading…
Add table
Add a link
Reference in a new issue