Python/OpenCV proof of concept plus first native Swift port
Snapshot before the Python POC is removed. The POC (main.py, effects.py, panel.py, shaders.py, effects/*.glsl, flake.nix) is the reference for the native app under native/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
6591138bbf
27 changed files with 2076 additions and 0 deletions
5
.gitignore
vendored
Normal file
5
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
.build/
|
||||
captures/
|
||||
__pycache__/
|
||||
.DS_Store
|
||||
headless.png
|
||||
227
effects.py
Normal file
227
effects.py
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
"""Effect layers.
|
||||
|
||||
Each layer takes the current BGR frame plus a shared Context (motion analysis
|
||||
of the raw camera frame) and returns a new BGR frame. Layers run top to bottom
|
||||
in the order returned by default_layers(), like a chain of TOPs.
|
||||
"""
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
rng = np.random.default_rng()
|
||||
|
||||
|
||||
class Param:
|
||||
def __init__(self, value, lo=0.0, hi=1.0, integer=False):
|
||||
self.value, self.lo, self.hi, self.integer = value, lo, hi, integer
|
||||
|
||||
@property
|
||||
def norm(self):
|
||||
return (self.value - self.lo) / (self.hi - self.lo)
|
||||
|
||||
@norm.setter
|
||||
def norm(self, n):
|
||||
v = self.lo + min(max(n, 0.0), 1.0) * (self.hi - self.lo)
|
||||
self.value = round(v) if self.integer else v
|
||||
|
||||
|
||||
class Layer:
|
||||
name = "layer"
|
||||
enabled = True
|
||||
|
||||
def __init__(self, **params):
|
||||
self.params = params
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.params[key].value
|
||||
|
||||
def apply(self, frame, ctx):
|
||||
return frame
|
||||
|
||||
|
||||
class Context:
|
||||
"""Per-frame motion analysis, computed at 1/4 resolution and shared by all layers."""
|
||||
|
||||
SCALE = 4
|
||||
|
||||
def __init__(self, size):
|
||||
self.small = (size[0] // self.SCALE, size[1] // self.SCALE)
|
||||
self.prev = None
|
||||
self.t = 0.0
|
||||
self.dt = 1 / 30
|
||||
|
||||
def update(self, frame, dt):
|
||||
self.dt = dt
|
||||
self.t += dt
|
||||
gray = cv2.cvtColor(cv2.resize(frame, self.small, interpolation=cv2.INTER_AREA), cv2.COLOR_BGR2GRAY)
|
||||
if self.prev is None:
|
||||
self.prev = gray
|
||||
self.motion = cv2.absdiff(gray, self.prev) # where things changed
|
||||
self.energy = float(self.motion.mean()) / 255 # how much changed overall
|
||||
self._pair = (self.prev, gray)
|
||||
self._flow = None
|
||||
self.prev = gray
|
||||
|
||||
@property
|
||||
def flow(self):
|
||||
"""Dense optical flow (small-res pixels/frame). Lazy: only paid for if a layer asks."""
|
||||
if self._flow is None:
|
||||
self._flow = cv2.calcOpticalFlowFarneback(*self._pair, None, 0.5, 2, 15, 2, 5, 1.1, 0)
|
||||
return self._flow
|
||||
|
||||
|
||||
def hue_bgr(hue):
|
||||
"""Hue in 0..1 (wraps) -> saturated BGR colour with components in 0..1."""
|
||||
hsv = np.uint8([[[int(hue % 1.0 * 180), 255, 255]]])
|
||||
return cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)[0, 0] / 255.0
|
||||
|
||||
|
||||
def tint(gray, color):
|
||||
return cv2.merge([cv2.convertScaleAbs(gray, alpha=float(c)) for c in color])
|
||||
|
||||
|
||||
class Colorize(Layer):
|
||||
name = "Colorize"
|
||||
enabled = False
|
||||
PALETTES = [cv2.COLORMAP_INFERNO, cv2.COLORMAP_OCEAN, cv2.COLORMAP_TWILIGHT_SHIFTED, cv2.COLORMAP_TURBO]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(mix=Param(0.8), palette=Param(0, 0, len(self.PALETTES) - 1, integer=True))
|
||||
|
||||
def apply(self, frame, ctx):
|
||||
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||||
mapped = cv2.applyColorMap(gray, self.PALETTES[self["palette"]])
|
||||
return cv2.addWeighted(frame, 1 - self["mix"], mapped, self["mix"], 0)
|
||||
|
||||
|
||||
class Edges(Layer):
|
||||
name = "Neon Edges"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(detail=Param(0.6), glow=Param(0.35), source=Param(0.35),
|
||||
hue=Param(0.55), cycle=Param(0.3))
|
||||
|
||||
def apply(self, frame, ctx):
|
||||
gray = cv2.GaussianBlur(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), (5, 5), 0)
|
||||
lo = 20 + 180 * (1 - self["detail"])
|
||||
edges = cv2.Canny(gray, lo, lo * 2)
|
||||
sigma = self["glow"] * 12
|
||||
if sigma > 0.5:
|
||||
edges = cv2.addWeighted(edges, 1, cv2.GaussianBlur(edges, (0, 0), sigma), 2.5, 0)
|
||||
color = hue_bgr(self["hue"] + ctx.t * self["cycle"] * 0.2)
|
||||
return cv2.add(cv2.convertScaleAbs(frame, alpha=self["source"]), tint(edges, color))
|
||||
|
||||
|
||||
class Particles(Layer):
|
||||
"""Particles spawn where there is motion and get carried along by the optical flow."""
|
||||
|
||||
name = "Flow Particles"
|
||||
MAX = 6000
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(count=Param(3000, 200, self.MAX, integer=True), speed=Param(1.5, 0, 4),
|
||||
trail=Param(0.88, 0, 0.98), hue=Param(0.08))
|
||||
self.buf = None
|
||||
|
||||
def apply(self, frame, ctx):
|
||||
h, w = frame.shape[:2]
|
||||
if self.buf is None:
|
||||
self.buf = np.zeros((h, w), np.uint8)
|
||||
self.pos = (rng.random((self.MAX, 2)) * (w, h)).astype(np.float32)
|
||||
self.life = rng.integers(20, 90, self.MAX)
|
||||
|
||||
n = self["count"]
|
||||
pos, life = self.pos[:n], self.life[:n]
|
||||
flow = ctx.flow
|
||||
fh, fw = flow.shape[:2]
|
||||
scale = w / fw
|
||||
ix = np.clip((pos[:, 0] / scale).astype(int), 0, fw - 1)
|
||||
iy = np.clip((pos[:, 1] / scale).astype(int), 0, fh - 1)
|
||||
new = pos + flow[iy, ix] * scale * self["speed"] + rng.normal(0, 0.5, (n, 2)).astype(np.float32)
|
||||
|
||||
life -= 1
|
||||
dead = (life <= 0) | (new[:, 0] < 0) | (new[:, 0] >= w) | (new[:, 1] < 0) | (new[:, 1] >= h)
|
||||
|
||||
self.buf = cv2.convertScaleAbs(self.buf, alpha=self["trail"])
|
||||
segments = np.stack([pos, new], axis=1).astype(np.int32)[~dead]
|
||||
cv2.polylines(self.buf, list(segments), False, 255, 1, cv2.LINE_AA)
|
||||
|
||||
k = int(dead.sum())
|
||||
if k:
|
||||
hot = np.flatnonzero(ctx.motion > 24)
|
||||
if hot.size > 50:
|
||||
ys, xs = np.divmod(rng.choice(hot, k), fw)
|
||||
new[dead] = (np.stack([xs, ys], axis=1) + rng.random((k, 2))) * scale
|
||||
else:
|
||||
new[dead] = rng.random((k, 2)) * (w, h)
|
||||
life[dead] = rng.integers(20, 90, k)
|
||||
self.pos[:n] = new
|
||||
|
||||
glow = cv2.add(self.buf, cv2.GaussianBlur(self.buf, (0, 0), 3))
|
||||
return cv2.add(frame, tint(glow, hue_bgr(self["hue"])))
|
||||
|
||||
|
||||
class Feedback(Layer):
|
||||
"""Classic feedback loop: the previous output is transformed, faded and blended back in."""
|
||||
|
||||
name = "Feedback"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(decay=Param(0.92, 0, 0.995), zoom=Param(0.012, -0.05, 0.05),
|
||||
rotate=Param(0.3, -3, 3), **{"flow push": Param(0.0, 0, 4)})
|
||||
self.prev = None
|
||||
|
||||
def apply(self, frame, ctx):
|
||||
h, w = frame.shape[:2]
|
||||
if self.prev is None:
|
||||
self.prev = frame
|
||||
self.grid = np.stack(np.meshgrid(np.arange(w, dtype=np.float32),
|
||||
np.arange(h, dtype=np.float32)), axis=-1)
|
||||
m = cv2.getRotationMatrix2D((w / 2, h / 2), self["rotate"], 1 + self["zoom"])
|
||||
fb = cv2.warpAffine(self.prev, m, (w, h), borderMode=cv2.BORDER_REFLECT)
|
||||
if self["flow push"] > 0:
|
||||
# smear the feedback along the direction things are moving
|
||||
flow = cv2.resize(ctx.flow, (w, h)) * (self["flow push"] * Context.SCALE)
|
||||
fb = cv2.remap(fb, self.grid - flow, None, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)
|
||||
out = cv2.max(frame, cv2.convertScaleAbs(fb, alpha=self["decay"]))
|
||||
self.prev = out
|
||||
return out
|
||||
|
||||
|
||||
class RGBShift(Layer):
|
||||
name = "RGB Shift"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(amount=Param(4, 0, 40), **{"motion react": Param(0.5)})
|
||||
|
||||
def apply(self, frame, ctx):
|
||||
d = int(self["amount"] + self["motion react"] * ctx.energy * 400)
|
||||
if d == 0:
|
||||
return frame
|
||||
b, g, r = cv2.split(frame)
|
||||
return cv2.merge([np.roll(b, d, axis=1), g, np.roll(r, -d, axis=1)])
|
||||
|
||||
|
||||
class Kaleido(Layer):
|
||||
name = "Kaleidoscope"
|
||||
enabled = False
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(segments=Param(6, 2, 16, integer=True), spin=Param(0.2, -2, 2))
|
||||
self.radius = None
|
||||
self.angle = 0.0
|
||||
|
||||
def apply(self, frame, ctx):
|
||||
h, w = frame.shape[:2]
|
||||
if self.radius is None:
|
||||
ys, xs = np.indices((h, w), dtype=np.float32)
|
||||
self.radius, self.theta = cv2.cartToPolar(xs - w / 2, ys - h / 2)
|
||||
self.angle += self["spin"] * ctx.dt
|
||||
wedge = 2 * np.pi / self["segments"]
|
||||
# fold every angle into one mirrored wedge, then sample the source from that wedge
|
||||
a = np.abs(np.mod(self.theta + np.float32(self.angle), np.float32(wedge)) - np.float32(wedge / 2))
|
||||
x, y = cv2.polarToCart(self.radius, a - np.float32(np.pi / 2))
|
||||
return cv2.remap(frame, x + w / 2, y + h / 2, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)
|
||||
|
||||
|
||||
def default_layers():
|
||||
return [Colorize(), Edges(), Particles(), Feedback(), RGBShift(), Kaleido()]
|
||||
17
effects/halftone.glsl
Normal file
17
effects/halftone.glsl
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
// Halftone dots sized by brightness; dots turn hot orange wherever something moves.
|
||||
uniform float cell; // 4.0 .. 40.0 = 10.0
|
||||
uniform float heat; // 0.0 .. 6.0 = 3.0
|
||||
|
||||
void main() {
|
||||
vec2 grid = resolution / cell;
|
||||
vec2 centre = (floor(uv * grid) + 0.5) / grid;
|
||||
vec3 c = texture(camera, centre).rgb;
|
||||
float luma = dot(c, vec3(0.299, 0.587, 0.114));
|
||||
|
||||
float radius = 0.5 * sqrt(luma);
|
||||
float dist = length(fract(uv * grid) - 0.5);
|
||||
float mask = 1.0 - smoothstep(radius - 0.05, radius + 0.05, dist);
|
||||
|
||||
float moving = clamp(texture(motion, centre).r * heat, 0.0, 1.0);
|
||||
color = vec4(mix(c, vec3(1.0, 0.35, 0.1), moving) * mask, 1.0);
|
||||
}
|
||||
13
effects/liquid.glsl
Normal file
13
effects/liquid.glsl
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
// Liquid trails: the previous output is dragged along the motion, swirled a little,
|
||||
// and slowly fades back to the live camera.
|
||||
uniform float persistence; // 0.0 .. 0.99 = 0.92
|
||||
uniform float drag; // 0.0 .. 6.0 = 2.5
|
||||
uniform float swirl; // 0.0 .. 0.01 = 0.002
|
||||
|
||||
void main() {
|
||||
vec2 centred = uv - 0.5;
|
||||
vec2 from = uv - texture(flow, uv).xy * drag + vec2(-centred.y, centred.x) * swirl;
|
||||
vec3 trail = texture(prev, from).rgb;
|
||||
vec3 live = texture(camera, uv).rgb;
|
||||
color = vec4(mix(live, trail, persistence), 1.0);
|
||||
}
|
||||
12
effects/wave-shift.glsl
Normal file
12
effects/wave-shift.glsl
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
// RGB split that ripples down the image and gets pushed around by your movement.
|
||||
uniform float amount; // 0.0 .. 0.05 = 0.012
|
||||
uniform float waves; // 1.0 .. 80.0 = 24.0
|
||||
uniform float react; // 0.0 .. 3.0 = 1.0
|
||||
|
||||
void main() {
|
||||
vec2 d = vec2(amount * sin(uv.y * waves + time * 3.0), 0.0);
|
||||
d += texture(flow, uv).xy * react;
|
||||
color = vec4(texture(camera, uv + d).r,
|
||||
texture(camera, uv).g,
|
||||
texture(camera, uv - d).b, 1.0);
|
||||
}
|
||||
27
flake.lock
generated
Normal file
27
flake.lock
generated
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1791097562,
|
||||
"narHash": "sha256-K9O9lG3UZNFCnwrz0ViWgP3Kgqukg1DvvzsrkFJfcvU=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "a322c65bf7233456788a9ebf03bc067db7dc090a",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
53
flake.nix
Normal file
53
flake.nix
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
{
|
||||
description = "motionlab - realtime camera + motion graphics playground";
|
||||
|
||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
let
|
||||
systems = [ "aarch64-darwin" "x86_64-darwin" "x86_64-linux" "aarch64-linux" ];
|
||||
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system});
|
||||
|
||||
# not in nixpkgs; sends frames to the OBS virtual camera (macOS) or v4l2loopback (Linux)
|
||||
pyvirtualcamFor = ps: ps.buildPythonPackage rec {
|
||||
pname = "pyvirtualcam";
|
||||
version = "0.15.0";
|
||||
pyproject = true;
|
||||
src = nixpkgs.legacyPackages.${ps.python.stdenv.hostPlatform.system}.fetchFromGitHub {
|
||||
owner = "letmaik";
|
||||
repo = "pyvirtualcam";
|
||||
tag = "v${version}";
|
||||
fetchSubmodules = true;
|
||||
hash = "sha256-JtFchhuvVYYhrYOOcsHPC79/Iezyo5QppV4tY9nrjgQ=";
|
||||
};
|
||||
build-system = [ ps.setuptools ps.pybind11 ];
|
||||
dependencies = [ ps.numpy ];
|
||||
doCheck = false;
|
||||
pythonImportsCheck = [ "pyvirtualcam" ];
|
||||
};
|
||||
|
||||
pythonFor = pkgs: pkgs.python3.withPackages (ps: [
|
||||
ps.numpy
|
||||
ps.moderngl
|
||||
(pyvirtualcamFor ps)
|
||||
# macOS gets a Cocoa window out of the box; Linux needs GTK for cv2.imshow
|
||||
(if pkgs.stdenv.hostPlatform.isLinux
|
||||
then ps.opencv4.override { enableGtk3 = true; }
|
||||
else ps.opencv4)
|
||||
]);
|
||||
in
|
||||
{
|
||||
devShells = forAllSystems (pkgs: {
|
||||
default = pkgs.mkShell { packages = [ (pythonFor pkgs) ]; };
|
||||
});
|
||||
|
||||
apps = forAllSystems (pkgs: {
|
||||
default = {
|
||||
type = "app";
|
||||
program = toString (pkgs.writeShellScript "motionlab" ''
|
||||
exec ${pythonFor pkgs}/bin/python ${self}/main.py "$@"
|
||||
'');
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
191
main.py
Normal file
191
main.py
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
"""motionlab - a tiny TouchDesigner-style playground: live camera in, motion-reactive graphics out."""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from effects import Context, default_layers, hue_bgr
|
||||
from panel import Panel
|
||||
from shaders import ShaderBank
|
||||
|
||||
SIZE = (960, 540)
|
||||
VCAM_SIZE = (1280, 720)
|
||||
WINDOW = "motionlab"
|
||||
|
||||
|
||||
class Synthetic:
|
||||
"""Drifting blobs, so the app (and --headless) works without a camera."""
|
||||
|
||||
def __init__(self):
|
||||
self.i = 0
|
||||
|
||||
def read(self):
|
||||
w, h = SIZE
|
||||
t = self.i / 30
|
||||
self.i += 1
|
||||
img = np.full((h, w, 3), (30, 20, 20), np.uint8)
|
||||
for k in range(5):
|
||||
x = int(w * (0.5 + 0.4 * np.sin(t * (0.7 + 0.2 * k) + k)))
|
||||
y = int(h * (0.5 + 0.4 * np.cos(t * (0.5 + 0.3 * k) + 2 * k)))
|
||||
cv2.circle(img, (x, y), 40 + 12 * k, hue_bgr(k / 5) * 255, -1)
|
||||
return True, cv2.GaussianBlur(img, (0, 0), 4)
|
||||
|
||||
def release(self):
|
||||
pass
|
||||
|
||||
|
||||
def open_source(args):
|
||||
if args.demo or args.headless:
|
||||
return Synthetic()
|
||||
backend = cv2.CAP_AVFOUNDATION if sys.platform == "darwin" else cv2.CAP_ANY
|
||||
cap = cv2.VideoCapture(args.camera, backend)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
|
||||
# without camera permission macOS can hand back all-black frames instead of failing
|
||||
for _ in range(10):
|
||||
ok, frame = cap.read()
|
||||
if ok and frame.any():
|
||||
return cap
|
||||
cap.release()
|
||||
print("Could not read from the camera - falling back to the synthetic demo source.\n"
|
||||
"On macOS: allow your terminal under System Settings > Privacy & Security > Camera, then rerun.",
|
||||
file=sys.stderr)
|
||||
return Synthetic()
|
||||
|
||||
|
||||
def open_virtualcam():
|
||||
"""Virtual camera other apps (Meet, Zoom, ...) can pick as their webcam."""
|
||||
import pyvirtualcam
|
||||
try:
|
||||
cam = pyvirtualcam.Camera(*VCAM_SIZE, 30, fmt=pyvirtualcam.PixelFormat.BGR)
|
||||
except RuntimeError as e:
|
||||
sys.exit(f"Could not start the virtual camera: {e}")
|
||||
print(f"virtual camera on: pick '{cam.device}' as the camera in your call app")
|
||||
return cam
|
||||
|
||||
|
||||
def fit(frame):
|
||||
"""Centre-crop to the output aspect ratio, then resize."""
|
||||
w, h = SIZE
|
||||
fh, fw = frame.shape[:2]
|
||||
cw = min(fw, fh * w // h)
|
||||
ch = min(fh, fw * h // w)
|
||||
x, y = (fw - cw) // 2, (fh - ch) // 2
|
||||
return cv2.resize(frame[y:y + ch, x:x + cw], SIZE, interpolation=cv2.INTER_AREA)
|
||||
|
||||
|
||||
def capture_path(ext):
|
||||
os.makedirs("captures", exist_ok=True)
|
||||
return os.path.join("captures", time.strftime(f"motionlab-%Y%m%d-%H%M%S.{ext}"))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--camera", type=int, default=0, help="camera index (default 0)")
|
||||
ap.add_argument("--demo", action="store_true", help="use the synthetic source instead of the camera")
|
||||
ap.add_argument("--virtualcam", action="store_true", help="also send the output to a virtual camera for video calls")
|
||||
ap.add_argument("--headless", type=int, metavar="N", help="render N synthetic frames without a window, then exit")
|
||||
ap.add_argument("--out", default="headless.png", help="where --headless writes its last frame")
|
||||
ap.add_argument("--effects", default="effects", help="folder of .glsl shader effects (default ./effects)")
|
||||
ap.add_argument("--only", metavar="NAMES", help="enable only these layers, comma separated (e.g. --only liquid,halftone)")
|
||||
args = ap.parse_args()
|
||||
|
||||
w, h = SIZE
|
||||
src = open_source(args)
|
||||
layers = default_layers()
|
||||
# prefer ./effects so edits are picked up live; fall back to the copy shipped next to this file
|
||||
effects_dir = args.effects if os.path.isdir(args.effects) else os.path.join(os.path.dirname(__file__), "effects")
|
||||
bank = ShaderBank(effects_dir, layers)
|
||||
if args.only:
|
||||
wanted = {name.strip().lower() for name in args.only.split(",")}
|
||||
unknown = wanted - {layer.name.lower() for layer in layers}
|
||||
if unknown:
|
||||
sys.exit(f"unknown layer(s): {', '.join(sorted(unknown))}. Available: {', '.join(l.name for l in layers)}")
|
||||
for layer in layers:
|
||||
layer.enabled = layer.name.lower() in wanted
|
||||
if args.headless and bank.errors:
|
||||
sys.exit(f"shader compile failed: {', '.join(bank.errors)}")
|
||||
ctx = Context(SIZE)
|
||||
panel = Panel(layers)
|
||||
canvas = np.empty((h, w + Panel.WIDTH, 3), np.uint8)
|
||||
show_panel, writer, fps = True, None, 30.0
|
||||
vcam = open_virtualcam() if args.virtualcam else None
|
||||
|
||||
if not args.headless:
|
||||
cv2.namedWindow(WINDOW, cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO)
|
||||
cv2.resizeWindow(WINDOW, w + Panel.WIDTH, h)
|
||||
cv2.setMouseCallback(WINDOW, panel.on_mouse)
|
||||
|
||||
start = last = time.perf_counter()
|
||||
frames = 0
|
||||
while True:
|
||||
ok, frame = src.read()
|
||||
if not ok:
|
||||
break
|
||||
now = time.perf_counter()
|
||||
dt = 1 / 30 if args.headless else max(now - last, 1e-3)
|
||||
last = now
|
||||
fps += (1 / dt - fps) * 0.1
|
||||
|
||||
frame = cv2.flip(fit(frame), 1) # mirror, so it behaves like a mirror
|
||||
if frames % 15 == 0:
|
||||
bank.refresh() # pick up new and edited shader files
|
||||
ctx.update(frame, dt)
|
||||
out = frame
|
||||
for layer in layers:
|
||||
if layer.enabled:
|
||||
out = layer.apply(out, ctx)
|
||||
|
||||
if writer:
|
||||
writer.write(out)
|
||||
if vcam:
|
||||
# un-mirror: the preview is a mirror, but the people on the call should see the real thing
|
||||
vcam.send(cv2.resize(cv2.flip(out, 1), VCAM_SIZE))
|
||||
view = out
|
||||
if show_panel:
|
||||
canvas[:, :w] = out
|
||||
panel.draw(canvas, w, f"{fps:3.0f} fps" + (" REC" if writer else "") + (" CAM" if vcam else ""))
|
||||
view = canvas
|
||||
else:
|
||||
panel.hits = []
|
||||
|
||||
frames += 1
|
||||
if args.headless:
|
||||
if frames >= args.headless:
|
||||
cv2.imwrite(args.out, view)
|
||||
print(f"{frames} frames, {(time.perf_counter() - start) / frames * 1000:.1f} ms/frame -> {args.out}")
|
||||
break
|
||||
continue
|
||||
|
||||
cv2.imshow(WINDOW, view)
|
||||
key = cv2.waitKey(1) & 0xFF
|
||||
if key in (ord("q"), 27) or cv2.getWindowProperty(WINDOW, cv2.WND_PROP_VISIBLE) < 1:
|
||||
break
|
||||
elif key == ord("h"):
|
||||
show_panel = not show_panel
|
||||
elif key == ord("s"):
|
||||
path = capture_path("png")
|
||||
cv2.imwrite(path, out)
|
||||
print("saved", path)
|
||||
elif key == ord("r"):
|
||||
if writer:
|
||||
writer.release()
|
||||
writer = None
|
||||
else:
|
||||
path = capture_path("mp4")
|
||||
writer = cv2.VideoWriter(path, cv2.VideoWriter_fourcc(*"mp4v"), round(fps), SIZE)
|
||||
print("recording", path)
|
||||
|
||||
if writer:
|
||||
writer.release()
|
||||
if vcam:
|
||||
vcam.close()
|
||||
src.release()
|
||||
cv2.destroyAllWindows()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
10
native/Package.swift
Normal file
10
native/Package.swift
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
// swift-tools-version:5.9
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "MotionLab",
|
||||
platforms: [.macOS(.v13)],
|
||||
targets: [
|
||||
.executableTarget(name: "MotionLab", path: "Sources/MotionLab")
|
||||
]
|
||||
)
|
||||
180
native/Sources/MotionLab/Analysis.swift
Normal file
180
native/Sources/MotionLab/Analysis.swift
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
import Metal
|
||||
|
||||
/// Host-side motion analysis, shared by all effects. Every camera frame it produces:
|
||||
///
|
||||
/// flow which way things moved (small 2-channel texture, uv units per frame)
|
||||
/// particles a picture of particles that spawn where something moves and drift along the flow
|
||||
///
|
||||
/// Effects read these as `in.flow` and `in.particles`.
|
||||
final class Analysis {
|
||||
private static let flowWidth = 160, flowHeight = 90
|
||||
private static let stateWidth = 128, stateHeight = 64 // one texel per particle
|
||||
private static let particleSpeed: Float = 1.5
|
||||
private static let trailFade: Float = 0.88
|
||||
|
||||
// each pair is [this frame, previous frame], swapped at the start of every update
|
||||
private var luma: [MTLTexture]
|
||||
private var flowTextures: [MTLTexture]
|
||||
private var state: [MTLTexture]
|
||||
private var trail: [MTLTexture]
|
||||
|
||||
private let lumaPipeline: MTLRenderPipelineState
|
||||
private let flowPipeline: MTLRenderPipelineState
|
||||
private let movePipeline: MTLRenderPipelineState
|
||||
private let fadePipeline: MTLRenderPipelineState
|
||||
private let dotPipeline: MTLRenderPipelineState
|
||||
|
||||
var flow: MTLTexture { flowTextures[0] }
|
||||
var particles: MTLTexture { trail[0] }
|
||||
|
||||
init(device: MTLDevice, queue: MTLCommandQueue, width: Int, height: Int) throws {
|
||||
func pair(_ w: Int, _ h: Int, _ format: MTLPixelFormat) -> [MTLTexture] {
|
||||
[makeTarget(device, width: w, height: h, format: format), makeTarget(device, width: w, height: h, format: format)]
|
||||
}
|
||||
luma = pair(Analysis.flowWidth, Analysis.flowHeight, .r16Float)
|
||||
flowTextures = pair(Analysis.flowWidth, Analysis.flowHeight, .rg16Float)
|
||||
state = pair(Analysis.stateWidth, Analysis.stateHeight, .rgba32Float)
|
||||
trail = pair(width, height, .r8Unorm)
|
||||
|
||||
let source = shaderPrelude + analysisShaders
|
||||
lumaPipeline = try makePipeline(device, source: source, fragment: "lumaFragment", format: .r16Float)
|
||||
flowPipeline = try makePipeline(device, source: source, fragment: "flowFragment", format: .rg16Float)
|
||||
movePipeline = try makePipeline(device, source: source, fragment: "moveFragment", format: .rgba32Float)
|
||||
fadePipeline = try makePipeline(device, source: source, fragment: "fadeFragment", format: .r8Unorm)
|
||||
dotPipeline = try makePipeline(device, source: source, vertex: "dotVertex", fragment: "dotFragment",
|
||||
format: .r8Unorm, additive: true)
|
||||
|
||||
// new textures hold garbage; start everything from zero
|
||||
if let commands = queue.makeCommandBuffer() {
|
||||
for texture in luma + flowTextures + state + trail { commands.pass(into: texture, clear: true) }
|
||||
commands.commit()
|
||||
}
|
||||
}
|
||||
|
||||
func update(_ commands: MTLCommandBuffer, camera: MTLTexture, particles wantsParticles: Bool) {
|
||||
luma.swapAt(0, 1)
|
||||
commands.pass(into: luma[0]) { $0.fullscreen(lumaPipeline, textures: [camera]) }
|
||||
|
||||
flowTextures.swapAt(0, 1)
|
||||
commands.pass(into: flowTextures[0]) {
|
||||
$0.fullscreen(flowPipeline, textures: [luma[0], luma[1], flowTextures[1]])
|
||||
}
|
||||
|
||||
guard wantsParticles else { return }
|
||||
state.swapAt(0, 1)
|
||||
commands.pass(into: state[0]) {
|
||||
$0.fullscreen(movePipeline, textures: [state[1], flowTextures[0], luma[0], luma[1]],
|
||||
uniforms: [Float.random(in: 0..<1000), Analysis.particleSpeed, 0, 0])
|
||||
}
|
||||
trail.swapAt(0, 1)
|
||||
commands.pass(into: trail[0]) { encoder in
|
||||
encoder.fullscreen(fadePipeline, textures: [trail[1]], uniforms: [Analysis.trailFade, 0, 0, 0])
|
||||
encoder.setRenderPipelineState(dotPipeline)
|
||||
encoder.setVertexTexture(state[0], index: 0)
|
||||
encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: Analysis.stateWidth * Analysis.stateHeight)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private let analysisShaders = """
|
||||
static float brightness(float3 c) { return dot(c, float3(0.299, 0.587, 0.114)); }
|
||||
static float hash(float2 p) { return fract(sin(dot(p, float2(127.1, 311.7))) * 43758.5453); }
|
||||
|
||||
// Camera -> small, blurred brightness image. Flow is measured on this.
|
||||
fragment float4 lumaFragment(VertexOut v [[stage_in]], texture2d<float> camera [[texture(0)]]) {
|
||||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||
float2 texel = 1.0 / float2(camera.get_width(), camera.get_height());
|
||||
float sum = 0.0;
|
||||
for (int y = 0; y < 4; y++) {
|
||||
for (int x = 0; x < 4; x++) {
|
||||
sum += brightness(camera.sample(s, v.uv + (float2(x, y) - 1.5) * 2.0 * texel).rgb);
|
||||
}
|
||||
}
|
||||
return float4(sum / 16.0);
|
||||
}
|
||||
|
||||
// Lucas-Kanade optical flow: for each pixel, the shift that best explains how its 7x7
|
||||
// neighbourhood changed between the previous and the current brightness image.
|
||||
fragment float4 flowFragment(VertexOut v [[stage_in]],
|
||||
texture2d<float> now [[texture(0)]],
|
||||
texture2d<float> before [[texture(1)]],
|
||||
texture2d<float> flowBefore [[texture(2)]]) {
|
||||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||
float2 texel = 1.0 / float2(now.get_width(), now.get_height());
|
||||
float a = 1e-3, b = 0.0, c = 1e-3, p = 0.0, q = 0.0; // the 1e-3 keeps flat areas at zero flow
|
||||
for (int y = -3; y <= 3; y++) {
|
||||
for (int x = -3; x <= 3; x++) {
|
||||
float2 uv = v.uv + float2(x, y) * texel;
|
||||
float ix = (now.sample(s, uv + float2(texel.x, 0.0)).r - now.sample(s, uv - float2(texel.x, 0.0)).r) * 0.5;
|
||||
float iy = (now.sample(s, uv + float2(0.0, texel.y)).r - now.sample(s, uv - float2(0.0, texel.y)).r) * 0.5;
|
||||
float it = now.sample(s, uv).r - before.sample(s, uv).r;
|
||||
a += ix * ix; b += ix * iy; c += iy * iy;
|
||||
p += ix * it; q += iy * it;
|
||||
}
|
||||
}
|
||||
float2 shift = -float2(c * p - b * q, a * q - b * p) / (a * c - b * b); // in small-image pixels
|
||||
float len = length(shift);
|
||||
if (len > 4.0) { shift *= 4.0 / len; }
|
||||
float2 smoothed = mix(flowBefore.sample(s, v.uv).xy, shift * texel, 0.6);
|
||||
return float4(smoothed, 0.0, 1.0);
|
||||
}
|
||||
|
||||
// One texel per particle: xy = position (uv), z = frames left to live, w = 1 if it was born from motion.
|
||||
fragment float4 moveFragment(VertexOut v [[stage_in]],
|
||||
texture2d<float> state [[texture(0)]],
|
||||
texture2d<float> flow [[texture(1)]],
|
||||
texture2d<float> now [[texture(2)]],
|
||||
texture2d<float> before [[texture(3)]],
|
||||
constant float4 &u [[buffer(0)]]) { // x = random seed, y = speed
|
||||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||
float4 particle = state.read(uint2(v.position.xy));
|
||||
float2 seed = v.position.xy * 0.37 + u.x;
|
||||
float2 pos = particle.xy + flow.sample(s, particle.xy).xy * u.y;
|
||||
pos += (float2(hash(seed), hash(seed + 7.3)) - 0.5) * 0.0015;
|
||||
float life = particle.z - 1.0;
|
||||
float born = particle.w;
|
||||
|
||||
if (life <= 0.0 || pos.x < 0.0 || pos.x > 1.0 || pos.y < 0.0 || pos.y > 1.0) {
|
||||
// respawn: try a few random spots and take the first one where something is moving
|
||||
pos = float2(hash(seed + 1.1), hash(seed + 2.2));
|
||||
born = 0.0;
|
||||
life = 2.0 + hash(seed + 5.5) * 4.0; // nothing moving: stay hidden and try again soon
|
||||
for (int i = 0; i < 6; i++) {
|
||||
float2 spot = float2(hash(seed + 3.3 + float(i)), hash(seed + 9.9 + float(i) * 1.7));
|
||||
if (abs(now.sample(s, spot).r - before.sample(s, spot).r) > 0.04) {
|
||||
pos = spot;
|
||||
born = 1.0;
|
||||
life = 20.0 + hash(seed + 5.5) * 70.0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return float4(pos, life, born);
|
||||
}
|
||||
|
||||
// Particle trails: fade what was there, then add a dot per particle on top.
|
||||
fragment float4 fadeFragment(VertexOut v [[stage_in]], texture2d<float> trail [[texture(0)]],
|
||||
constant float4 &u [[buffer(0)]]) { // x = fade
|
||||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||
return trail.sample(s, v.uv) * u.x;
|
||||
}
|
||||
|
||||
struct DotOut {
|
||||
float4 position [[position]];
|
||||
float size [[point_size]];
|
||||
};
|
||||
|
||||
vertex DotOut dotVertex(uint id [[vertex_id]], texture2d<float> state [[texture(0)]]) {
|
||||
uint width = state.get_width();
|
||||
float4 particle = state.read(uint2(id % width, id / width));
|
||||
DotOut out;
|
||||
out.position = float4(particle.x * 2.0 - 1.0, 1.0 - particle.y * 2.0, 0.0, 1.0);
|
||||
if (particle.w < 0.5) { out.position.x = 4.0; } // hidden particles are moved off screen
|
||||
out.size = 3.0;
|
||||
return out;
|
||||
}
|
||||
|
||||
fragment float4 dotFragment(DotOut in [[stage_in]], float2 coord [[point_coord]]) {
|
||||
return float4(smoothstep(0.5, 0.1, length(coord - 0.5)) * 0.9);
|
||||
}
|
||||
"""
|
||||
95
native/Sources/MotionLab/Camera.swift
Normal file
95
native/Sources/MotionLab/Camera.swift
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import AVFoundation
|
||||
import CoreVideo
|
||||
import Metal
|
||||
|
||||
enum CameraError: Error, CustomStringConvertible {
|
||||
case noPermission
|
||||
case noCamera
|
||||
|
||||
var description: String {
|
||||
switch self {
|
||||
case .noPermission:
|
||||
return "No camera permission. Allow your terminal under System Settings > Privacy & Security > Camera, then restart it."
|
||||
case .noCamera:
|
||||
return "No camera found."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Streams the built-in camera as Metal textures; `latest` is the newest frame and the one before it.
|
||||
final class Camera: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
||||
private let session = AVCaptureSession()
|
||||
private var cache: CVMetalTextureCache?
|
||||
private let lock = NSLock()
|
||||
// each entry keeps its CVMetalTexture, which keeps the pixel buffer behind the Metal texture alive
|
||||
private var current: (texture: MTLTexture, backing: CVMetalTexture)?
|
||||
private var previous: (texture: MTLTexture, backing: CVMetalTexture)?
|
||||
|
||||
private var serial = 0 // counts frames, so the renderer can tell a new frame from one it already saw
|
||||
|
||||
var latest: (current: MTLTexture, last: MTLTexture, serial: Int)? {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
guard let current else { return nil }
|
||||
return (current.texture, (previous ?? current).texture, serial)
|
||||
}
|
||||
|
||||
init(device: MTLDevice) throws {
|
||||
super.init()
|
||||
guard Camera.requestAccess() else { throw CameraError.noPermission }
|
||||
// prefer the built-in camera over an iPhone that happens to be nearby
|
||||
guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .unspecified)
|
||||
?? AVCaptureDevice.default(for: .video)
|
||||
else { throw CameraError.noCamera }
|
||||
|
||||
CVMetalTextureCacheCreate(nil, nil, device, nil, &cache)
|
||||
let output = AVCaptureVideoDataOutput()
|
||||
output.videoSettings = [
|
||||
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferMetalCompatibilityKey as String: true,
|
||||
]
|
||||
output.alwaysDiscardsLateVideoFrames = true
|
||||
output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "motionlab.camera"))
|
||||
|
||||
session.beginConfiguration()
|
||||
session.addInput(try AVCaptureDeviceInput(device: camera))
|
||||
session.addOutput(output)
|
||||
if session.canSetSessionPreset(.hd1280x720) { session.sessionPreset = .hd1280x720 }
|
||||
print("camera: \(camera.localizedName)")
|
||||
if let connection = output.connection(with: .video), connection.isVideoMirroringSupported {
|
||||
connection.automaticallyAdjustsVideoMirroring = false
|
||||
connection.isVideoMirrored = true // behave like a mirror
|
||||
}
|
||||
session.commitConfiguration()
|
||||
session.startRunning()
|
||||
}
|
||||
|
||||
private static func requestAccess() -> Bool {
|
||||
switch AVCaptureDevice.authorizationStatus(for: .video) {
|
||||
case .authorized:
|
||||
return true
|
||||
case .notDetermined:
|
||||
let done = DispatchSemaphore(value: 0)
|
||||
var granted = false
|
||||
AVCaptureDevice.requestAccess(for: .video) { granted = $0; done.signal() }
|
||||
done.wait()
|
||||
return granted
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
|
||||
guard let cache, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
||||
var created: CVMetalTexture?
|
||||
CVMetalTextureCacheCreateTextureFromImage(
|
||||
nil, cache, buffer, nil, .bgra8Unorm,
|
||||
CVPixelBufferGetWidth(buffer), CVPixelBufferGetHeight(buffer), 0, &created)
|
||||
guard let created, let frame = CVMetalTextureGetTexture(created) else { return }
|
||||
lock.lock()
|
||||
previous = current
|
||||
current = (frame, created)
|
||||
serial += 1
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
198
native/Sources/MotionLab/Output.swift
Normal file
198
native/Sources/MotionLab/Output.swift
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
import AVFoundation
|
||||
import CoreMediaIO
|
||||
import ImageIO
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
func writePNG(bgra pixels: inout [UInt8], width: Int, height: Int, to url: URL) {
|
||||
let bitmapInfo = CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue
|
||||
guard let context = CGContext(data: &pixels, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4,
|
||||
space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: bitmapInfo),
|
||||
let image = context.makeImage(),
|
||||
let destination = CGImageDestinationCreateWithURL(url as CFURL, UTType.png.identifier as CFString, 1, nil)
|
||||
else { return }
|
||||
CGImageDestinationAddImage(destination, image, nil)
|
||||
CGImageDestinationFinalize(destination)
|
||||
}
|
||||
|
||||
/// Writes frames to an H.264 .mp4 file.
|
||||
final class Recorder {
|
||||
let url: URL
|
||||
private let writer: AVAssetWriter
|
||||
private let input: AVAssetWriterInput
|
||||
private let adaptor: AVAssetWriterInputPixelBufferAdaptor
|
||||
private let width: Int, height: Int
|
||||
private var startTime: Double?
|
||||
|
||||
init(url: URL, width: Int, height: Int) throws {
|
||||
self.url = url
|
||||
self.width = width
|
||||
self.height = height
|
||||
writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
|
||||
input = AVAssetWriterInput(mediaType: .video, outputSettings: [
|
||||
AVVideoCodecKey: AVVideoCodecType.h264,
|
||||
AVVideoWidthKey: width,
|
||||
AVVideoHeightKey: height,
|
||||
])
|
||||
input.expectsMediaDataInRealTime = true
|
||||
adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [
|
||||
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferWidthKey as String: width,
|
||||
kCVPixelBufferHeightKey as String: height,
|
||||
])
|
||||
writer.add(input)
|
||||
guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) }
|
||||
writer.startSession(atSourceTime: .zero)
|
||||
}
|
||||
|
||||
func append(bgra pixels: [UInt8], at time: Double) {
|
||||
guard input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return }
|
||||
var created: CVPixelBuffer?
|
||||
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created)
|
||||
guard let buffer = created else { return }
|
||||
|
||||
CVPixelBufferLockBaseAddress(buffer, [])
|
||||
if let base = CVPixelBufferGetBaseAddress(buffer) {
|
||||
let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
|
||||
pixels.withUnsafeBytes { source in
|
||||
for y in 0..<height {
|
||||
memcpy(base + y * rowBytes, source.baseAddress! + y * width * 4, width * 4)
|
||||
}
|
||||
}
|
||||
}
|
||||
CVPixelBufferUnlockBaseAddress(buffer, [])
|
||||
|
||||
if startTime == nil { startTime = time }
|
||||
adaptor.append(buffer, withPresentationTime: CMTime(seconds: time - startTime!, preferredTimescale: 600))
|
||||
}
|
||||
|
||||
func finish() {
|
||||
input.markAsFinished()
|
||||
let done = DispatchSemaphore(value: 0)
|
||||
writer.finishWriting { done.signal() }
|
||||
done.wait()
|
||||
}
|
||||
}
|
||||
|
||||
struct VirtualCameraError: Error, CustomStringConvertible {
|
||||
let description: String
|
||||
}
|
||||
|
||||
/// Feeds frames into OBS's virtual camera, so Meet, Zoom and anything else that takes a webcam can use them.
|
||||
/// OBS itself must not be running its own virtual camera at the same time.
|
||||
final class VirtualCamera {
|
||||
// fixed identifier of the camera OBS installs (from obs-studio's mac-virtualcam plugin)
|
||||
private static let obsDeviceUID = "7626645E-4425-469E-9D8B-97E0FA59AC75"
|
||||
|
||||
private let deviceID: CMIODeviceID
|
||||
private let streamID: CMIOStreamID
|
||||
private let queue: CMSimpleQueue
|
||||
private let format: CMFormatDescription
|
||||
private let pool: CVPixelBufferPool
|
||||
|
||||
init(width: Int, height: Int) throws {
|
||||
func address(_ selector: Int) -> CMIOObjectPropertyAddress {
|
||||
CMIOObjectPropertyAddress(mSelector: CMIOObjectPropertySelector(selector),
|
||||
mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
|
||||
mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain))
|
||||
}
|
||||
func objects(of object: CMIOObjectID, _ selector: Int) -> [CMIOObjectID] {
|
||||
var property = address(selector)
|
||||
var size: UInt32 = 0, used: UInt32 = 0
|
||||
CMIOObjectGetPropertyDataSize(object, &property, 0, nil, &size)
|
||||
var ids = [CMIOObjectID](repeating: 0, count: Int(size) / MemoryLayout<CMIOObjectID>.size)
|
||||
CMIOObjectGetPropertyData(object, &property, 0, nil, size, &used, &ids)
|
||||
return ids
|
||||
}
|
||||
func uid(of device: CMIOObjectID) -> String? {
|
||||
var property = address(kCMIODevicePropertyDeviceUID)
|
||||
var value: Unmanaged<CFString>?
|
||||
var used: UInt32 = 0
|
||||
CMIOObjectGetPropertyData(device, &property, 0, nil, UInt32(MemoryLayout<Unmanaged<CFString>?>.size), &used, &value)
|
||||
return value?.takeRetainedValue() as String?
|
||||
}
|
||||
|
||||
let devices = objects(of: CMIOObjectID(kCMIOObjectSystemObject), kCMIOHardwarePropertyDevices)
|
||||
guard let device = devices.first(where: { uid(of: $0) == VirtualCamera.obsDeviceUID }) else {
|
||||
throw VirtualCameraError(description: """
|
||||
OBS Virtual Camera not found. One-time setup: enable OBS under System Settings > General > \
|
||||
Login Items & Extensions > Camera Extensions, then in OBS click Start Virtual Camera, Stop Virtual Camera, and quit OBS.
|
||||
""")
|
||||
}
|
||||
// the camera has two streams: [0] is what apps watch, [1] is where frames are fed in
|
||||
let streams = objects(of: device, kCMIODevicePropertyStreams)
|
||||
guard streams.count >= 2 else { throw VirtualCameraError(description: "OBS Virtual Camera has no input stream.") }
|
||||
deviceID = device
|
||||
streamID = streams[1]
|
||||
|
||||
var createdQueue: Unmanaged<CMSimpleQueue>?
|
||||
CMIOStreamCopyBufferQueue(streamID, { _, _, _ in }, nil, &createdQueue)
|
||||
var createdFormat: CMFormatDescription?
|
||||
CMVideoFormatDescriptionCreate(allocator: nil, codecType: kCVPixelFormatType_422YpCbCr8,
|
||||
width: Int32(width), height: Int32(height), extensions: nil,
|
||||
formatDescriptionOut: &createdFormat)
|
||||
var createdPool: CVPixelBufferPool?
|
||||
CVPixelBufferPoolCreate(nil, nil, [
|
||||
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_422YpCbCr8,
|
||||
kCVPixelBufferWidthKey: width,
|
||||
kCVPixelBufferHeightKey: height,
|
||||
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary,
|
||||
] as CFDictionary, &createdPool)
|
||||
guard let createdQueue, let createdFormat, let createdPool else {
|
||||
throw VirtualCameraError(description: "Could not set up the OBS Virtual Camera stream.")
|
||||
}
|
||||
queue = createdQueue.takeRetainedValue()
|
||||
format = createdFormat
|
||||
pool = createdPool
|
||||
guard CMIODeviceStartStream(deviceID, streamID) == noErr else {
|
||||
throw VirtualCameraError(description: "Could not start the OBS Virtual Camera stream (is OBS using it?).")
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
CMIODeviceStopStream(deviceID, streamID)
|
||||
}
|
||||
|
||||
/// Sends one frame. The picture is flipped back left-to-right: the preview is a mirror,
|
||||
/// but the people on the call should see the real thing.
|
||||
func send(bgra pixels: [UInt8], width: Int, height: Int) {
|
||||
var created: CVPixelBuffer?
|
||||
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created)
|
||||
guard let buffer = created else { return }
|
||||
|
||||
CVPixelBufferLockBaseAddress(buffer, [])
|
||||
if let base = CVPixelBufferGetBaseAddress(buffer)?.assumingMemoryBound(to: UInt8.self) {
|
||||
let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
|
||||
pixels.withUnsafeBufferPointer { source in
|
||||
// BGRA -> UYVY (BT.601 video range): two pixels share one U and one V
|
||||
for y in 0..<height {
|
||||
let row = base + y * rowBytes
|
||||
for pair in 0..<(width / 2) {
|
||||
let first = (y * width + (width - 1 - pair * 2)) * 4
|
||||
let second = first - 4
|
||||
let b0 = Int(source[first]), g0 = Int(source[first + 1]), r0 = Int(source[first + 2])
|
||||
let b1 = Int(source[second]), g1 = Int(source[second + 1]), r1 = Int(source[second + 2])
|
||||
row[pair * 4] = UInt8(clamping: ((-38 * r0 - 74 * g0 + 112 * b0 + 128) >> 8) + 128)
|
||||
row[pair * 4 + 1] = UInt8(clamping: ((66 * r0 + 129 * g0 + 25 * b0 + 128) >> 8) + 16)
|
||||
row[pair * 4 + 2] = UInt8(clamping: ((112 * r0 - 94 * g0 - 18 * b0 + 128) >> 8) + 128)
|
||||
row[pair * 4 + 3] = UInt8(clamping: ((66 * r1 + 129 * g1 + 25 * b1 + 128) >> 8) + 16)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
CVPixelBufferUnlockBaseAddress(buffer, [])
|
||||
|
||||
var timing = CMSampleTimingInfo(
|
||||
duration: .invalid,
|
||||
presentationTimeStamp: CMTime(value: CMTimeValue(clock_gettime_nsec_np(CLOCK_UPTIME_RAW)), timescale: 1_000_000_000),
|
||||
decodeTimeStamp: .invalid)
|
||||
var sample: CMSampleBuffer?
|
||||
CMSampleBufferCreateForImageBuffer(allocator: nil, imageBuffer: buffer, dataReady: true, makeDataReadyCallback: nil,
|
||||
refcon: nil, formatDescription: format, sampleTiming: &timing, sampleBufferOut: &sample)
|
||||
if let sample {
|
||||
// the queue takes ownership of the sample
|
||||
if CMSimpleQueueEnqueue(queue, element: Unmanaged.passRetained(sample).toOpaque()) != noErr {
|
||||
Unmanaged.passUnretained(sample).release()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
92
native/Sources/MotionLab/Panel.swift
Normal file
92
native/Sources/MotionLab/Panel.swift
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import AppKit
|
||||
|
||||
private final class TopAlignedView: NSView {
|
||||
override var isFlipped: Bool { true } // so the scroll view starts at the top
|
||||
}
|
||||
|
||||
/// Side panel: every effect with a checkbox, and the sliders of the ones that are switched on.
|
||||
final class Panel: NSObject {
|
||||
static let width: CGFloat = 230
|
||||
|
||||
let view = NSScrollView()
|
||||
private let stack = NSStackView()
|
||||
private let statusLabel = NSTextField(labelWithString: "")
|
||||
private let renderer: Renderer
|
||||
|
||||
init(renderer: Renderer) {
|
||||
self.renderer = renderer
|
||||
super.init()
|
||||
stack.orientation = .vertical
|
||||
stack.alignment = .leading
|
||||
stack.spacing = 4
|
||||
stack.translatesAutoresizingMaskIntoConstraints = false
|
||||
|
||||
let document = TopAlignedView()
|
||||
document.translatesAutoresizingMaskIntoConstraints = false
|
||||
document.addSubview(stack)
|
||||
view.documentView = document
|
||||
view.hasVerticalScroller = true
|
||||
view.drawsBackground = false
|
||||
NSLayoutConstraint.activate([
|
||||
view.widthAnchor.constraint(equalToConstant: Panel.width),
|
||||
document.widthAnchor.constraint(equalTo: view.contentView.widthAnchor),
|
||||
stack.topAnchor.constraint(equalTo: document.topAnchor, constant: 14),
|
||||
stack.leadingAnchor.constraint(equalTo: document.leadingAnchor, constant: 14),
|
||||
stack.trailingAnchor.constraint(equalTo: document.trailingAnchor, constant: -14),
|
||||
stack.bottomAnchor.constraint(equalTo: document.bottomAnchor, constant: -14),
|
||||
])
|
||||
statusLabel.font = .boldSystemFont(ofSize: 11)
|
||||
statusLabel.textColor = .systemRed
|
||||
rebuild()
|
||||
}
|
||||
|
||||
private func label(_ text: String, dim: Bool = true) -> NSTextField {
|
||||
let field = NSTextField(labelWithString: text)
|
||||
field.font = .systemFont(ofSize: 11)
|
||||
field.textColor = dim ? .secondaryLabelColor : .labelColor
|
||||
return field
|
||||
}
|
||||
|
||||
func rebuild() {
|
||||
stack.arrangedSubviews.forEach { $0.removeFromSuperview() }
|
||||
statusLabel.stringValue = renderer.status
|
||||
stack.addArrangedSubview(statusLabel)
|
||||
|
||||
for (layerIndex, layer) in renderer.layers.enumerated() {
|
||||
let title = layer.error == nil ? layer.name : "\(layer.name) (shader error)"
|
||||
let checkbox = NSButton(checkboxWithTitle: title, target: self, action: #selector(toggled(_:)))
|
||||
checkbox.state = layer.enabled ? .on : .off
|
||||
checkbox.tag = layerIndex
|
||||
checkbox.font = .systemFont(ofSize: 13, weight: .medium)
|
||||
stack.addArrangedSubview(checkbox)
|
||||
guard layer.enabled else { continue }
|
||||
|
||||
for (paramIndex, param) in layer.params.enumerated() {
|
||||
let slider = NSSlider(value: Double(param.value), minValue: Double(param.min), maxValue: Double(param.max),
|
||||
target: self, action: #selector(sliderMoved(_:)))
|
||||
slider.tag = layerIndex * maxParams + paramIndex
|
||||
slider.controlSize = .small
|
||||
stack.addArrangedSubview(label(param.name))
|
||||
stack.addArrangedSubview(slider)
|
||||
slider.widthAnchor.constraint(equalTo: stack.widthAnchor).isActive = true
|
||||
}
|
||||
stack.setCustomSpacing(12, after: stack.arrangedSubviews.last!)
|
||||
}
|
||||
if renderer.layers.isEmpty { stack.addArrangedSubview(label("no .metal files found")) }
|
||||
|
||||
stack.setCustomSpacing(16, after: stack.arrangedSubviews.last!)
|
||||
stack.addArrangedSubview(label("s screenshot\nr record\nq quit"))
|
||||
}
|
||||
|
||||
func refreshStatus() {
|
||||
statusLabel.stringValue = renderer.status
|
||||
}
|
||||
|
||||
@objc private func toggled(_ checkbox: NSButton) {
|
||||
renderer.setEnabled(checkbox.tag, checkbox.state == .on)
|
||||
}
|
||||
|
||||
@objc private func sliderMoved(_ slider: NSSlider) {
|
||||
renderer.setParam(layer: slider.tag / maxParams, param: slider.tag % maxParams, value: Float(slider.doubleValue))
|
||||
}
|
||||
}
|
||||
287
native/Sources/MotionLab/Renderer.swift
Normal file
287
native/Sources/MotionLab/Renderer.swift
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
import MetalKit
|
||||
|
||||
/// One effect file in the chain.
|
||||
final class Layer {
|
||||
let url: URL
|
||||
let name: String
|
||||
var enabled = false
|
||||
var order = 50
|
||||
var params: [EffectParam] = []
|
||||
var error: String?
|
||||
fileprivate var pipeline: MTLRenderPipelineState?
|
||||
fileprivate var modified: Date?
|
||||
fileprivate var usesParticles = false
|
||||
fileprivate var targets: [MTLTexture] = [] // [this frame, previous frame], swapped after every draw
|
||||
|
||||
init(url: URL) {
|
||||
self.url = url
|
||||
self.name = url.deletingPathExtension().lastPathComponent
|
||||
}
|
||||
}
|
||||
|
||||
private let passthrough = "float4 effect(float2 uv, Inputs in) { return in.image.sample(in.smp, uv); }"
|
||||
|
||||
/// Runs every enabled effect in order on each new camera frame, shows the result in the window,
|
||||
/// and hands it to whatever outputs are active (recording, virtual camera, screenshots).
|
||||
final class Renderer: NSObject, MTKViewDelegate {
|
||||
private let device: MTLDevice
|
||||
private let queue: MTLCommandQueue
|
||||
private let camera: Camera
|
||||
private let effectsDirectory: URL
|
||||
private let copyPipeline: MTLRenderPipelineState
|
||||
private var analysis: Analysis?
|
||||
private var plain: MTLTexture? // holds the camera picture when no effect is enabled
|
||||
private var result: MTLTexture? // the finished frame
|
||||
private var lastSerial = -1
|
||||
private let startTime = CACurrentMediaTime()
|
||||
private var frameCount = 0
|
||||
private var pixels: [UInt8] = []
|
||||
|
||||
private(set) var layers: [Layer] = []
|
||||
/// Called when layers were added, removed, reloaded or toggled.
|
||||
var onLayersChanged: (() -> Void)?
|
||||
/// Called when recording or the virtual camera starts or stops.
|
||||
var onStatusChanged: (() -> Void)?
|
||||
|
||||
/// When set, the app exits after this many camera frames, first saving that frame as a PNG if a url is given
|
||||
/// (used to check effects without watching).
|
||||
var snapshot: (frames: Int, url: URL?)?
|
||||
/// When set, recording into this file starts with the first frame.
|
||||
var recordURL: URL?
|
||||
/// When true, the OBS virtual camera is opened on the first frame, at the size of the output.
|
||||
var wantsVirtualCamera = false
|
||||
private var virtualCamera: VirtualCamera? { didSet { onStatusChanged?() } }
|
||||
private var recorder: Recorder? { didSet { onStatusChanged?() } }
|
||||
private var screenshotRequested = false
|
||||
|
||||
var status: String {
|
||||
[recorder != nil ? "REC" : nil, virtualCamera != nil ? "CAM" : nil].compactMap { $0 }.joined(separator: " ")
|
||||
}
|
||||
|
||||
init(device: MTLDevice, camera: Camera, effectsDirectory: URL) throws {
|
||||
self.device = device
|
||||
self.camera = camera
|
||||
self.effectsDirectory = effectsDirectory
|
||||
self.queue = device.makeCommandQueue()!
|
||||
self.copyPipeline = try makePipeline(device, source: effectShader(body: passthrough, params: []),
|
||||
fragment: "effectFragment", format: .bgra8Unorm)
|
||||
super.init()
|
||||
refreshFiles()
|
||||
}
|
||||
|
||||
// MARK: effect files
|
||||
|
||||
/// Picks up new, deleted and edited effect files. On a compile error the last working version keeps running.
|
||||
private func refreshFiles() {
|
||||
let files = ((try? FileManager.default.contentsOfDirectory(at: effectsDirectory, includingPropertiesForKeys: nil)) ?? [])
|
||||
.filter { $0.pathExtension == "metal" }
|
||||
var changed = false
|
||||
|
||||
let before = layers.count
|
||||
layers.removeAll { layer in !files.contains(layer.url) }
|
||||
changed = layers.count != before
|
||||
for url in files where !layers.contains(where: { $0.url == url }) {
|
||||
layers.append(Layer(url: url))
|
||||
changed = true
|
||||
}
|
||||
for layer in layers where reload(layer) { changed = true }
|
||||
|
||||
if changed {
|
||||
layers.sort { ($0.order, $0.name) < ($1.order, $1.name) }
|
||||
onLayersChanged?()
|
||||
}
|
||||
}
|
||||
|
||||
private func reload(_ layer: Layer) -> Bool {
|
||||
let modified = (try? layer.url.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate
|
||||
guard let modified, modified != layer.modified else { return false }
|
||||
let firstLoad = layer.modified == nil
|
||||
layer.modified = modified
|
||||
do {
|
||||
let source = try String(contentsOf: layer.url, encoding: .utf8)
|
||||
var info = parseEffect(source)
|
||||
layer.pipeline = try makePipeline(device, source: effectShader(body: source, params: info.params),
|
||||
fragment: "effectFragment", format: .bgra8Unorm)
|
||||
for i in info.params.indices { // keep slider positions across reloads
|
||||
if let old = layer.params.first(where: { $0.name == info.params[i].name }) {
|
||||
info.params[i].value = min(max(old.value, info.params[i].min), info.params[i].max)
|
||||
}
|
||||
}
|
||||
layer.params = info.params
|
||||
layer.order = info.order
|
||||
layer.usesParticles = source.contains("in.particles")
|
||||
layer.error = nil
|
||||
if firstLoad { layer.enabled = info.startsOn }
|
||||
print("[\(layer.name)] loaded")
|
||||
} catch {
|
||||
layer.error = error.localizedDescription
|
||||
print("[\(layer.name)] shader error:\n\(error.localizedDescription)")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var failedLayers: [String] { layers.filter { $0.error != nil }.map(\.name) }
|
||||
|
||||
/// Enables exactly the named layers. Returns the names that don't exist.
|
||||
func enableOnly(_ names: [String]) -> [String] {
|
||||
for layer in layers { layer.enabled = names.contains(layer.name) }
|
||||
return names.filter { name in !layers.contains { $0.name == name } }
|
||||
}
|
||||
|
||||
func setEnabled(_ index: Int, _ enabled: Bool) {
|
||||
guard layers.indices.contains(index) else { return }
|
||||
layers[index].enabled = enabled
|
||||
if !enabled { layers[index].targets = [] } // free its memory, and start clean next time
|
||||
onLayersChanged?()
|
||||
}
|
||||
|
||||
func setParam(layer: Int, param: Int, value: Float) {
|
||||
guard layers.indices.contains(layer), layers[layer].params.indices.contains(param) else { return }
|
||||
let p = layers[layer].params[param]
|
||||
layers[layer].params[param].value = min(max(value, p.min), p.max)
|
||||
}
|
||||
|
||||
// MARK: outputs
|
||||
|
||||
func requestScreenshot() { screenshotRequested = true }
|
||||
|
||||
func toggleRecording() {
|
||||
if let recorder {
|
||||
self.recorder = nil
|
||||
recorder.finish()
|
||||
print("saved \(recorder.url.path)")
|
||||
} else if let result {
|
||||
do {
|
||||
recorder = try Recorder(url: capturesURL(extension: "mp4"), width: result.width, height: result.height)
|
||||
print("recording \(recorder!.url.path)")
|
||||
} catch {
|
||||
print("could not start recording: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func finish() {
|
||||
if recorder != nil { toggleRecording() }
|
||||
}
|
||||
|
||||
private func capturesURL(extension ext: String) -> URL {
|
||||
let directory = URL(fileURLWithPath: "captures")
|
||||
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
||||
let stamp = DateFormatter()
|
||||
stamp.dateFormat = "yyyyMMdd-HHmmss"
|
||||
return directory.appendingPathComponent("motionlab-\(stamp.string(from: Date())).\(ext)")
|
||||
}
|
||||
|
||||
/// Runs once per camera frame, after the GPU work was submitted.
|
||||
private func deliver(_ commands: MTLCommandBuffer) {
|
||||
frameCount += 1
|
||||
let snapshotDue = snapshot.map { frameCount >= $0.frames } ?? false
|
||||
guard let result else { return }
|
||||
let width = result.width, height = result.height
|
||||
if let url = recordURL {
|
||||
recordURL = nil
|
||||
do { recorder = try Recorder(url: url, width: width, height: height) } catch { print("could not start recording: \(error.localizedDescription)") }
|
||||
}
|
||||
if wantsVirtualCamera {
|
||||
wantsVirtualCamera = false
|
||||
do {
|
||||
virtualCamera = try VirtualCamera(width: width, height: height)
|
||||
print("virtual camera on: pick 'OBS Virtual Camera' as the camera in your call or streaming app")
|
||||
} catch {
|
||||
print("virtual camera off: \(error)")
|
||||
}
|
||||
}
|
||||
guard recorder != nil || virtualCamera != nil || screenshotRequested || snapshotDue else { return }
|
||||
|
||||
commands.waitUntilCompleted()
|
||||
if pixels.count != width * height * 4 { pixels = [UInt8](repeating: 0, count: width * height * 4) }
|
||||
result.getBytes(&pixels, bytesPerRow: width * 4, from: MTLRegionMake2D(0, 0, width, height), mipmapLevel: 0)
|
||||
|
||||
recorder?.append(bgra: pixels, at: CACurrentMediaTime())
|
||||
virtualCamera?.send(bgra: pixels, width: width, height: height)
|
||||
if screenshotRequested {
|
||||
screenshotRequested = false
|
||||
let url = capturesURL(extension: "png")
|
||||
writePNG(bgra: &pixels, width: width, height: height, to: url)
|
||||
print("saved \(url.path)")
|
||||
}
|
||||
if snapshotDue {
|
||||
if let url = snapshot?.url {
|
||||
writePNG(bgra: &pixels, width: width, height: height, to: url)
|
||||
print("\(frameCount) frames -> \(url.path)")
|
||||
}
|
||||
finish()
|
||||
exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: drawing
|
||||
|
||||
private func process(_ frames: (current: MTLTexture, last: MTLTexture, serial: Int), _ commands: MTLCommandBuffer) throws {
|
||||
let width = frames.current.width, height = frames.current.height
|
||||
if analysis == nil { analysis = try Analysis(device: device, queue: queue, width: width, height: height) }
|
||||
guard let analysis else { return }
|
||||
analysis.update(commands, camera: frames.current, particles: layers.contains { $0.enabled && $0.usesParticles })
|
||||
|
||||
let time = Float(CACurrentMediaTime() - startTime)
|
||||
var image = frames.current
|
||||
for layer in layers where layer.enabled {
|
||||
guard let pipeline = layer.pipeline else { continue }
|
||||
if layer.targets.first?.width != width || layer.targets.first?.height != height {
|
||||
layer.targets = [makeTarget(device, width: width, height: height), makeTarget(device, width: width, height: height)]
|
||||
for target in layer.targets { commands.pass(into: target, clear: true) }
|
||||
}
|
||||
// must match `struct Uniforms` in the shader: time, padding, resolution.xy, params[16]
|
||||
var uniforms: [Float] = [time, 0, Float(width), Float(height)]
|
||||
uniforms += layer.params.map(\.value) + [Float](repeating: 0, count: maxParams - layer.params.count)
|
||||
let textures = [image, frames.current, frames.last, layer.targets[1], analysis.flow, analysis.particles]
|
||||
commands.pass(into: layer.targets[0]) { $0.fullscreen(pipeline, textures: textures, uniforms: uniforms) }
|
||||
image = layer.targets[0]
|
||||
layer.targets.swapAt(0, 1)
|
||||
}
|
||||
|
||||
if image === frames.current {
|
||||
if plain?.width != width || plain?.height != height { plain = makeTarget(device, width: width, height: height) }
|
||||
commands.pass(into: plain!) { $0.fullscreen(copyPipeline, textures: [image], uniforms: [0, 0, 0, 0]) }
|
||||
image = plain!
|
||||
}
|
||||
result = image
|
||||
}
|
||||
|
||||
/// The largest rectangle with the output's aspect ratio that fits the window, centred.
|
||||
private func fitted(_ output: MTLTexture, in size: CGSize) -> MTLViewport {
|
||||
let scale = min(size.width / Double(output.width), size.height / Double(output.height))
|
||||
let width = Double(output.width) * scale, height = Double(output.height) * scale
|
||||
return MTLViewport(originX: (size.width - width) / 2, originY: (size.height - height) / 2,
|
||||
width: width, height: height, znear: 0, zfar: 1)
|
||||
}
|
||||
|
||||
func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {}
|
||||
|
||||
func draw(in view: MTKView) {
|
||||
guard let frames = camera.latest, let commands = queue.makeCommandBuffer() else { return }
|
||||
|
||||
// effects advance once per camera frame; in between, the window just shows the last result again
|
||||
let isNewFrame = frames.serial != lastSerial
|
||||
if isNewFrame {
|
||||
lastSerial = frames.serial
|
||||
if frameCount % 15 == 0 { refreshFiles() }
|
||||
do {
|
||||
try process(frames, commands)
|
||||
} catch {
|
||||
print("render setup failed: \(error.localizedDescription)")
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
if let result, let drawable = view.currentDrawable, let windowPass = view.currentRenderPassDescriptor,
|
||||
let encoder = commands.makeRenderCommandEncoder(descriptor: windowPass) {
|
||||
encoder.setViewport(fitted(result, in: view.drawableSize))
|
||||
encoder.fullscreen(copyPipeline, textures: [result], uniforms: [0, 0, 0, 0])
|
||||
encoder.endEncoding()
|
||||
commands.present(drawable)
|
||||
}
|
||||
commands.commit()
|
||||
if isNewFrame { deliver(commands) }
|
||||
}
|
||||
}
|
||||
161
native/Sources/MotionLab/Shaders.swift
Normal file
161
native/Sources/MotionLab/Shaders.swift
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import Metal
|
||||
|
||||
/// A slider an effect file declares with a comment line: `// slider <name> <min> <max> <default>`
|
||||
struct EffectParam {
|
||||
let name: String
|
||||
let min: Float
|
||||
let max: Float
|
||||
var value: Float
|
||||
}
|
||||
|
||||
/// Everything an effect file says about itself in `//` comment lines:
|
||||
///
|
||||
/// // slider <name> <min> <max> <default> a slider, read in the shader as in.p.<name>
|
||||
/// // order <number> position in the chain, low runs first (default 50)
|
||||
/// // on enabled when the app starts
|
||||
struct EffectInfo {
|
||||
var params: [EffectParam] = []
|
||||
var order = 50
|
||||
var startsOn = false
|
||||
}
|
||||
|
||||
let maxParams = 16
|
||||
|
||||
func parseEffect(_ source: String) -> EffectInfo {
|
||||
var info = EffectInfo()
|
||||
for line in source.split(separator: "\n") {
|
||||
let parts = line.split(separator: " ")
|
||||
guard parts.count >= 2, parts[0] == "//" else { continue }
|
||||
switch parts[1] {
|
||||
case "slider" where parts.count == 6:
|
||||
if let min = Float(parts[3]), let max = Float(parts[4]), let value = Float(parts[5]) {
|
||||
info.params.append(EffectParam(name: String(parts[2]), min: min, max: max, value: value))
|
||||
}
|
||||
case "order" where parts.count == 3:
|
||||
info.order = Int(parts[2]) ?? info.order
|
||||
case "on" where parts.count == 2:
|
||||
info.startsOn = true
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
info.params = Array(info.params.prefix(maxParams))
|
||||
return info
|
||||
}
|
||||
|
||||
/// Shared by every shader in the app: a triangle that covers the whole target.
|
||||
let shaderPrelude = """
|
||||
#include <metal_stdlib>
|
||||
using namespace metal;
|
||||
|
||||
struct VertexOut {
|
||||
float4 position [[position]];
|
||||
float2 uv;
|
||||
};
|
||||
|
||||
vertex VertexOut fullscreenVertex(uint id [[vertex_id]]) {
|
||||
float2 p = float2((id << 1) & 2, id & 2);
|
||||
VertexOut out;
|
||||
out.position = float4(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0, 0.0, 1.0);
|
||||
out.uv = p;
|
||||
return out;
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
/// Wraps an effect file into a complete Metal shader. An effect file only defines:
|
||||
///
|
||||
/// float4 effect(float2 uv, Inputs in) { ... }
|
||||
///
|
||||
/// `uv` runs from (0,0) top-left to (1,1) bottom-right.
|
||||
func effectShader(body: String, params: [EffectParam]) -> String {
|
||||
let fields = params.isEmpty ? "float unused;" : params.map { "float \($0.name);" }.joined(separator: " ")
|
||||
let values = params.isEmpty ? "0.0" : params.indices.map { "u.params[\($0)]" }.joined(separator: ", ")
|
||||
return shaderPrelude + """
|
||||
struct Uniforms {
|
||||
float time;
|
||||
float2 resolution;
|
||||
float params[\(maxParams)];
|
||||
};
|
||||
|
||||
struct Params { \(fields) };
|
||||
|
||||
struct Inputs {
|
||||
texture2d<float> image; // the picture so far: output of the effect above, or the camera for the first one
|
||||
texture2d<float> camera; // the untouched camera frame
|
||||
texture2d<float> last; // the camera frame before this one (compare with camera to find motion)
|
||||
texture2d<float> prev; // this effect's own output from the previous frame (for trails)
|
||||
texture2d<float> flow; // .xy = which way and how far things moved since the last frame, in uv units
|
||||
texture2d<float> particles; // .r = brightness of particles that drift along with the motion
|
||||
sampler smp;
|
||||
float time; // seconds since start
|
||||
float2 resolution; // output size in pixels
|
||||
Params p; // this effect's sliders
|
||||
};
|
||||
|
||||
#line 1
|
||||
\(body)
|
||||
|
||||
fragment float4 effectFragment(VertexOut v [[stage_in]],
|
||||
texture2d<float> image [[texture(0)]],
|
||||
texture2d<float> camera [[texture(1)]],
|
||||
texture2d<float> last [[texture(2)]],
|
||||
texture2d<float> prev [[texture(3)]],
|
||||
texture2d<float> flow [[texture(4)]],
|
||||
texture2d<float> particles [[texture(5)]],
|
||||
constant Uniforms &u [[buffer(0)]]) {
|
||||
constexpr sampler smp(filter::linear, address::clamp_to_edge);
|
||||
Params p = { \(values) };
|
||||
Inputs in = { image, camera, last, prev, flow, particles, smp, u.time, u.resolution, p };
|
||||
return effect(v.uv, in);
|
||||
}
|
||||
"""
|
||||
}
|
||||
|
||||
func makePipeline(_ device: MTLDevice, source: String, vertex: String = "fullscreenVertex", fragment: String,
|
||||
format: MTLPixelFormat, additive: Bool = false) throws -> MTLRenderPipelineState {
|
||||
let library = try device.makeLibrary(source: source, options: nil)
|
||||
let descriptor = MTLRenderPipelineDescriptor()
|
||||
descriptor.vertexFunction = library.makeFunction(name: vertex)
|
||||
descriptor.fragmentFunction = library.makeFunction(name: fragment)
|
||||
let attachment = descriptor.colorAttachments[0]!
|
||||
attachment.pixelFormat = format
|
||||
if additive {
|
||||
attachment.isBlendingEnabled = true
|
||||
attachment.sourceRGBBlendFactor = .one
|
||||
attachment.destinationRGBBlendFactor = .one
|
||||
attachment.sourceAlphaBlendFactor = .one
|
||||
attachment.destinationAlphaBlendFactor = .one
|
||||
}
|
||||
return try device.makeRenderPipelineState(descriptor: descriptor)
|
||||
}
|
||||
|
||||
func makeTarget(_ device: MTLDevice, width: Int, height: Int, format: MTLPixelFormat = .bgra8Unorm) -> MTLTexture {
|
||||
let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width, height: height, mipmapped: false)
|
||||
descriptor.usage = [.renderTarget, .shaderRead]
|
||||
return device.makeTexture(descriptor: descriptor)!
|
||||
}
|
||||
|
||||
extension MTLCommandBuffer {
|
||||
/// One render pass into `target`. With `clear`, the target starts black; otherwise its old contents are ignored.
|
||||
func pass(into target: MTLTexture, clear: Bool = false, _ body: (MTLRenderCommandEncoder) -> Void = { _ in }) {
|
||||
let descriptor = MTLRenderPassDescriptor()
|
||||
descriptor.colorAttachments[0].texture = target
|
||||
descriptor.colorAttachments[0].loadAction = clear ? .clear : .dontCare
|
||||
descriptor.colorAttachments[0].storeAction = .store
|
||||
guard let encoder = makeRenderCommandEncoder(descriptor: descriptor) else { return }
|
||||
body(encoder)
|
||||
encoder.endEncoding()
|
||||
}
|
||||
}
|
||||
|
||||
extension MTLRenderCommandEncoder {
|
||||
func fullscreen(_ pipeline: MTLRenderPipelineState, textures: [MTLTexture], uniforms: [Float] = []) {
|
||||
setRenderPipelineState(pipeline)
|
||||
setFragmentTextures(textures, range: 0..<textures.count)
|
||||
if !uniforms.isEmpty {
|
||||
setFragmentBytes(uniforms, length: uniforms.count * MemoryLayout<Float>.stride, index: 0)
|
||||
}
|
||||
drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3)
|
||||
}
|
||||
}
|
||||
110
native/Sources/MotionLab/main.swift
Normal file
110
native/Sources/MotionLab/main.swift
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
// MotionLab: camera -> chain of Metal shader effects -> window, recording, virtual camera.
|
||||
//
|
||||
// swift run -c release MotionLab [options]
|
||||
//
|
||||
// --effects DIR folder of .metal effect files (default ./effects)
|
||||
// --only a,b,c start with exactly these effects enabled
|
||||
// --virtualcam also send the output to the OBS virtual camera (for Meet, Zoom, ...)
|
||||
// --snapshot OUT.png save one frame after --frames N camera frames (default 90) and exit
|
||||
// --record OUT.mp4 record --frames N camera frames (default 90) and exit
|
||||
//
|
||||
// In the window: s saves a screenshot, r starts/stops recording (both into ./captures), q or Esc quits.
|
||||
// Effect files are reloaded when saved.
|
||||
|
||||
import AppKit
|
||||
import MetalKit
|
||||
|
||||
func option(_ name: String) -> String? {
|
||||
let arguments = CommandLine.arguments
|
||||
guard let i = arguments.firstIndex(of: name), i + 1 < arguments.count else { return nil }
|
||||
return arguments[i + 1]
|
||||
}
|
||||
|
||||
func fail(_ message: String) -> Never {
|
||||
FileHandle.standardError.write((message + "\n").data(using: .utf8)!)
|
||||
exit(1)
|
||||
}
|
||||
|
||||
final class AppDelegate: NSObject, NSApplicationDelegate {
|
||||
var onQuit: (() -> Void)?
|
||||
func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
|
||||
func applicationWillTerminate(_ notification: Notification) { onQuit?() }
|
||||
}
|
||||
|
||||
setvbuf(stdout, nil, _IOLBF, 0) // show "loaded" / shader errors immediately, even when piped
|
||||
|
||||
guard let device = MTLCreateSystemDefaultDevice() else { fail("Metal is not available on this machine.") }
|
||||
let effectsDirectory = URL(fileURLWithPath: option("--effects") ?? "effects")
|
||||
|
||||
let renderer: Renderer
|
||||
do {
|
||||
renderer = try Renderer(device: device, camera: Camera(device: device), effectsDirectory: effectsDirectory)
|
||||
} catch {
|
||||
fail("\(error)")
|
||||
}
|
||||
if renderer.layers.isEmpty { print("No .metal files in \(effectsDirectory.path) - showing the plain camera.") }
|
||||
if let names = option("--only") {
|
||||
let unknown = renderer.enableOnly(names.split(separator: ",").map(String.init))
|
||||
if !unknown.isEmpty {
|
||||
fail("Unknown effect(s): \(unknown.joined(separator: ", ")). Available: \(renderer.layers.map(\.name).joined(separator: ", "))")
|
||||
}
|
||||
}
|
||||
if option("--snapshot") != nil || option("--record") != nil {
|
||||
if !renderer.failedLayers.isEmpty { fail("shader compile failed: \(renderer.failedLayers.joined(separator: ", "))") }
|
||||
renderer.snapshot = (Int(option("--frames") ?? "90") ?? 90, option("--snapshot").map { URL(fileURLWithPath: $0) })
|
||||
renderer.recordURL = option("--record").map { URL(fileURLWithPath: $0) }
|
||||
}
|
||||
renderer.wantsVirtualCamera = CommandLine.arguments.contains("--virtualcam")
|
||||
|
||||
let app = NSApplication.shared
|
||||
let delegate = AppDelegate()
|
||||
delegate.onQuit = { renderer.finish() }
|
||||
app.delegate = delegate
|
||||
app.setActivationPolicy(.regular)
|
||||
|
||||
let size = NSSize(width: 960 + Panel.width, height: 540)
|
||||
let window = NSWindow(contentRect: NSRect(origin: .zero, size: size),
|
||||
styleMask: [.titled, .closable, .miniaturizable, .resizable],
|
||||
backing: .buffered, defer: false)
|
||||
let view = MTKView(frame: .zero, device: device)
|
||||
view.colorPixelFormat = .bgra8Unorm
|
||||
view.delegate = renderer
|
||||
|
||||
let panel = Panel(renderer: renderer)
|
||||
renderer.onLayersChanged = { panel.rebuild() }
|
||||
renderer.onStatusChanged = { panel.refreshStatus() }
|
||||
|
||||
NSEvent.addLocalMonitorForEvents(matching: .keyDown) { event in
|
||||
if event.modifierFlags.contains(.command) { return event }
|
||||
switch event.charactersIgnoringModifiers {
|
||||
case "s": renderer.requestScreenshot()
|
||||
case "r": renderer.toggleRecording()
|
||||
case "q", "\u{1B}": app.terminate(nil)
|
||||
default: return event
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
let content = NSView(frame: NSRect(origin: .zero, size: size))
|
||||
for subview in [view, panel.view] {
|
||||
subview.translatesAutoresizingMaskIntoConstraints = false
|
||||
content.addSubview(subview)
|
||||
}
|
||||
NSLayoutConstraint.activate([
|
||||
view.leadingAnchor.constraint(equalTo: content.leadingAnchor),
|
||||
view.topAnchor.constraint(equalTo: content.topAnchor),
|
||||
view.bottomAnchor.constraint(equalTo: content.bottomAnchor),
|
||||
view.trailingAnchor.constraint(equalTo: panel.view.leadingAnchor),
|
||||
panel.view.trailingAnchor.constraint(equalTo: content.trailingAnchor),
|
||||
panel.view.topAnchor.constraint(equalTo: content.topAnchor),
|
||||
panel.view.bottomAnchor.constraint(equalTo: content.bottomAnchor),
|
||||
])
|
||||
|
||||
window.title = "MotionLab"
|
||||
window.contentMinSize = NSSize(width: 480 + Panel.width, height: 270)
|
||||
window.contentView = content
|
||||
window.setContentSize(size) // auto layout would otherwise pick its own window size
|
||||
window.center()
|
||||
window.makeKeyAndOrderFront(nil)
|
||||
app.activate(ignoringOtherApps: true)
|
||||
app.run()
|
||||
12
native/effects/colorize.metal
Normal file
12
native/effects/colorize.metal
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
// Recolours the picture by brightness with a smooth colour ramp.
|
||||
// order 10
|
||||
// slider amount 0.0 1.0 0.8
|
||||
// slider palette 0.0 1.0 0.0
|
||||
// slider contrast 0.5 3.0 1.2
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float3 c = in.image.sample(in.smp, uv).rgb;
|
||||
float luma = pow(dot(c, float3(0.299, 0.587, 0.114)), in.p.contrast);
|
||||
float3 ramp = 0.5 + 0.5 * cos(6.2832 * (luma * 0.8 + in.p.palette + float3(0.55, 0.4, 0.2)));
|
||||
float3 mapped = ramp * (0.15 + 0.85 * luma);
|
||||
return float4(mix(c, mapped, in.p.amount), 1.0);
|
||||
}
|
||||
18
native/effects/halftone.metal
Normal file
18
native/effects/halftone.metal
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
// Halftone dots sized by brightness; dots turn hot orange wherever something moves.
|
||||
// order 70
|
||||
// slider cell 4.0 40.0 12.0
|
||||
// slider heat 0.0 20.0 8.0
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float2 grid = in.resolution / in.p.cell;
|
||||
float2 centre = (floor(uv * grid) + 0.5) / grid;
|
||||
float3 c = in.image.sample(in.smp, centre).rgb;
|
||||
float luma = dot(c, float3(0.299, 0.587, 0.114));
|
||||
|
||||
float radius = 0.5 * sqrt(luma);
|
||||
float dist = length(fract(uv * grid) - 0.5);
|
||||
float mask = 1.0 - smoothstep(radius - 0.05, radius + 0.05, dist);
|
||||
|
||||
float moved = length(in.camera.sample(in.smp, centre).rgb - in.last.sample(in.smp, centre).rgb);
|
||||
float3 hot = mix(c, float3(1.0, 0.35, 0.1), clamp(moved * in.p.heat, 0.0, 1.0));
|
||||
return float4(hot * mask, 1.0);
|
||||
}
|
||||
13
native/effects/kaleidoscope.metal
Normal file
13
native/effects/kaleidoscope.metal
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
// Folds the picture into mirrored wedges around the centre.
|
||||
// order 90
|
||||
// slider segments 2.0 16.0 6.0
|
||||
// slider spin -2.0 2.0 0.2
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float2 shape = float2(in.resolution.x / in.resolution.y, 1.0);
|
||||
float2 c = (uv - 0.5) * shape;
|
||||
float wedge = 6.2832 / floor(in.p.segments);
|
||||
float a = atan2(c.y, c.x) + in.time * in.p.spin;
|
||||
a = abs(a - wedge * floor(a / wedge) - wedge * 0.5) - 1.5708; // fold into one mirrored wedge, pointing up
|
||||
float2 from = float2(cos(a), sin(a)) * length(c) / shape + 0.5;
|
||||
return float4(in.image.sample(in.smp, from).rgb, 1.0);
|
||||
}
|
||||
13
native/effects/liquid.metal
Normal file
13
native/effects/liquid.metal
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
// Liquid trails: the previous output is dragged along the motion, swirled a little,
|
||||
// and slowly fades back to the live picture.
|
||||
// order 45
|
||||
// slider persistence 0.0 0.99 0.92
|
||||
// slider drag 0.0 6.0 2.5
|
||||
// slider swirl 0.0 0.01 0.002
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float2 centred = uv - 0.5;
|
||||
float2 from = uv - in.flow.sample(in.smp, uv).xy * in.p.drag + float2(-centred.y, centred.x) * in.p.swirl;
|
||||
float3 trail = in.prev.sample(in.smp, from).rgb;
|
||||
float3 live = in.image.sample(in.smp, uv).rgb;
|
||||
return float4(mix(live, trail, in.p.persistence), 1.0);
|
||||
}
|
||||
20
native/effects/motion-glow.metal
Normal file
20
native/effects/motion-glow.metal
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
// Whatever moves lights up and the glow lingers. The heat is carried from frame to frame
|
||||
// in the alpha channel of this effect's own previous output.
|
||||
// order 50
|
||||
// slider sensitivity 1.0 30.0 10.0
|
||||
// slider linger 0.5 0.99 0.93
|
||||
// slider dim 0.0 1.0 0.45
|
||||
// slider hue 0.0 1.0 0.07
|
||||
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
|
||||
static float3 hueColor(float h) {
|
||||
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
|
||||
}
|
||||
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float moved = length(in.camera.sample(in.smp, uv).rgb - in.last.sample(in.smp, uv).rgb);
|
||||
float fresh = smoothstep(0.3, 1.0, moved * in.p.sensitivity);
|
||||
float heat = max(in.prev.sample(in.smp, uv).a * in.p.linger, fresh);
|
||||
|
||||
float3 glow = hueColor(in.p.hue);
|
||||
return float4(in.image.sample(in.smp, uv).rgb * in.p.dim + glow * heat, heat);
|
||||
}
|
||||
38
native/effects/neon-edges.metal
Normal file
38
native/effects/neon-edges.metal
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
// Glowing outlines with a slowly cycling colour.
|
||||
// order 20
|
||||
// slider detail 1.0 12.0 5.0
|
||||
// slider glow 0.0 6.0 2.5
|
||||
// slider source 0.0 1.0 0.35
|
||||
// slider hue 0.0 1.0 0.55
|
||||
// slider cycle 0.0 1.0 0.3
|
||||
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
|
||||
static float3 hueColor(float h) {
|
||||
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
|
||||
}
|
||||
|
||||
static float luma(Inputs in, float2 uv) {
|
||||
return dot(in.image.sample(in.smp, uv).rgb, float3(0.299, 0.587, 0.114));
|
||||
}
|
||||
|
||||
// how strongly brightness changes around this point
|
||||
static float edge(Inputs in, float2 uv) {
|
||||
float2 t = 1.5 / in.resolution;
|
||||
float dx = luma(in, uv + float2(t.x, 0.0)) - luma(in, uv - float2(t.x, 0.0));
|
||||
float dy = luma(in, uv + float2(0.0, t.y)) - luma(in, uv - float2(0.0, t.y));
|
||||
return length(float2(dx, dy));
|
||||
}
|
||||
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float sharp = smoothstep(0.1, 0.5, edge(in, uv) * in.p.detail);
|
||||
float halo = 0.0; // edges found in a ring around this pixel = a cheap glow
|
||||
for (int i = 0; i < 12; i++) {
|
||||
float a = 6.2832 * float(i) / 12.0;
|
||||
float2 dir = float2(cos(a), sin(a)) / in.resolution;
|
||||
halo += edge(in, uv + dir * 4.0) + edge(in, uv + dir * 9.0);
|
||||
}
|
||||
halo = halo / 24.0 * in.p.detail * in.p.glow;
|
||||
|
||||
float3 tint = hueColor(in.p.hue + in.time * in.p.cycle * 0.2);
|
||||
float3 base = in.image.sample(in.smp, uv).rgb * in.p.source;
|
||||
return float4(base + tint * (sharp + halo), 1.0);
|
||||
}
|
||||
22
native/effects/particles.metal
Normal file
22
native/effects/particles.metal
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
// Particles spawn where you move and drift along with the motion.
|
||||
// order 30
|
||||
// on
|
||||
// slider brightness 0.0 3.0 1.4
|
||||
// slider glow 0.0 4.0 1.5
|
||||
// slider hue 0.0 1.0 0.08
|
||||
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
|
||||
static float3 hueColor(float h) {
|
||||
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
|
||||
}
|
||||
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float core = in.particles.sample(in.smp, uv).r;
|
||||
float halo = 0.0;
|
||||
for (int i = 0; i < 8; i++) {
|
||||
float a = 6.2832 * float(i) / 8.0;
|
||||
halo += in.particles.sample(in.smp, uv + float2(cos(a), sin(a)) * 5.0 / in.resolution).r;
|
||||
}
|
||||
float3 tint = hueColor(in.p.hue);
|
||||
float3 base = in.image.sample(in.smp, uv).rgb;
|
||||
return float4(base + tint * (core * in.p.brightness + halo / 8.0 * in.p.glow), 1.0);
|
||||
}
|
||||
16
native/effects/trails.metal
Normal file
16
native/effects/trails.metal
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
// Feedback trails: the previous output is zoomed, turned and faded, so bright things leave streaks.
|
||||
// "push" smears the trails in the direction things are moving.
|
||||
// order 40
|
||||
// on
|
||||
// slider fade 0.5 0.995 0.92
|
||||
// slider zoom -0.04 0.04 0.012
|
||||
// slider spin -0.05 0.05 0.005
|
||||
// slider push 0.0 4.0 0.0
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float2 c = uv - 0.5;
|
||||
float2 turned = float2(c.x * cos(in.p.spin) - c.y * sin(in.p.spin), c.x * sin(in.p.spin) + c.y * cos(in.p.spin));
|
||||
float2 from = turned * (1.0 - in.p.zoom) + 0.5 - in.flow.sample(in.smp, uv).xy * in.p.push;
|
||||
float3 trail = in.prev.sample(in.smp, from).rgb * in.p.fade;
|
||||
float3 live = in.image.sample(in.smp, uv).rgb;
|
||||
return float4(max(live, trail), 1.0);
|
||||
}
|
||||
13
native/effects/wave-shift.metal
Normal file
13
native/effects/wave-shift.metal
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
// RGB split that ripples down the image and gets pushed around by your movement.
|
||||
// order 60
|
||||
// slider amount 0.0 0.05 0.012
|
||||
// slider waves 1.0 80.0 24.0
|
||||
// slider speed 0.0 10.0 3.0
|
||||
// slider react 0.0 3.0 1.0
|
||||
float4 effect(float2 uv, Inputs in) {
|
||||
float2 d = float2(in.p.amount * sin(uv.y * in.p.waves + in.time * in.p.speed), 0.0);
|
||||
d += in.flow.sample(in.smp, uv).xy * in.p.react;
|
||||
return float4(in.image.sample(in.smp, uv + d).r,
|
||||
in.image.sample(in.smp, uv).g,
|
||||
in.image.sample(in.smp, uv - d).b, 1.0);
|
||||
}
|
||||
61
panel.py
Normal file
61
panel.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
"""Control panel drawn straight onto the output canvas: click a layer to toggle it, drag its sliders."""
|
||||
import cv2
|
||||
|
||||
BG = (26, 24, 22)
|
||||
TRACK = (52, 48, 46)
|
||||
FG = (228, 228, 228)
|
||||
DIM = (125, 125, 125)
|
||||
ACCENT = (255, 176, 64)
|
||||
FONT = cv2.FONT_HERSHEY_SIMPLEX
|
||||
|
||||
|
||||
class Panel:
|
||||
WIDTH = 260
|
||||
|
||||
def __init__(self, layers):
|
||||
self.layers = layers
|
||||
self.hits = [] # (x0, y0, x1, y1, target, bar_x0, bar_x1), rebuilt on every draw
|
||||
self.drag = None
|
||||
|
||||
def draw(self, canvas, x0, status):
|
||||
h = canvas.shape[0]
|
||||
canvas[:, x0:] = BG
|
||||
self.hits = []
|
||||
x, right = x0 + 14, x0 + self.WIDTH - 14
|
||||
cv2.putText(canvas, "MOTIONLAB", (x, 26), FONT, 0.55, FG, 1, cv2.LINE_AA)
|
||||
cv2.putText(canvas, status, (x + 124, 26), FONT, 0.4, DIM, 1, cv2.LINE_AA)
|
||||
|
||||
y = 44
|
||||
for layer in self.layers:
|
||||
on = layer.enabled
|
||||
cv2.rectangle(canvas, (x, y), (x + 12, y + 12), ACCENT if on else DIM, -1 if on else 1)
|
||||
cv2.putText(canvas, layer.name, (x + 22, y + 11), FONT, 0.48, FG if on else DIM, 1, cv2.LINE_AA)
|
||||
self.hits.append((x0, y - 5, x0 + self.WIDTH, y + 17, layer, 0, 0))
|
||||
y += 22
|
||||
if not on:
|
||||
continue
|
||||
bar = x + 100
|
||||
for name, p in layer.params.items():
|
||||
cv2.putText(canvas, name, (x + 4, y + 8), FONT, 0.36, DIM, 1, cv2.LINE_AA)
|
||||
cv2.rectangle(canvas, (bar, y), (right, y + 8), TRACK, -1)
|
||||
cv2.rectangle(canvas, (bar, y), (bar + int((right - bar) * p.norm), y + 8), ACCENT, -1)
|
||||
self.hits.append((bar - 8, y - 4, right + 8, y + 12, p, bar, right))
|
||||
y += 16
|
||||
y += 4
|
||||
|
||||
cv2.putText(canvas, "[h] panel [s] shot [r] rec [q] quit", (x, h - 12), FONT, 0.36, DIM, 1, cv2.LINE_AA)
|
||||
|
||||
def on_mouse(self, event, mx, my, flags, _):
|
||||
if event == cv2.EVENT_LBUTTONDOWN:
|
||||
for x0, y0, x1, y1, target, bar_x0, bar_x1 in self.hits:
|
||||
if x0 <= mx < x1 and y0 <= my < y1:
|
||||
if bar_x1:
|
||||
self.drag = (target, bar_x0, bar_x1)
|
||||
else:
|
||||
target.enabled = not target.enabled
|
||||
break
|
||||
elif event == cv2.EVENT_LBUTTONUP:
|
||||
self.drag = None
|
||||
if self.drag and event in (cv2.EVENT_LBUTTONDOWN, cv2.EVENT_MOUSEMOVE):
|
||||
param, bar_x0, bar_x1 = self.drag
|
||||
param.norm = (mx - bar_x0) / (bar_x1 - bar_x0)
|
||||
172
shaders.py
Normal file
172
shaders.py
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
"""Shader layers: every effects/*.glsl file becomes a layer, reloaded whenever the file is saved.
|
||||
|
||||
A shader file only contains its own sliders and a main(). The host supplies the rest:
|
||||
|
||||
in vec2 uv; pixel position, (0,0) top-left .. (1,1) bottom-right
|
||||
out vec4 color; what to write for this pixel
|
||||
uniform sampler2D camera; the incoming frame (output of the layer above), RGB
|
||||
uniform sampler2D prev; this layer's own output from the previous frame
|
||||
uniform sampler2D flow; .xy = how far this pixel moved since last frame, in uv units
|
||||
uniform sampler2D motion; .r = how much this pixel changed since last frame, 0..1
|
||||
uniform float time; seconds since start
|
||||
uniform float energy; overall amount of motion in the frame, 0..1
|
||||
uniform vec2 resolution; output size in pixels
|
||||
|
||||
Sliders are plain float uniforms with a range comment:
|
||||
|
||||
uniform float amount; // 0.0 .. 0.05 = 0.01
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
|
||||
import moderngl
|
||||
import numpy as np
|
||||
|
||||
from effects import Layer, Param
|
||||
|
||||
VERTEX = """#version 410
|
||||
out vec2 uv;
|
||||
void main() {
|
||||
uv = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2);
|
||||
gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0);
|
||||
}
|
||||
"""
|
||||
|
||||
# "#line 1" makes compiler errors report line numbers of the user's file
|
||||
HEADER = """#version 410
|
||||
uniform sampler2D camera;
|
||||
uniform sampler2D prev;
|
||||
uniform sampler2D flow;
|
||||
uniform sampler2D motion;
|
||||
uniform float time;
|
||||
uniform float energy;
|
||||
uniform vec2 resolution;
|
||||
in vec2 uv;
|
||||
out vec4 color;
|
||||
#line 1
|
||||
"""
|
||||
|
||||
NUM = r"(-?[\d.]+)"
|
||||
PARAM_RE = re.compile(rf"uniform\s+float\s+(\w+)\s*;(?:[ \t]*//\s*{NUM}\s*\.\.\s*{NUM}\s*=\s*{NUM})?")
|
||||
|
||||
|
||||
class ShaderLayer(Layer):
|
||||
enabled = False
|
||||
|
||||
def __init__(self, gl, path):
|
||||
super().__init__()
|
||||
self.gl, self.path = gl, path
|
||||
self.name = os.path.splitext(os.path.basename(path))[0]
|
||||
self.program = None
|
||||
self.error = None
|
||||
self.mtime = None
|
||||
self.targets = None
|
||||
self.reload()
|
||||
|
||||
def reload(self):
|
||||
"""Recompile if the file changed. On a compile error the last working version keeps running."""
|
||||
try:
|
||||
mtime = os.stat(self.path).st_mtime
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if mtime == self.mtime:
|
||||
return
|
||||
self.mtime = mtime
|
||||
with open(self.path) as f:
|
||||
source = f.read()
|
||||
try:
|
||||
program = self.gl.program(vertex_shader=VERTEX, fragment_shader=HEADER + source)
|
||||
except moderngl.Error as e:
|
||||
self.error = str(e)
|
||||
print(f"[{self.name}] shader error:\n{e}", flush=True)
|
||||
return
|
||||
self.error = None
|
||||
self.program = program
|
||||
self.vao = self.gl.vertex_array(program, [])
|
||||
old = self.params
|
||||
self.params = {name: Param(float(default or 0.5), float(lo or 0), float(hi or 1))
|
||||
for name, lo, hi, default in PARAM_RE.findall(source)}
|
||||
for name, p in self.params.items(): # keep slider positions across reloads
|
||||
if name in old:
|
||||
p.value = min(max(old[name].value, p.lo), p.hi)
|
||||
print(f"[{self.name}] loaded", flush=True)
|
||||
|
||||
def _target(self, size):
|
||||
texture = self.gl.texture(size, 4)
|
||||
texture.repeat_x = texture.repeat_y = False
|
||||
return texture, self.gl.framebuffer([texture])
|
||||
|
||||
def _texture(self, attr, size, components, dtype="f1"):
|
||||
if not hasattr(self, attr):
|
||||
texture = self.gl.texture(size, components, dtype=dtype)
|
||||
texture.repeat_x = texture.repeat_y = False
|
||||
setattr(self, attr, texture)
|
||||
return getattr(self, attr)
|
||||
|
||||
def _set(self, name, value):
|
||||
uniform = self.program.get(name, None) # absent if the shader doesn't use it
|
||||
if uniform is not None:
|
||||
uniform.value = value
|
||||
return uniform is not None
|
||||
|
||||
def apply(self, frame, ctx):
|
||||
if self.program is None:
|
||||
return frame
|
||||
h, w = frame.shape[:2]
|
||||
if self.targets is None:
|
||||
# two render targets, swapped every frame, so the shader can read its previous output
|
||||
self.targets = [self._target((w, h)), self._target((w, h))]
|
||||
(_, framebuffer), (prev, _) = self.targets
|
||||
|
||||
camera = self._texture("camera", (w, h), 3)
|
||||
camera.swizzle = "BGR1" # frames are BGR; shaders see RGB
|
||||
camera.write(np.ascontiguousarray(frame))
|
||||
camera.use(0)
|
||||
self._set("camera", 0)
|
||||
prev.use(1)
|
||||
self._set("prev", 1)
|
||||
if self._set("flow", 2):
|
||||
fh, fw = ctx.flow.shape[:2]
|
||||
flow = self._texture("flow_tex", (fw, fh), 2, "f4")
|
||||
flow.write((ctx.flow / (fw, fh)).astype("f4"))
|
||||
flow.use(2)
|
||||
if self._set("motion", 3):
|
||||
mh, mw = ctx.motion.shape
|
||||
motion = self._texture("motion_tex", (mw, mh), 1)
|
||||
motion.write(ctx.motion)
|
||||
motion.use(3)
|
||||
self._set("time", ctx.t)
|
||||
self._set("energy", ctx.energy)
|
||||
self._set("resolution", (w, h))
|
||||
for name, p in self.params.items():
|
||||
self._set(name, p.value)
|
||||
|
||||
framebuffer.use()
|
||||
self.vao.render(moderngl.TRIANGLES, vertices=3)
|
||||
out = np.frombuffer(framebuffer.read(components=3), np.uint8).reshape(h, w, 3)
|
||||
self.targets.reverse()
|
||||
return np.ascontiguousarray(out[:, :, ::-1])
|
||||
|
||||
|
||||
class ShaderBank:
|
||||
"""Watches a folder of .glsl files and keeps one ShaderLayer per file in the shared layer list."""
|
||||
|
||||
def __init__(self, directory, layers):
|
||||
self.directory, self.layers = directory, layers
|
||||
self.known = {}
|
||||
self.gl = moderngl.create_context(standalone=True, require=410)
|
||||
self.refresh()
|
||||
|
||||
def refresh(self):
|
||||
if os.path.isdir(self.directory):
|
||||
for filename in sorted(os.listdir(self.directory)):
|
||||
path = os.path.join(self.directory, filename)
|
||||
if filename.endswith(".glsl") and path not in self.known:
|
||||
self.known[path] = ShaderLayer(self.gl, path)
|
||||
self.layers.append(self.known[path])
|
||||
for layer in self.known.values():
|
||||
layer.reload()
|
||||
|
||||
@property
|
||||
def errors(self):
|
||||
return [layer.name for layer in self.known.values() if layer.error]
|
||||
Loading…
Add table
Add a link
Reference in a new issue