0xrsydn/motionlab
1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Python/OpenCV proof of concept plus first native Swift port

Browse source 
Snapshot before the Python POC is removed. The POC (main.py, effects.py,
panel.py, shaders.py, effects/*.glsl, flake.nix) is the reference for the
native app under native/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Rasyidan Akbar F. 2026-10-05 20:26:47 +07:00
commit 6591138bbf
27 changed files with 2076 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

10
native/Package.swift Normal file
View file

@ -0,0 +1,10 @@
// swift-tools-version:5.9
import PackageDescription
let package = Package(
name: "MotionLab",
platforms: [.macOS(.v13)],
targets: [
.executableTarget(name: "MotionLab", path: "Sources/MotionLab")
]
)

180
native/Sources/MotionLab/Analysis.swift Normal file
View file

@ -0,0 +1,180 @@
import Metal
/// Host-side motion analysis, shared by all effects. Every camera frame it produces:
///
/// flow which way things moved (small 2-channel texture, uv units per frame)
/// particles a picture of particles that spawn where something moves and drift along the flow
///
/// Effects read these as `in.flow` and `in.particles`.
final class Analysis {
private static let flowWidth = 160, flowHeight = 90
private static let stateWidth = 128, stateHeight = 64 // one texel per particle
private static let particleSpeed: Float = 1.5
private static let trailFade: Float = 0.88
// each pair is [this frame, previous frame], swapped at the start of every update
private var luma: [MTLTexture]
private var flowTextures: [MTLTexture]
private var state: [MTLTexture]
private var trail: [MTLTexture]
private let lumaPipeline: MTLRenderPipelineState
private let flowPipeline: MTLRenderPipelineState
private let movePipeline: MTLRenderPipelineState
private let fadePipeline: MTLRenderPipelineState
private let dotPipeline: MTLRenderPipelineState
var flow: MTLTexture { flowTextures[0] }
var particles: MTLTexture { trail[0] }
init(device: MTLDevice, queue: MTLCommandQueue, width: Int, height: Int) throws {
func pair(_ w: Int, _ h: Int, _ format: MTLPixelFormat) -> [MTLTexture] {
[makeTarget(device, width: w, height: h, format: format), makeTarget(device, width: w, height: h, format: format)]
}
luma = pair(Analysis.flowWidth, Analysis.flowHeight, .r16Float)
flowTextures = pair(Analysis.flowWidth, Analysis.flowHeight, .rg16Float)
state = pair(Analysis.stateWidth, Analysis.stateHeight, .rgba32Float)
trail = pair(width, height, .r8Unorm)
let source = shaderPrelude + analysisShaders
lumaPipeline = try makePipeline(device, source: source, fragment: "lumaFragment", format: .r16Float)
flowPipeline = try makePipeline(device, source: source, fragment: "flowFragment", format: .rg16Float)
movePipeline = try makePipeline(device, source: source, fragment: "moveFragment", format: .rgba32Float)
fadePipeline = try makePipeline(device, source: source, fragment: "fadeFragment", format: .r8Unorm)
dotPipeline = try makePipeline(device, source: source, vertex: "dotVertex", fragment: "dotFragment",
format: .r8Unorm, additive: true)
// new textures hold garbage; start everything from zero
if let commands = queue.makeCommandBuffer() {
for texture in luma + flowTextures + state + trail { commands.pass(into: texture, clear: true) }
commands.commit()
}
}
func update(_ commands: MTLCommandBuffer, camera: MTLTexture, particles wantsParticles: Bool) {
luma.swapAt(0, 1)
commands.pass(into: luma[0]) { $0.fullscreen(lumaPipeline, textures: [camera]) }
flowTextures.swapAt(0, 1)
commands.pass(into: flowTextures[0]) {
$0.fullscreen(flowPipeline, textures: [luma[0], luma[1], flowTextures[1]])
}
guard wantsParticles else { return }
state.swapAt(0, 1)
commands.pass(into: state[0]) {
$0.fullscreen(movePipeline, textures: [state[1], flowTextures[0], luma[0], luma[1]],
uniforms: [Float.random(in: 0..<1000), Analysis.particleSpeed, 0, 0])
}
trail.swapAt(0, 1)
commands.pass(into: trail[0]) { encoder in
encoder.fullscreen(fadePipeline, textures: [trail[1]], uniforms: [Analysis.trailFade, 0, 0, 0])
encoder.setRenderPipelineState(dotPipeline)
encoder.setVertexTexture(state[0], index: 0)
encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: Analysis.stateWidth * Analysis.stateHeight)
}
}
}
private let analysisShaders = """
static float brightness(float3 c) { return dot(c, float3(0.299, 0.587, 0.114)); }
static float hash(float2 p) { return fract(sin(dot(p, float2(127.1, 311.7))) * 43758.5453); }
// Camera -> small, blurred brightness image. Flow is measured on this.
fragment float4 lumaFragment(VertexOut v [[stage_in]], texture2d<float> camera [[texture(0)]]) {
constexpr sampler s(filter::linear, address::clamp_to_edge);
float2 texel = 1.0 / float2(camera.get_width(), camera.get_height());
float sum = 0.0;
for (int y = 0; y < 4; y++) {
for (int x = 0; x < 4; x++) {
sum += brightness(camera.sample(s, v.uv + (float2(x, y) - 1.5) * 2.0 * texel).rgb);
}
}
return float4(sum / 16.0);
}
// Lucas-Kanade optical flow: for each pixel, the shift that best explains how its 7x7
// neighbourhood changed between the previous and the current brightness image.
fragment float4 flowFragment(VertexOut v [[stage_in]],
texture2d<float> now [[texture(0)]],
texture2d<float> before [[texture(1)]],
texture2d<float> flowBefore [[texture(2)]]) {
constexpr sampler s(filter::linear, address::clamp_to_edge);
float2 texel = 1.0 / float2(now.get_width(), now.get_height());
float a = 1e-3, b = 0.0, c = 1e-3, p = 0.0, q = 0.0; // the 1e-3 keeps flat areas at zero flow
for (int y = -3; y <= 3; y++) {
for (int x = -3; x <= 3; x++) {
float2 uv = v.uv + float2(x, y) * texel;
float ix = (now.sample(s, uv + float2(texel.x, 0.0)).r - now.sample(s, uv - float2(texel.x, 0.0)).r) * 0.5;
float iy = (now.sample(s, uv + float2(0.0, texel.y)).r - now.sample(s, uv - float2(0.0, texel.y)).r) * 0.5;
float it = now.sample(s, uv).r - before.sample(s, uv).r;
a += ix * ix; b += ix * iy; c += iy * iy;
p += ix * it; q += iy * it;
}
}
float2 shift = -float2(c * p - b * q, a * q - b * p) / (a * c - b * b); // in small-image pixels
float len = length(shift);
if (len > 4.0) { shift *= 4.0 / len; }
float2 smoothed = mix(flowBefore.sample(s, v.uv).xy, shift * texel, 0.6);
return float4(smoothed, 0.0, 1.0);
}
// One texel per particle: xy = position (uv), z = frames left to live, w = 1 if it was born from motion.
fragment float4 moveFragment(VertexOut v [[stage_in]],
texture2d<float> state [[texture(0)]],
texture2d<float> flow [[texture(1)]],
texture2d<float> now [[texture(2)]],
texture2d<float> before [[texture(3)]],
constant float4 &u [[buffer(0)]]) { // x = random seed, y = speed
constexpr sampler s(filter::linear, address::clamp_to_edge);
float4 particle = state.read(uint2(v.position.xy));
float2 seed = v.position.xy * 0.37 + u.x;
float2 pos = particle.xy + flow.sample(s, particle.xy).xy * u.y;
pos += (float2(hash(seed), hash(seed + 7.3)) - 0.5) * 0.0015;
float life = particle.z - 1.0;
float born = particle.w;
if (life <= 0.0 || pos.x < 0.0 || pos.x > 1.0 || pos.y < 0.0 || pos.y > 1.0) {
// respawn: try a few random spots and take the first one where something is moving
pos = float2(hash(seed + 1.1), hash(seed + 2.2));
born = 0.0;
life = 2.0 + hash(seed + 5.5) * 4.0; // nothing moving: stay hidden and try again soon
for (int i = 0; i < 6; i++) {
float2 spot = float2(hash(seed + 3.3 + float(i)), hash(seed + 9.9 + float(i) * 1.7));
if (abs(now.sample(s, spot).r - before.sample(s, spot).r) > 0.04) {
pos = spot;
born = 1.0;
life = 20.0 + hash(seed + 5.5) * 70.0;
break;
}
}
}
return float4(pos, life, born);
}
// Particle trails: fade what was there, then add a dot per particle on top.
fragment float4 fadeFragment(VertexOut v [[stage_in]], texture2d<float> trail [[texture(0)]],
constant float4 &u [[buffer(0)]]) { // x = fade
constexpr sampler s(filter::linear, address::clamp_to_edge);
return trail.sample(s, v.uv) * u.x;
}
struct DotOut {
float4 position [[position]];
float size [[point_size]];
};
vertex DotOut dotVertex(uint id [[vertex_id]], texture2d<float> state [[texture(0)]]) {
uint width = state.get_width();
float4 particle = state.read(uint2(id % width, id / width));
DotOut out;
out.position = float4(particle.x * 2.0 - 1.0, 1.0 - particle.y * 2.0, 0.0, 1.0);
if (particle.w < 0.5) { out.position.x = 4.0; } // hidden particles are moved off screen
out.size = 3.0;
return out;
}
fragment float4 dotFragment(DotOut in [[stage_in]], float2 coord [[point_coord]]) {
return float4(smoothstep(0.5, 0.1, length(coord - 0.5)) * 0.9);
}
"""

95
native/Sources/MotionLab/Camera.swift Normal file
View file

@ -0,0 +1,95 @@
import AVFoundation
import CoreVideo
import Metal
enum CameraError: Error, CustomStringConvertible {
case noPermission
case noCamera
var description: String {
switch self {
case .noPermission:
return "No camera permission. Allow your terminal under System Settings > Privacy & Security > Camera, then restart it."
case .noCamera:
return "No camera found."
}
}
}
/// Streams the built-in camera as Metal textures; `latest` is the newest frame and the one before it.
final class Camera: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
private let session = AVCaptureSession()
private var cache: CVMetalTextureCache?
private let lock = NSLock()
// each entry keeps its CVMetalTexture, which keeps the pixel buffer behind the Metal texture alive
private var current: (texture: MTLTexture, backing: CVMetalTexture)?
private var previous: (texture: MTLTexture, backing: CVMetalTexture)?
private var serial = 0 // counts frames, so the renderer can tell a new frame from one it already saw
var latest: (current: MTLTexture, last: MTLTexture, serial: Int)? {
lock.lock()
defer { lock.unlock() }
guard let current else { return nil }
return (current.texture, (previous ?? current).texture, serial)
}
init(device: MTLDevice) throws {
super.init()
guard Camera.requestAccess() else { throw CameraError.noPermission }
// prefer the built-in camera over an iPhone that happens to be nearby
guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .unspecified)
?? AVCaptureDevice.default(for: .video)
else { throw CameraError.noCamera }
CVMetalTextureCacheCreate(nil, nil, device, nil, &cache)
let output = AVCaptureVideoDataOutput()
output.videoSettings = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
kCVPixelBufferMetalCompatibilityKey as String: true,
]
output.alwaysDiscardsLateVideoFrames = true
output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "motionlab.camera"))
session.beginConfiguration()
session.addInput(try AVCaptureDeviceInput(device: camera))
session.addOutput(output)
if session.canSetSessionPreset(.hd1280x720) { session.sessionPreset = .hd1280x720 }
print("camera: \(camera.localizedName)")
if let connection = output.connection(with: .video), connection.isVideoMirroringSupported {
connection.automaticallyAdjustsVideoMirroring = false
connection.isVideoMirrored = true // behave like a mirror
}
session.commitConfiguration()
session.startRunning()
}
private static func requestAccess() -> Bool {
switch AVCaptureDevice.authorizationStatus(for: .video) {
case .authorized:
return true
case .notDetermined:
let done = DispatchSemaphore(value: 0)
var granted = false
AVCaptureDevice.requestAccess(for: .video) { granted = $0; done.signal() }
done.wait()
return granted
default:
return false
}
}
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let cache, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
var created: CVMetalTexture?
CVMetalTextureCacheCreateTextureFromImage(
nil, cache, buffer, nil, .bgra8Unorm,
CVPixelBufferGetWidth(buffer), CVPixelBufferGetHeight(buffer), 0, &created)
guard let created, let frame = CVMetalTextureGetTexture(created) else { return }
lock.lock()
previous = current
current = (frame, created)
serial += 1
lock.unlock()
}
}

198
native/Sources/MotionLab/Output.swift Normal file
View file

@ -0,0 +1,198 @@
import AVFoundation
import CoreMediaIO
import ImageIO
import UniformTypeIdentifiers
func writePNG(bgra pixels: inout [UInt8], width: Int, height: Int, to url: URL) {
let bitmapInfo = CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue
guard let context = CGContext(data: &pixels, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4,
space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: bitmapInfo),
let image = context.makeImage(),
let destination = CGImageDestinationCreateWithURL(url as CFURL, UTType.png.identifier as CFString, 1, nil)
else { return }
CGImageDestinationAddImage(destination, image, nil)
CGImageDestinationFinalize(destination)
}
/// Writes frames to an H.264 .mp4 file.
final class Recorder {
let url: URL
private let writer: AVAssetWriter
private let input: AVAssetWriterInput
private let adaptor: AVAssetWriterInputPixelBufferAdaptor
private let width: Int, height: Int
private var startTime: Double?
init(url: URL, width: Int, height: Int) throws {
self.url = url
self.width = width
self.height = height
writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
input = AVAssetWriterInput(mediaType: .video, outputSettings: [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
])
input.expectsMediaDataInRealTime = true
adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
kCVPixelBufferWidthKey as String: width,
kCVPixelBufferHeightKey as String: height,
])
writer.add(input)
guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) }
writer.startSession(atSourceTime: .zero)
}
func append(bgra pixels: [UInt8], at time: Double) {
guard input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return }
var created: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created)
guard let buffer = created else { return }
CVPixelBufferLockBaseAddress(buffer, [])
if let base = CVPixelBufferGetBaseAddress(buffer) {
let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
pixels.withUnsafeBytes { source in
for y in 0..<height {
memcpy(base + y * rowBytes, source.baseAddress! + y * width * 4, width * 4)
}
}
}
CVPixelBufferUnlockBaseAddress(buffer, [])
if startTime == nil { startTime = time }
adaptor.append(buffer, withPresentationTime: CMTime(seconds: time - startTime!, preferredTimescale: 600))
}
func finish() {
input.markAsFinished()
let done = DispatchSemaphore(value: 0)
writer.finishWriting { done.signal() }
done.wait()
}
}
struct VirtualCameraError: Error, CustomStringConvertible {
let description: String
}
/// Feeds frames into OBS's virtual camera, so Meet, Zoom and anything else that takes a webcam can use them.
/// OBS itself must not be running its own virtual camera at the same time.
final class VirtualCamera {
// fixed identifier of the camera OBS installs (from obs-studio's mac-virtualcam plugin)
private static let obsDeviceUID = "7626645E-4425-469E-9D8B-97E0FA59AC75"
private let deviceID: CMIODeviceID
private let streamID: CMIOStreamID
private let queue: CMSimpleQueue
private let format: CMFormatDescription
private let pool: CVPixelBufferPool
init(width: Int, height: Int) throws {
func address(_ selector: Int) -> CMIOObjectPropertyAddress {
CMIOObjectPropertyAddress(mSelector: CMIOObjectPropertySelector(selector),
mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain))
}
func objects(of object: CMIOObjectID, _ selector: Int) -> [CMIOObjectID] {
var property = address(selector)
var size: UInt32 = 0, used: UInt32 = 0
CMIOObjectGetPropertyDataSize(object, &property, 0, nil, &size)
var ids = [CMIOObjectID](repeating: 0, count: Int(size) / MemoryLayout<CMIOObjectID>.size)
CMIOObjectGetPropertyData(object, &property, 0, nil, size, &used, &ids)
return ids
}
func uid(of device: CMIOObjectID) -> String? {
var property = address(kCMIODevicePropertyDeviceUID)
var value: Unmanaged<CFString>?
var used: UInt32 = 0
CMIOObjectGetPropertyData(device, &property, 0, nil, UInt32(MemoryLayout<Unmanaged<CFString>?>.size), &used, &value)
return value?.takeRetainedValue() as String?
}
let devices = objects(of: CMIOObjectID(kCMIOObjectSystemObject), kCMIOHardwarePropertyDevices)
guard let device = devices.first(where: { uid(of: $0) == VirtualCamera.obsDeviceUID }) else {
throw VirtualCameraError(description: """
OBS Virtual Camera not found. One-time setup: enable OBS under System Settings > General > \
Login Items & Extensions > Camera Extensions, then in OBS click Start Virtual Camera, Stop Virtual Camera, and quit OBS.
""")
}
// the camera has two streams: [0] is what apps watch, [1] is where frames are fed in
let streams = objects(of: device, kCMIODevicePropertyStreams)
guard streams.count >= 2 else { throw VirtualCameraError(description: "OBS Virtual Camera has no input stream.") }
deviceID = device
streamID = streams[1]
var createdQueue: Unmanaged<CMSimpleQueue>?
CMIOStreamCopyBufferQueue(streamID, { _, _, _ in }, nil, &createdQueue)
var createdFormat: CMFormatDescription?
CMVideoFormatDescriptionCreate(allocator: nil, codecType: kCVPixelFormatType_422YpCbCr8,
width: Int32(width), height: Int32(height), extensions: nil,
formatDescriptionOut: &createdFormat)
var createdPool: CVPixelBufferPool?
CVPixelBufferPoolCreate(nil, nil, [
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_422YpCbCr8,
kCVPixelBufferWidthKey: width,
kCVPixelBufferHeightKey: height,
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary,
] as CFDictionary, &createdPool)
guard let createdQueue, let createdFormat, let createdPool else {
throw VirtualCameraError(description: "Could not set up the OBS Virtual Camera stream.")
}
queue = createdQueue.takeRetainedValue()
format = createdFormat
pool = createdPool
guard CMIODeviceStartStream(deviceID, streamID) == noErr else {
throw VirtualCameraError(description: "Could not start the OBS Virtual Camera stream (is OBS using it?).")
}
}
deinit {
CMIODeviceStopStream(deviceID, streamID)
}
/// Sends one frame. The picture is flipped back left-to-right: the preview is a mirror,
/// but the people on the call should see the real thing.
func send(bgra pixels: [UInt8], width: Int, height: Int) {
var created: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created)
guard let buffer = created else { return }
CVPixelBufferLockBaseAddress(buffer, [])
if let base = CVPixelBufferGetBaseAddress(buffer)?.assumingMemoryBound(to: UInt8.self) {
let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
pixels.withUnsafeBufferPointer { source in
// BGRA -> UYVY (BT.601 video range): two pixels share one U and one V
for y in 0..<height {
let row = base + y * rowBytes
for pair in 0..<(width / 2) {
let first = (y * width + (width - 1 - pair * 2)) * 4
let second = first - 4
let b0 = Int(source[first]), g0 = Int(source[first + 1]), r0 = Int(source[first + 2])
let b1 = Int(source[second]), g1 = Int(source[second + 1]), r1 = Int(source[second + 2])
row[pair * 4] = UInt8(clamping: ((-38 * r0 - 74 * g0 + 112 * b0 + 128) >> 8) + 128)
row[pair * 4 + 1] = UInt8(clamping: ((66 * r0 + 129 * g0 + 25 * b0 + 128) >> 8) + 16)
row[pair * 4 + 2] = UInt8(clamping: ((112 * r0 - 94 * g0 - 18 * b0 + 128) >> 8) + 128)
row[pair * 4 + 3] = UInt8(clamping: ((66 * r1 + 129 * g1 + 25 * b1 + 128) >> 8) + 16)
}
}
}
}
CVPixelBufferUnlockBaseAddress(buffer, [])
var timing = CMSampleTimingInfo(
duration: .invalid,
presentationTimeStamp: CMTime(value: CMTimeValue(clock_gettime_nsec_np(CLOCK_UPTIME_RAW)), timescale: 1_000_000_000),
decodeTimeStamp: .invalid)
var sample: CMSampleBuffer?
CMSampleBufferCreateForImageBuffer(allocator: nil, imageBuffer: buffer, dataReady: true, makeDataReadyCallback: nil,
refcon: nil, formatDescription: format, sampleTiming: &timing, sampleBufferOut: &sample)
if let sample {
// the queue takes ownership of the sample
if CMSimpleQueueEnqueue(queue, element: Unmanaged.passRetained(sample).toOpaque()) != noErr {
Unmanaged.passUnretained(sample).release()
}
}
}
}

92
native/Sources/MotionLab/Panel.swift Normal file
View file

@ -0,0 +1,92 @@
import AppKit
private final class TopAlignedView: NSView {
override var isFlipped: Bool { true } // so the scroll view starts at the top
}
/// Side panel: every effect with a checkbox, and the sliders of the ones that are switched on.
final class Panel: NSObject {
static let width: CGFloat = 230
let view = NSScrollView()
private let stack = NSStackView()
private let statusLabel = NSTextField(labelWithString: "")
private let renderer: Renderer
init(renderer: Renderer) {
self.renderer = renderer
super.init()
stack.orientation = .vertical
stack.alignment = .leading
stack.spacing = 4
stack.translatesAutoresizingMaskIntoConstraints = false
let document = TopAlignedView()
document.translatesAutoresizingMaskIntoConstraints = false
document.addSubview(stack)
view.documentView = document
view.hasVerticalScroller = true
view.drawsBackground = false
NSLayoutConstraint.activate([
view.widthAnchor.constraint(equalToConstant: Panel.width),
document.widthAnchor.constraint(equalTo: view.contentView.widthAnchor),
stack.topAnchor.constraint(equalTo: document.topAnchor, constant: 14),
stack.leadingAnchor.constraint(equalTo: document.leadingAnchor, constant: 14),
stack.trailingAnchor.constraint(equalTo: document.trailingAnchor, constant: -14),
stack.bottomAnchor.constraint(equalTo: document.bottomAnchor, constant: -14),
])
statusLabel.font = .boldSystemFont(ofSize: 11)
statusLabel.textColor = .systemRed
rebuild()
}
private func label(_ text: String, dim: Bool = true) -> NSTextField {
let field = NSTextField(labelWithString: text)
field.font = .systemFont(ofSize: 11)
field.textColor = dim ? .secondaryLabelColor : .labelColor
return field
}
func rebuild() {
stack.arrangedSubviews.forEach { $0.removeFromSuperview() }
statusLabel.stringValue = renderer.status
stack.addArrangedSubview(statusLabel)
for (layerIndex, layer) in renderer.layers.enumerated() {
let title = layer.error == nil ? layer.name : "\(layer.name) (shader error)"
let checkbox = NSButton(checkboxWithTitle: title, target: self, action: #selector(toggled(_:)))
checkbox.state = layer.enabled ? .on : .off
checkbox.tag = layerIndex
checkbox.font = .systemFont(ofSize: 13, weight: .medium)
stack.addArrangedSubview(checkbox)
guard layer.enabled else { continue }
for (paramIndex, param) in layer.params.enumerated() {
let slider = NSSlider(value: Double(param.value), minValue: Double(param.min), maxValue: Double(param.max),
target: self, action: #selector(sliderMoved(_:)))
slider.tag = layerIndex * maxParams + paramIndex
slider.controlSize = .small
stack.addArrangedSubview(label(param.name))
stack.addArrangedSubview(slider)
slider.widthAnchor.constraint(equalTo: stack.widthAnchor).isActive = true
}
stack.setCustomSpacing(12, after: stack.arrangedSubviews.last!)
}
if renderer.layers.isEmpty { stack.addArrangedSubview(label("no .metal files found")) }
stack.setCustomSpacing(16, after: stack.arrangedSubviews.last!)
stack.addArrangedSubview(label("s screenshot\nr record\nq quit"))
}
func refreshStatus() {
statusLabel.stringValue = renderer.status
}
@objc private func toggled(_ checkbox: NSButton) {
renderer.setEnabled(checkbox.tag, checkbox.state == .on)
}
@objc private func sliderMoved(_ slider: NSSlider) {
renderer.setParam(layer: slider.tag / maxParams, param: slider.tag % maxParams, value: Float(slider.doubleValue))
}
}

287
native/Sources/MotionLab/Renderer.swift Normal file
View file

@ -0,0 +1,287 @@
import MetalKit
/// One effect file in the chain.
final class Layer {
let url: URL
let name: String
var enabled = false
var order = 50
var params: [EffectParam] = []
var error: String?
fileprivate var pipeline: MTLRenderPipelineState?
fileprivate var modified: Date?
fileprivate var usesParticles = false
fileprivate var targets: [MTLTexture] = [] // [this frame, previous frame], swapped after every draw
init(url: URL) {
self.url = url
self.name = url.deletingPathExtension().lastPathComponent
}
}
private let passthrough = "float4 effect(float2 uv, Inputs in) { return in.image.sample(in.smp, uv); }"
/// Runs every enabled effect in order on each new camera frame, shows the result in the window,
/// and hands it to whatever outputs are active (recording, virtual camera, screenshots).
final class Renderer: NSObject, MTKViewDelegate {
private let device: MTLDevice
private let queue: MTLCommandQueue
private let camera: Camera
private let effectsDirectory: URL
private let copyPipeline: MTLRenderPipelineState
private var analysis: Analysis?
private var plain: MTLTexture? // holds the camera picture when no effect is enabled
private var result: MTLTexture? // the finished frame
private var lastSerial = -1
private let startTime = CACurrentMediaTime()
private var frameCount = 0
private var pixels: [UInt8] = []
private(set) var layers: [Layer] = []
/// Called when layers were added, removed, reloaded or toggled.
var onLayersChanged: (() -> Void)?
/// Called when recording or the virtual camera starts or stops.
var onStatusChanged: (() -> Void)?
/// When set, the app exits after this many camera frames, first saving that frame as a PNG if a url is given
/// (used to check effects without watching).
var snapshot: (frames: Int, url: URL?)?
/// When set, recording into this file starts with the first frame.
var recordURL: URL?
/// When true, the OBS virtual camera is opened on the first frame, at the size of the output.
var wantsVirtualCamera = false
private var virtualCamera: VirtualCamera? { didSet { onStatusChanged?() } }
private var recorder: Recorder? { didSet { onStatusChanged?() } }
private var screenshotRequested = false
var status: String {
[recorder != nil ? "REC" : nil, virtualCamera != nil ? "CAM" : nil].compactMap { $0 }.joined(separator: " ")
}
init(device: MTLDevice, camera: Camera, effectsDirectory: URL) throws {
self.device = device
self.camera = camera
self.effectsDirectory = effectsDirectory
self.queue = device.makeCommandQueue()!
self.copyPipeline = try makePipeline(device, source: effectShader(body: passthrough, params: []),
fragment: "effectFragment", format: .bgra8Unorm)
super.init()
refreshFiles()
}
// MARK: effect files
/// Picks up new, deleted and edited effect files. On a compile error the last working version keeps running.
private func refreshFiles() {
let files = ((try? FileManager.default.contentsOfDirectory(at: effectsDirectory, includingPropertiesForKeys: nil)) ?? [])
.filter { $0.pathExtension == "metal" }
var changed = false
let before = layers.count
layers.removeAll { layer in !files.contains(layer.url) }
changed = layers.count != before
for url in files where !layers.contains(where: { $0.url == url }) {
layers.append(Layer(url: url))
changed = true
}
for layer in layers where reload(layer) { changed = true }
if changed {
layers.sort { ($0.order, $0.name) < ($1.order, $1.name) }
onLayersChanged?()
}
}
private func reload(_ layer: Layer) -> Bool {
let modified = (try? layer.url.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate
guard let modified, modified != layer.modified else { return false }
let firstLoad = layer.modified == nil
layer.modified = modified
do {
let source = try String(contentsOf: layer.url, encoding: .utf8)
var info = parseEffect(source)
layer.pipeline = try makePipeline(device, source: effectShader(body: source, params: info.params),
fragment: "effectFragment", format: .bgra8Unorm)
for i in info.params.indices { // keep slider positions across reloads
if let old = layer.params.first(where: { $0.name == info.params[i].name }) {
info.params[i].value = min(max(old.value, info.params[i].min), info.params[i].max)
}
}
layer.params = info.params
layer.order = info.order
layer.usesParticles = source.contains("in.particles")
layer.error = nil
if firstLoad { layer.enabled = info.startsOn }
print("[\(layer.name)] loaded")
} catch {
layer.error = error.localizedDescription
print("[\(layer.name)] shader error:\n\(error.localizedDescription)")
}
return true
}
var failedLayers: [String] { layers.filter { $0.error != nil }.map(\.name) }
/// Enables exactly the named layers. Returns the names that don't exist.
func enableOnly(_ names: [String]) -> [String] {
for layer in layers { layer.enabled = names.contains(layer.name) }
return names.filter { name in !layers.contains { $0.name == name } }
}
func setEnabled(_ index: Int, _ enabled: Bool) {
guard layers.indices.contains(index) else { return }
layers[index].enabled = enabled
if !enabled { layers[index].targets = [] } // free its memory, and start clean next time
onLayersChanged?()
}
func setParam(layer: Int, param: Int, value: Float) {
guard layers.indices.contains(layer), layers[layer].params.indices.contains(param) else { return }
let p = layers[layer].params[param]
layers[layer].params[param].value = min(max(value, p.min), p.max)
}
// MARK: outputs
func requestScreenshot() { screenshotRequested = true }
func toggleRecording() {
if let recorder {
self.recorder = nil
recorder.finish()
print("saved \(recorder.url.path)")
} else if let result {
do {
recorder = try Recorder(url: capturesURL(extension: "mp4"), width: result.width, height: result.height)
print("recording \(recorder!.url.path)")
} catch {
print("could not start recording: \(error.localizedDescription)")
}
}
}
func finish() {
if recorder != nil { toggleRecording() }
}
private func capturesURL(extension ext: String) -> URL {
let directory = URL(fileURLWithPath: "captures")
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
let stamp = DateFormatter()
stamp.dateFormat = "yyyyMMdd-HHmmss"
return directory.appendingPathComponent("motionlab-\(stamp.string(from: Date())).\(ext)")
}
/// Runs once per camera frame, after the GPU work was submitted.
private func deliver(_ commands: MTLCommandBuffer) {
frameCount += 1
let snapshotDue = snapshot.map { frameCount >= $0.frames } ?? false
guard let result else { return }
let width = result.width, height = result.height
if let url = recordURL {
recordURL = nil
do { recorder = try Recorder(url: url, width: width, height: height) } catch { print("could not start recording: \(error.localizedDescription)") }
}
if wantsVirtualCamera {
wantsVirtualCamera = false
do {
virtualCamera = try VirtualCamera(width: width, height: height)
print("virtual camera on: pick 'OBS Virtual Camera' as the camera in your call or streaming app")
} catch {
print("virtual camera off: \(error)")
}
}
guard recorder != nil || virtualCamera != nil || screenshotRequested || snapshotDue else { return }
commands.waitUntilCompleted()
if pixels.count != width * height * 4 { pixels = [UInt8](repeating: 0, count: width * height * 4) }
result.getBytes(&pixels, bytesPerRow: width * 4, from: MTLRegionMake2D(0, 0, width, height), mipmapLevel: 0)
recorder?.append(bgra: pixels, at: CACurrentMediaTime())
virtualCamera?.send(bgra: pixels, width: width, height: height)
if screenshotRequested {
screenshotRequested = false
let url = capturesURL(extension: "png")
writePNG(bgra: &pixels, width: width, height: height, to: url)
print("saved \(url.path)")
}
if snapshotDue {
if let url = snapshot?.url {
writePNG(bgra: &pixels, width: width, height: height, to: url)
print("\(frameCount) frames -> \(url.path)")
}
finish()
exit(0)
}
}
// MARK: drawing
private func process(_ frames: (current: MTLTexture, last: MTLTexture, serial: Int), _ commands: MTLCommandBuffer) throws {
let width = frames.current.width, height = frames.current.height
if analysis == nil { analysis = try Analysis(device: device, queue: queue, width: width, height: height) }
guard let analysis else { return }
analysis.update(commands, camera: frames.current, particles: layers.contains { $0.enabled && $0.usesParticles })
let time = Float(CACurrentMediaTime() - startTime)
var image = frames.current
for layer in layers where layer.enabled {
guard let pipeline = layer.pipeline else { continue }
if layer.targets.first?.width != width || layer.targets.first?.height != height {
layer.targets = [makeTarget(device, width: width, height: height), makeTarget(device, width: width, height: height)]
for target in layer.targets { commands.pass(into: target, clear: true) }
}
// must match `struct Uniforms` in the shader: time, padding, resolution.xy, params[16]
var uniforms: [Float] = [time, 0, Float(width), Float(height)]
uniforms += layer.params.map(\.value) + [Float](repeating: 0, count: maxParams - layer.params.count)
let textures = [image, frames.current, frames.last, layer.targets[1], analysis.flow, analysis.particles]
commands.pass(into: layer.targets[0]) { $0.fullscreen(pipeline, textures: textures, uniforms: uniforms) }
image = layer.targets[0]
layer.targets.swapAt(0, 1)
}
if image === frames.current {
if plain?.width != width || plain?.height != height { plain = makeTarget(device, width: width, height: height) }
commands.pass(into: plain!) { $0.fullscreen(copyPipeline, textures: [image], uniforms: [0, 0, 0, 0]) }
image = plain!
}
result = image
}
/// The largest rectangle with the output's aspect ratio that fits the window, centred.
private func fitted(_ output: MTLTexture, in size: CGSize) -> MTLViewport {
let scale = min(size.width / Double(output.width), size.height / Double(output.height))
let width = Double(output.width) * scale, height = Double(output.height) * scale
return MTLViewport(originX: (size.width - width) / 2, originY: (size.height - height) / 2,
width: width, height: height, znear: 0, zfar: 1)
}
func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {}
func draw(in view: MTKView) {
guard let frames = camera.latest, let commands = queue.makeCommandBuffer() else { return }
// effects advance once per camera frame; in between, the window just shows the last result again
let isNewFrame = frames.serial != lastSerial
if isNewFrame {
lastSerial = frames.serial
if frameCount % 15 == 0 { refreshFiles() }
do {
try process(frames, commands)
} catch {
print("render setup failed: \(error.localizedDescription)")
exit(1)
}
}
if let result, let drawable = view.currentDrawable, let windowPass = view.currentRenderPassDescriptor,
let encoder = commands.makeRenderCommandEncoder(descriptor: windowPass) {
encoder.setViewport(fitted(result, in: view.drawableSize))
encoder.fullscreen(copyPipeline, textures: [result], uniforms: [0, 0, 0, 0])
encoder.endEncoding()
commands.present(drawable)
}
commands.commit()
if isNewFrame { deliver(commands) }
}
}

161
native/Sources/MotionLab/Shaders.swift Normal file
View file

@ -0,0 +1,161 @@
import Metal
/// A slider an effect file declares with a comment line: `// slider <name> <min> <max> <default>`
struct EffectParam {
let name: String
let min: Float
let max: Float
var value: Float
}
/// Everything an effect file says about itself in `//` comment lines:
///
/// // slider <name> <min> <max> <default> a slider, read in the shader as in.p.<name>
/// // order <number> position in the chain, low runs first (default 50)
/// // on enabled when the app starts
struct EffectInfo {
var params: [EffectParam] = []
var order = 50
var startsOn = false
}
let maxParams = 16
func parseEffect(_ source: String) -> EffectInfo {
var info = EffectInfo()
for line in source.split(separator: "\n") {
let parts = line.split(separator: " ")
guard parts.count >= 2, parts[0] == "//" else { continue }
switch parts[1] {
case "slider" where parts.count == 6:
if let min = Float(parts[3]), let max = Float(parts[4]), let value = Float(parts[5]) {
info.params.append(EffectParam(name: String(parts[2]), min: min, max: max, value: value))
}
case "order" where parts.count == 3:
info.order = Int(parts[2]) ?? info.order
case "on" where parts.count == 2:
info.startsOn = true
default:
break
}
}
info.params = Array(info.params.prefix(maxParams))
return info
}
/// Shared by every shader in the app: a triangle that covers the whole target.
let shaderPrelude = """
#include <metal_stdlib>
using namespace metal;
struct VertexOut {
float4 position [[position]];
float2 uv;
};
vertex VertexOut fullscreenVertex(uint id [[vertex_id]]) {
float2 p = float2((id << 1) & 2, id & 2);
VertexOut out;
out.position = float4(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0, 0.0, 1.0);
out.uv = p;
return out;
}
"""
/// Wraps an effect file into a complete Metal shader. An effect file only defines:
///
/// float4 effect(float2 uv, Inputs in) { ... }
///
/// `uv` runs from (0,0) top-left to (1,1) bottom-right.
func effectShader(body: String, params: [EffectParam]) -> String {
let fields = params.isEmpty ? "float unused;" : params.map { "float \($0.name);" }.joined(separator: " ")
let values = params.isEmpty ? "0.0" : params.indices.map { "u.params[\($0)]" }.joined(separator: ", ")
return shaderPrelude + """
struct Uniforms {
float time;
float2 resolution;
float params[\(maxParams)];
};
struct Params { \(fields) };
struct Inputs {
texture2d<float> image; // the picture so far: output of the effect above, or the camera for the first one
texture2d<float> camera; // the untouched camera frame
texture2d<float> last; // the camera frame before this one (compare with camera to find motion)
texture2d<float> prev; // this effect's own output from the previous frame (for trails)
texture2d<float> flow; // .xy = which way and how far things moved since the last frame, in uv units
texture2d<float> particles; // .r = brightness of particles that drift along with the motion
sampler smp;
float time; // seconds since start
float2 resolution; // output size in pixels
Params p; // this effect's sliders
};
#line 1
\(body)
fragment float4 effectFragment(VertexOut v [[stage_in]],
texture2d<float> image [[texture(0)]],
texture2d<float> camera [[texture(1)]],
texture2d<float> last [[texture(2)]],
texture2d<float> prev [[texture(3)]],
texture2d<float> flow [[texture(4)]],
texture2d<float> particles [[texture(5)]],
constant Uniforms &u [[buffer(0)]]) {
constexpr sampler smp(filter::linear, address::clamp_to_edge);
Params p = { \(values) };
Inputs in = { image, camera, last, prev, flow, particles, smp, u.time, u.resolution, p };
return effect(v.uv, in);
}
"""
}
func makePipeline(_ device: MTLDevice, source: String, vertex: String = "fullscreenVertex", fragment: String,
format: MTLPixelFormat, additive: Bool = false) throws -> MTLRenderPipelineState {
let library = try device.makeLibrary(source: source, options: nil)
let descriptor = MTLRenderPipelineDescriptor()
descriptor.vertexFunction = library.makeFunction(name: vertex)
descriptor.fragmentFunction = library.makeFunction(name: fragment)
let attachment = descriptor.colorAttachments[0]!
attachment.pixelFormat = format
if additive {
attachment.isBlendingEnabled = true
attachment.sourceRGBBlendFactor = .one
attachment.destinationRGBBlendFactor = .one
attachment.sourceAlphaBlendFactor = .one
attachment.destinationAlphaBlendFactor = .one
}
return try device.makeRenderPipelineState(descriptor: descriptor)
}
func makeTarget(_ device: MTLDevice, width: Int, height: Int, format: MTLPixelFormat = .bgra8Unorm) -> MTLTexture {
let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width, height: height, mipmapped: false)
descriptor.usage = [.renderTarget, .shaderRead]
return device.makeTexture(descriptor: descriptor)!
}
extension MTLCommandBuffer {
/// One render pass into `target`. With `clear`, the target starts black; otherwise its old contents are ignored.
func pass(into target: MTLTexture, clear: Bool = false, _ body: (MTLRenderCommandEncoder) -> Void = { _ in }) {
let descriptor = MTLRenderPassDescriptor()
descriptor.colorAttachments[0].texture = target
descriptor.colorAttachments[0].loadAction = clear ? .clear : .dontCare
descriptor.colorAttachments[0].storeAction = .store
guard let encoder = makeRenderCommandEncoder(descriptor: descriptor) else { return }
body(encoder)
encoder.endEncoding()
}
}
extension MTLRenderCommandEncoder {
func fullscreen(_ pipeline: MTLRenderPipelineState, textures: [MTLTexture], uniforms: [Float] = []) {
setRenderPipelineState(pipeline)
setFragmentTextures(textures, range: 0..<textures.count)
if !uniforms.isEmpty {
setFragmentBytes(uniforms, length: uniforms.count * MemoryLayout<Float>.stride, index: 0)
}
drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3)
}
}

110
native/Sources/MotionLab/main.swift Normal file
View file

@ -0,0 +1,110 @@
// MotionLab: camera -> chain of Metal shader effects -> window, recording, virtual camera.
//
// swift run -c release MotionLab [options]
//
// --effects DIR folder of .metal effect files (default ./effects)
// --only a,b,c start with exactly these effects enabled
// --virtualcam also send the output to the OBS virtual camera (for Meet, Zoom, ...)
// --snapshot OUT.png save one frame after --frames N camera frames (default 90) and exit
// --record OUT.mp4 record --frames N camera frames (default 90) and exit
//
// In the window: s saves a screenshot, r starts/stops recording (both into ./captures), q or Esc quits.
// Effect files are reloaded when saved.
import AppKit
import MetalKit
func option(_ name: String) -> String? {
let arguments = CommandLine.arguments
guard let i = arguments.firstIndex(of: name), i + 1 < arguments.count else { return nil }
return arguments[i + 1]
}
func fail(_ message: String) -> Never {
FileHandle.standardError.write((message + "\n").data(using: .utf8)!)
exit(1)
}
final class AppDelegate: NSObject, NSApplicationDelegate {
var onQuit: (() -> Void)?
func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
func applicationWillTerminate(_ notification: Notification) { onQuit?() }
}
setvbuf(stdout, nil, _IOLBF, 0) // show "loaded" / shader errors immediately, even when piped
guard let device = MTLCreateSystemDefaultDevice() else { fail("Metal is not available on this machine.") }
let effectsDirectory = URL(fileURLWithPath: option("--effects") ?? "effects")
let renderer: Renderer
do {
renderer = try Renderer(device: device, camera: Camera(device: device), effectsDirectory: effectsDirectory)
} catch {
fail("\(error)")
}
if renderer.layers.isEmpty { print("No .metal files in \(effectsDirectory.path) - showing the plain camera.") }
if let names = option("--only") {
let unknown = renderer.enableOnly(names.split(separator: ",").map(String.init))
if !unknown.isEmpty {
fail("Unknown effect(s): \(unknown.joined(separator: ", ")). Available: \(renderer.layers.map(\.name).joined(separator: ", "))")
}
}
if option("--snapshot") != nil || option("--record") != nil {
if !renderer.failedLayers.isEmpty { fail("shader compile failed: \(renderer.failedLayers.joined(separator: ", "))") }
renderer.snapshot = (Int(option("--frames") ?? "90") ?? 90, option("--snapshot").map { URL(fileURLWithPath: $0) })
renderer.recordURL = option("--record").map { URL(fileURLWithPath: $0) }
}
renderer.wantsVirtualCamera = CommandLine.arguments.contains("--virtualcam")
let app = NSApplication.shared
let delegate = AppDelegate()
delegate.onQuit = { renderer.finish() }
app.delegate = delegate
app.setActivationPolicy(.regular)
let size = NSSize(width: 960 + Panel.width, height: 540)
let window = NSWindow(contentRect: NSRect(origin: .zero, size: size),
styleMask: [.titled, .closable, .miniaturizable, .resizable],
backing: .buffered, defer: false)
let view = MTKView(frame: .zero, device: device)
view.colorPixelFormat = .bgra8Unorm
view.delegate = renderer
let panel = Panel(renderer: renderer)
renderer.onLayersChanged = { panel.rebuild() }
renderer.onStatusChanged = { panel.refreshStatus() }
NSEvent.addLocalMonitorForEvents(matching: .keyDown) { event in
if event.modifierFlags.contains(.command) { return event }
switch event.charactersIgnoringModifiers {
case "s": renderer.requestScreenshot()
case "r": renderer.toggleRecording()
case "q", "\u{1B}": app.terminate(nil)
default: return event
}
return nil
}
let content = NSView(frame: NSRect(origin: .zero, size: size))
for subview in [view, panel.view] {
subview.translatesAutoresizingMaskIntoConstraints = false
content.addSubview(subview)
}
NSLayoutConstraint.activate([
view.leadingAnchor.constraint(equalTo: content.leadingAnchor),
view.topAnchor.constraint(equalTo: content.topAnchor),
view.bottomAnchor.constraint(equalTo: content.bottomAnchor),
view.trailingAnchor.constraint(equalTo: panel.view.leadingAnchor),
panel.view.trailingAnchor.constraint(equalTo: content.trailingAnchor),
panel.view.topAnchor.constraint(equalTo: content.topAnchor),
panel.view.bottomAnchor.constraint(equalTo: content.bottomAnchor),
])
window.title = "MotionLab"
window.contentMinSize = NSSize(width: 480 + Panel.width, height: 270)
window.contentView = content
window.setContentSize(size) // auto layout would otherwise pick its own window size
window.center()
window.makeKeyAndOrderFront(nil)
app.activate(ignoringOtherApps: true)
app.run()

12
native/effects/colorize.metal Normal file
View file

@ -0,0 +1,12 @@
// Recolours the picture by brightness with a smooth colour ramp.
// order 10
// slider amount 0.0 1.0 0.8
// slider palette 0.0 1.0 0.0
// slider contrast 0.5 3.0 1.2
float4 effect(float2 uv, Inputs in) {
float3 c = in.image.sample(in.smp, uv).rgb;
float luma = pow(dot(c, float3(0.299, 0.587, 0.114)), in.p.contrast);
float3 ramp = 0.5 + 0.5 * cos(6.2832 * (luma * 0.8 + in.p.palette + float3(0.55, 0.4, 0.2)));
float3 mapped = ramp * (0.15 + 0.85 * luma);
return float4(mix(c, mapped, in.p.amount), 1.0);
}

18
native/effects/halftone.metal Normal file
View file

@ -0,0 +1,18 @@
// Halftone dots sized by brightness; dots turn hot orange wherever something moves.
// order 70
// slider cell 4.0 40.0 12.0
// slider heat 0.0 20.0 8.0
float4 effect(float2 uv, Inputs in) {
float2 grid = in.resolution / in.p.cell;
float2 centre = (floor(uv * grid) + 0.5) / grid;
float3 c = in.image.sample(in.smp, centre).rgb;
float luma = dot(c, float3(0.299, 0.587, 0.114));
float radius = 0.5 * sqrt(luma);
float dist = length(fract(uv * grid) - 0.5);
float mask = 1.0 - smoothstep(radius - 0.05, radius + 0.05, dist);
float moved = length(in.camera.sample(in.smp, centre).rgb - in.last.sample(in.smp, centre).rgb);
float3 hot = mix(c, float3(1.0, 0.35, 0.1), clamp(moved * in.p.heat, 0.0, 1.0));
return float4(hot * mask, 1.0);
}

13
native/effects/kaleidoscope.metal Normal file
View file

@ -0,0 +1,13 @@
// Folds the picture into mirrored wedges around the centre.
// order 90
// slider segments 2.0 16.0 6.0
// slider spin -2.0 2.0 0.2
float4 effect(float2 uv, Inputs in) {
float2 shape = float2(in.resolution.x / in.resolution.y, 1.0);
float2 c = (uv - 0.5) * shape;
float wedge = 6.2832 / floor(in.p.segments);
float a = atan2(c.y, c.x) + in.time * in.p.spin;
a = abs(a - wedge * floor(a / wedge) - wedge * 0.5) - 1.5708; // fold into one mirrored wedge, pointing up
float2 from = float2(cos(a), sin(a)) * length(c) / shape + 0.5;
return float4(in.image.sample(in.smp, from).rgb, 1.0);
}

13
native/effects/liquid.metal Normal file
View file

@ -0,0 +1,13 @@
// Liquid trails: the previous output is dragged along the motion, swirled a little,
// and slowly fades back to the live picture.
// order 45
// slider persistence 0.0 0.99 0.92
// slider drag 0.0 6.0 2.5
// slider swirl 0.0 0.01 0.002
float4 effect(float2 uv, Inputs in) {
float2 centred = uv - 0.5;
float2 from = uv - in.flow.sample(in.smp, uv).xy * in.p.drag + float2(-centred.y, centred.x) * in.p.swirl;
float3 trail = in.prev.sample(in.smp, from).rgb;
float3 live = in.image.sample(in.smp, uv).rgb;
return float4(mix(live, trail, in.p.persistence), 1.0);
}

20
native/effects/motion-glow.metal Normal file
View file

@ -0,0 +1,20 @@
// Whatever moves lights up and the glow lingers. The heat is carried from frame to frame
// in the alpha channel of this effect's own previous output.
// order 50
// slider sensitivity 1.0 30.0 10.0
// slider linger 0.5 0.99 0.93
// slider dim 0.0 1.0 0.45
// slider hue 0.0 1.0 0.07
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
static float3 hueColor(float h) {
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
}
float4 effect(float2 uv, Inputs in) {
float moved = length(in.camera.sample(in.smp, uv).rgb - in.last.sample(in.smp, uv).rgb);
float fresh = smoothstep(0.3, 1.0, moved * in.p.sensitivity);
float heat = max(in.prev.sample(in.smp, uv).a * in.p.linger, fresh);
float3 glow = hueColor(in.p.hue);
return float4(in.image.sample(in.smp, uv).rgb * in.p.dim + glow * heat, heat);
}

38
native/effects/neon-edges.metal Normal file
View file

@ -0,0 +1,38 @@
// Glowing outlines with a slowly cycling colour.
// order 20
// slider detail 1.0 12.0 5.0
// slider glow 0.0 6.0 2.5
// slider source 0.0 1.0 0.35
// slider hue 0.0 1.0 0.55
// slider cycle 0.0 1.0 0.3
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
static float3 hueColor(float h) {
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
}
static float luma(Inputs in, float2 uv) {
return dot(in.image.sample(in.smp, uv).rgb, float3(0.299, 0.587, 0.114));
}
// how strongly brightness changes around this point
static float edge(Inputs in, float2 uv) {
float2 t = 1.5 / in.resolution;
float dx = luma(in, uv + float2(t.x, 0.0)) - luma(in, uv - float2(t.x, 0.0));
float dy = luma(in, uv + float2(0.0, t.y)) - luma(in, uv - float2(0.0, t.y));
return length(float2(dx, dy));
}
float4 effect(float2 uv, Inputs in) {
float sharp = smoothstep(0.1, 0.5, edge(in, uv) * in.p.detail);
float halo = 0.0; // edges found in a ring around this pixel = a cheap glow
for (int i = 0; i < 12; i++) {
float a = 6.2832 * float(i) / 12.0;
float2 dir = float2(cos(a), sin(a)) / in.resolution;
halo += edge(in, uv + dir * 4.0) + edge(in, uv + dir * 9.0);
}
halo = halo / 24.0 * in.p.detail * in.p.glow;
float3 tint = hueColor(in.p.hue + in.time * in.p.cycle * 0.2);
float3 base = in.image.sample(in.smp, uv).rgb * in.p.source;
return float4(base + tint * (sharp + halo), 1.0);
}

22
native/effects/particles.metal Normal file
View file

@ -0,0 +1,22 @@
// Particles spawn where you move and drift along with the motion.
// order 30
// on
// slider brightness 0.0 3.0 1.4
// slider glow 0.0 4.0 1.5
// slider hue 0.0 1.0 0.08
// hue 0..1 -> a fully saturated colour (0 red, 0.08 orange, 0.33 green, 0.66 blue)
static float3 hueColor(float h) {
return clamp(abs(fract(h + float3(0.0, 2.0 / 3.0, 1.0 / 3.0)) * 6.0 - 3.0) - 1.0, 0.0, 1.0);
}
float4 effect(float2 uv, Inputs in) {
float core = in.particles.sample(in.smp, uv).r;
float halo = 0.0;
for (int i = 0; i < 8; i++) {
float a = 6.2832 * float(i) / 8.0;
halo += in.particles.sample(in.smp, uv + float2(cos(a), sin(a)) * 5.0 / in.resolution).r;
}
float3 tint = hueColor(in.p.hue);
float3 base = in.image.sample(in.smp, uv).rgb;
return float4(base + tint * (core * in.p.brightness + halo / 8.0 * in.p.glow), 1.0);
}

16
native/effects/trails.metal Normal file
View file

@ -0,0 +1,16 @@
// Feedback trails: the previous output is zoomed, turned and faded, so bright things leave streaks.
// "push" smears the trails in the direction things are moving.
// order 40
// on
// slider fade 0.5 0.995 0.92
// slider zoom -0.04 0.04 0.012
// slider spin -0.05 0.05 0.005
// slider push 0.0 4.0 0.0
float4 effect(float2 uv, Inputs in) {
float2 c = uv - 0.5;
float2 turned = float2(c.x * cos(in.p.spin) - c.y * sin(in.p.spin), c.x * sin(in.p.spin) + c.y * cos(in.p.spin));
float2 from = turned * (1.0 - in.p.zoom) + 0.5 - in.flow.sample(in.smp, uv).xy * in.p.push;
float3 trail = in.prev.sample(in.smp, from).rgb * in.p.fade;
float3 live = in.image.sample(in.smp, uv).rgb;
return float4(max(live, trail), 1.0);
}

13
native/effects/wave-shift.metal Normal file
View file

@ -0,0 +1,13 @@
// RGB split that ripples down the image and gets pushed around by your movement.
// order 60
// slider amount 0.0 0.05 0.012
// slider waves 1.0 80.0 24.0
// slider speed 0.0 10.0 3.0
// slider react 0.0 3.0 1.0
float4 effect(float2 uv, Inputs in) {
float2 d = float2(in.p.amount * sin(uv.y * in.p.waves + in.time * in.p.speed), 0.0);
d += in.flow.sample(in.smp, uv).xy * in.p.react;
return float4(in.image.sample(in.smp, uv + d).r,
in.image.sample(in.smp, uv).g,
in.image.sample(in.smp, uv - d).b, 1.0);
}