Python/OpenCV proof of concept plus first native Swift port
Snapshot before the Python POC is removed. The POC (main.py, effects.py, panel.py, shaders.py, effects/*.glsl, flake.nix) is the reference for the native app under native/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
6591138bbf
27 changed files with 2076 additions and 0 deletions
180
native/Sources/MotionLab/Analysis.swift
Normal file
180
native/Sources/MotionLab/Analysis.swift
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
import Metal
|
||||
|
||||
/// Host-side motion analysis, shared by all effects. Every camera frame it produces:
|
||||
///
|
||||
/// flow which way things moved (small 2-channel texture, uv units per frame)
|
||||
/// particles a picture of particles that spawn where something moves and drift along the flow
|
||||
///
|
||||
/// Effects read these as `in.flow` and `in.particles`.
|
||||
final class Analysis {
|
||||
private static let flowWidth = 160, flowHeight = 90
|
||||
private static let stateWidth = 128, stateHeight = 64 // one texel per particle
|
||||
private static let particleSpeed: Float = 1.5
|
||||
private static let trailFade: Float = 0.88
|
||||
|
||||
// each pair is [this frame, previous frame], swapped at the start of every update
|
||||
private var luma: [MTLTexture]
|
||||
private var flowTextures: [MTLTexture]
|
||||
private var state: [MTLTexture]
|
||||
private var trail: [MTLTexture]
|
||||
|
||||
private let lumaPipeline: MTLRenderPipelineState
|
||||
private let flowPipeline: MTLRenderPipelineState
|
||||
private let movePipeline: MTLRenderPipelineState
|
||||
private let fadePipeline: MTLRenderPipelineState
|
||||
private let dotPipeline: MTLRenderPipelineState
|
||||
|
||||
var flow: MTLTexture { flowTextures[0] }
|
||||
var particles: MTLTexture { trail[0] }
|
||||
|
||||
init(device: MTLDevice, queue: MTLCommandQueue, width: Int, height: Int) throws {
|
||||
func pair(_ w: Int, _ h: Int, _ format: MTLPixelFormat) -> [MTLTexture] {
|
||||
[makeTarget(device, width: w, height: h, format: format), makeTarget(device, width: w, height: h, format: format)]
|
||||
}
|
||||
luma = pair(Analysis.flowWidth, Analysis.flowHeight, .r16Float)
|
||||
flowTextures = pair(Analysis.flowWidth, Analysis.flowHeight, .rg16Float)
|
||||
state = pair(Analysis.stateWidth, Analysis.stateHeight, .rgba32Float)
|
||||
trail = pair(width, height, .r8Unorm)
|
||||
|
||||
let source = shaderPrelude + analysisShaders
|
||||
lumaPipeline = try makePipeline(device, source: source, fragment: "lumaFragment", format: .r16Float)
|
||||
flowPipeline = try makePipeline(device, source: source, fragment: "flowFragment", format: .rg16Float)
|
||||
movePipeline = try makePipeline(device, source: source, fragment: "moveFragment", format: .rgba32Float)
|
||||
fadePipeline = try makePipeline(device, source: source, fragment: "fadeFragment", format: .r8Unorm)
|
||||
dotPipeline = try makePipeline(device, source: source, vertex: "dotVertex", fragment: "dotFragment",
|
||||
format: .r8Unorm, additive: true)
|
||||
|
||||
// new textures hold garbage; start everything from zero
|
||||
if let commands = queue.makeCommandBuffer() {
|
||||
for texture in luma + flowTextures + state + trail { commands.pass(into: texture, clear: true) }
|
||||
commands.commit()
|
||||
}
|
||||
}
|
||||
|
||||
func update(_ commands: MTLCommandBuffer, camera: MTLTexture, particles wantsParticles: Bool) {
|
||||
luma.swapAt(0, 1)
|
||||
commands.pass(into: luma[0]) { $0.fullscreen(lumaPipeline, textures: [camera]) }
|
||||
|
||||
flowTextures.swapAt(0, 1)
|
||||
commands.pass(into: flowTextures[0]) {
|
||||
$0.fullscreen(flowPipeline, textures: [luma[0], luma[1], flowTextures[1]])
|
||||
}
|
||||
|
||||
guard wantsParticles else { return }
|
||||
state.swapAt(0, 1)
|
||||
commands.pass(into: state[0]) {
|
||||
$0.fullscreen(movePipeline, textures: [state[1], flowTextures[0], luma[0], luma[1]],
|
||||
uniforms: [Float.random(in: 0..<1000), Analysis.particleSpeed, 0, 0])
|
||||
}
|
||||
trail.swapAt(0, 1)
|
||||
commands.pass(into: trail[0]) { encoder in
|
||||
encoder.fullscreen(fadePipeline, textures: [trail[1]], uniforms: [Analysis.trailFade, 0, 0, 0])
|
||||
encoder.setRenderPipelineState(dotPipeline)
|
||||
encoder.setVertexTexture(state[0], index: 0)
|
||||
encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: Analysis.stateWidth * Analysis.stateHeight)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private let analysisShaders = """
|
||||
static float brightness(float3 c) { return dot(c, float3(0.299, 0.587, 0.114)); }
|
||||
static float hash(float2 p) { return fract(sin(dot(p, float2(127.1, 311.7))) * 43758.5453); }
|
||||
|
||||
// Camera -> small, blurred brightness image. Flow is measured on this.
|
||||
fragment float4 lumaFragment(VertexOut v [[stage_in]], texture2d<float> camera [[texture(0)]]) {
|
||||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||
float2 texel = 1.0 / float2(camera.get_width(), camera.get_height());
|
||||
float sum = 0.0;
|
||||
for (int y = 0; y < 4; y++) {
|
||||
for (int x = 0; x < 4; x++) {
|
||||
sum += brightness(camera.sample(s, v.uv + (float2(x, y) - 1.5) * 2.0 * texel).rgb);
|
||||
}
|
||||
}
|
||||
return float4(sum / 16.0);
|
||||
}
|
||||
|
||||
// Lucas-Kanade optical flow: for each pixel, the shift that best explains how its 7x7
|
||||
// neighbourhood changed between the previous and the current brightness image.
|
||||
fragment float4 flowFragment(VertexOut v [[stage_in]],
|
||||
texture2d<float> now [[texture(0)]],
|
||||
texture2d<float> before [[texture(1)]],
|
||||
texture2d<float> flowBefore [[texture(2)]]) {
|
||||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||
float2 texel = 1.0 / float2(now.get_width(), now.get_height());
|
||||
float a = 1e-3, b = 0.0, c = 1e-3, p = 0.0, q = 0.0; // the 1e-3 keeps flat areas at zero flow
|
||||
for (int y = -3; y <= 3; y++) {
|
||||
for (int x = -3; x <= 3; x++) {
|
||||
float2 uv = v.uv + float2(x, y) * texel;
|
||||
float ix = (now.sample(s, uv + float2(texel.x, 0.0)).r - now.sample(s, uv - float2(texel.x, 0.0)).r) * 0.5;
|
||||
float iy = (now.sample(s, uv + float2(0.0, texel.y)).r - now.sample(s, uv - float2(0.0, texel.y)).r) * 0.5;
|
||||
float it = now.sample(s, uv).r - before.sample(s, uv).r;
|
||||
a += ix * ix; b += ix * iy; c += iy * iy;
|
||||
p += ix * it; q += iy * it;
|
||||
}
|
||||
}
|
||||
float2 shift = -float2(c * p - b * q, a * q - b * p) / (a * c - b * b); // in small-image pixels
|
||||
float len = length(shift);
|
||||
if (len > 4.0) { shift *= 4.0 / len; }
|
||||
float2 smoothed = mix(flowBefore.sample(s, v.uv).xy, shift * texel, 0.6);
|
||||
return float4(smoothed, 0.0, 1.0);
|
||||
}
|
||||
|
||||
// One texel per particle: xy = position (uv), z = frames left to live, w = 1 if it was born from motion.
|
||||
fragment float4 moveFragment(VertexOut v [[stage_in]],
|
||||
texture2d<float> state [[texture(0)]],
|
||||
texture2d<float> flow [[texture(1)]],
|
||||
texture2d<float> now [[texture(2)]],
|
||||
texture2d<float> before [[texture(3)]],
|
||||
constant float4 &u [[buffer(0)]]) { // x = random seed, y = speed
|
||||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||
float4 particle = state.read(uint2(v.position.xy));
|
||||
float2 seed = v.position.xy * 0.37 + u.x;
|
||||
float2 pos = particle.xy + flow.sample(s, particle.xy).xy * u.y;
|
||||
pos += (float2(hash(seed), hash(seed + 7.3)) - 0.5) * 0.0015;
|
||||
float life = particle.z - 1.0;
|
||||
float born = particle.w;
|
||||
|
||||
if (life <= 0.0 || pos.x < 0.0 || pos.x > 1.0 || pos.y < 0.0 || pos.y > 1.0) {
|
||||
// respawn: try a few random spots and take the first one where something is moving
|
||||
pos = float2(hash(seed + 1.1), hash(seed + 2.2));
|
||||
born = 0.0;
|
||||
life = 2.0 + hash(seed + 5.5) * 4.0; // nothing moving: stay hidden and try again soon
|
||||
for (int i = 0; i < 6; i++) {
|
||||
float2 spot = float2(hash(seed + 3.3 + float(i)), hash(seed + 9.9 + float(i) * 1.7));
|
||||
if (abs(now.sample(s, spot).r - before.sample(s, spot).r) > 0.04) {
|
||||
pos = spot;
|
||||
born = 1.0;
|
||||
life = 20.0 + hash(seed + 5.5) * 70.0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return float4(pos, life, born);
|
||||
}
|
||||
|
||||
// Particle trails: fade what was there, then add a dot per particle on top.
|
||||
fragment float4 fadeFragment(VertexOut v [[stage_in]], texture2d<float> trail [[texture(0)]],
|
||||
constant float4 &u [[buffer(0)]]) { // x = fade
|
||||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||||
return trail.sample(s, v.uv) * u.x;
|
||||
}
|
||||
|
||||
struct DotOut {
|
||||
float4 position [[position]];
|
||||
float size [[point_size]];
|
||||
};
|
||||
|
||||
vertex DotOut dotVertex(uint id [[vertex_id]], texture2d<float> state [[texture(0)]]) {
|
||||
uint width = state.get_width();
|
||||
float4 particle = state.read(uint2(id % width, id / width));
|
||||
DotOut out;
|
||||
out.position = float4(particle.x * 2.0 - 1.0, 1.0 - particle.y * 2.0, 0.0, 1.0);
|
||||
if (particle.w < 0.5) { out.position.x = 4.0; } // hidden particles are moved off screen
|
||||
out.size = 3.0;
|
||||
return out;
|
||||
}
|
||||
|
||||
fragment float4 dotFragment(DotOut in [[stage_in]], float2 coord [[point_coord]]) {
|
||||
return float4(smoothstep(0.5, 0.1, length(coord - 0.5)) * 0.9);
|
||||
}
|
||||
"""
|
||||
95
native/Sources/MotionLab/Camera.swift
Normal file
95
native/Sources/MotionLab/Camera.swift
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import AVFoundation
|
||||
import CoreVideo
|
||||
import Metal
|
||||
|
||||
enum CameraError: Error, CustomStringConvertible {
|
||||
case noPermission
|
||||
case noCamera
|
||||
|
||||
var description: String {
|
||||
switch self {
|
||||
case .noPermission:
|
||||
return "No camera permission. Allow your terminal under System Settings > Privacy & Security > Camera, then restart it."
|
||||
case .noCamera:
|
||||
return "No camera found."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Streams the built-in camera as Metal textures; `latest` is the newest frame and the one before it.
|
||||
final class Camera: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
||||
private let session = AVCaptureSession()
|
||||
private var cache: CVMetalTextureCache?
|
||||
private let lock = NSLock()
|
||||
// each entry keeps its CVMetalTexture, which keeps the pixel buffer behind the Metal texture alive
|
||||
private var current: (texture: MTLTexture, backing: CVMetalTexture)?
|
||||
private var previous: (texture: MTLTexture, backing: CVMetalTexture)?
|
||||
|
||||
private var serial = 0 // counts frames, so the renderer can tell a new frame from one it already saw
|
||||
|
||||
var latest: (current: MTLTexture, last: MTLTexture, serial: Int)? {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
guard let current else { return nil }
|
||||
return (current.texture, (previous ?? current).texture, serial)
|
||||
}
|
||||
|
||||
init(device: MTLDevice) throws {
|
||||
super.init()
|
||||
guard Camera.requestAccess() else { throw CameraError.noPermission }
|
||||
// prefer the built-in camera over an iPhone that happens to be nearby
|
||||
guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .unspecified)
|
||||
?? AVCaptureDevice.default(for: .video)
|
||||
else { throw CameraError.noCamera }
|
||||
|
||||
CVMetalTextureCacheCreate(nil, nil, device, nil, &cache)
|
||||
let output = AVCaptureVideoDataOutput()
|
||||
output.videoSettings = [
|
||||
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferMetalCompatibilityKey as String: true,
|
||||
]
|
||||
output.alwaysDiscardsLateVideoFrames = true
|
||||
output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "motionlab.camera"))
|
||||
|
||||
session.beginConfiguration()
|
||||
session.addInput(try AVCaptureDeviceInput(device: camera))
|
||||
session.addOutput(output)
|
||||
if session.canSetSessionPreset(.hd1280x720) { session.sessionPreset = .hd1280x720 }
|
||||
print("camera: \(camera.localizedName)")
|
||||
if let connection = output.connection(with: .video), connection.isVideoMirroringSupported {
|
||||
connection.automaticallyAdjustsVideoMirroring = false
|
||||
connection.isVideoMirrored = true // behave like a mirror
|
||||
}
|
||||
session.commitConfiguration()
|
||||
session.startRunning()
|
||||
}
|
||||
|
||||
private static func requestAccess() -> Bool {
|
||||
switch AVCaptureDevice.authorizationStatus(for: .video) {
|
||||
case .authorized:
|
||||
return true
|
||||
case .notDetermined:
|
||||
let done = DispatchSemaphore(value: 0)
|
||||
var granted = false
|
||||
AVCaptureDevice.requestAccess(for: .video) { granted = $0; done.signal() }
|
||||
done.wait()
|
||||
return granted
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
|
||||
guard let cache, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
||||
var created: CVMetalTexture?
|
||||
CVMetalTextureCacheCreateTextureFromImage(
|
||||
nil, cache, buffer, nil, .bgra8Unorm,
|
||||
CVPixelBufferGetWidth(buffer), CVPixelBufferGetHeight(buffer), 0, &created)
|
||||
guard let created, let frame = CVMetalTextureGetTexture(created) else { return }
|
||||
lock.lock()
|
||||
previous = current
|
||||
current = (frame, created)
|
||||
serial += 1
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
198
native/Sources/MotionLab/Output.swift
Normal file
198
native/Sources/MotionLab/Output.swift
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
import AVFoundation
|
||||
import CoreMediaIO
|
||||
import ImageIO
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
func writePNG(bgra pixels: inout [UInt8], width: Int, height: Int, to url: URL) {
|
||||
let bitmapInfo = CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue
|
||||
guard let context = CGContext(data: &pixels, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4,
|
||||
space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: bitmapInfo),
|
||||
let image = context.makeImage(),
|
||||
let destination = CGImageDestinationCreateWithURL(url as CFURL, UTType.png.identifier as CFString, 1, nil)
|
||||
else { return }
|
||||
CGImageDestinationAddImage(destination, image, nil)
|
||||
CGImageDestinationFinalize(destination)
|
||||
}
|
||||
|
||||
/// Writes frames to an H.264 .mp4 file.
|
||||
final class Recorder {
|
||||
let url: URL
|
||||
private let writer: AVAssetWriter
|
||||
private let input: AVAssetWriterInput
|
||||
private let adaptor: AVAssetWriterInputPixelBufferAdaptor
|
||||
private let width: Int, height: Int
|
||||
private var startTime: Double?
|
||||
|
||||
init(url: URL, width: Int, height: Int) throws {
|
||||
self.url = url
|
||||
self.width = width
|
||||
self.height = height
|
||||
writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
|
||||
input = AVAssetWriterInput(mediaType: .video, outputSettings: [
|
||||
AVVideoCodecKey: AVVideoCodecType.h264,
|
||||
AVVideoWidthKey: width,
|
||||
AVVideoHeightKey: height,
|
||||
])
|
||||
input.expectsMediaDataInRealTime = true
|
||||
adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [
|
||||
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferWidthKey as String: width,
|
||||
kCVPixelBufferHeightKey as String: height,
|
||||
])
|
||||
writer.add(input)
|
||||
guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) }
|
||||
writer.startSession(atSourceTime: .zero)
|
||||
}
|
||||
|
||||
func append(bgra pixels: [UInt8], at time: Double) {
|
||||
guard input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return }
|
||||
var created: CVPixelBuffer?
|
||||
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created)
|
||||
guard let buffer = created else { return }
|
||||
|
||||
CVPixelBufferLockBaseAddress(buffer, [])
|
||||
if let base = CVPixelBufferGetBaseAddress(buffer) {
|
||||
let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
|
||||
pixels.withUnsafeBytes { source in
|
||||
for y in 0..<height {
|
||||
memcpy(base + y * rowBytes, source.baseAddress! + y * width * 4, width * 4)
|
||||
}
|
||||
}
|
||||
}
|
||||
CVPixelBufferUnlockBaseAddress(buffer, [])
|
||||
|
||||
if startTime == nil { startTime = time }
|
||||
adaptor.append(buffer, withPresentationTime: CMTime(seconds: time - startTime!, preferredTimescale: 600))
|
||||
}
|
||||
|
||||
func finish() {
|
||||
input.markAsFinished()
|
||||
let done = DispatchSemaphore(value: 0)
|
||||
writer.finishWriting { done.signal() }
|
||||
done.wait()
|
||||
}
|
||||
}
|
||||
|
||||
struct VirtualCameraError: Error, CustomStringConvertible {
|
||||
let description: String
|
||||
}
|
||||
|
||||
/// Feeds frames into OBS's virtual camera, so Meet, Zoom and anything else that takes a webcam can use them.
|
||||
/// OBS itself must not be running its own virtual camera at the same time.
|
||||
final class VirtualCamera {
|
||||
// fixed identifier of the camera OBS installs (from obs-studio's mac-virtualcam plugin)
|
||||
private static let obsDeviceUID = "7626645E-4425-469E-9D8B-97E0FA59AC75"
|
||||
|
||||
private let deviceID: CMIODeviceID
|
||||
private let streamID: CMIOStreamID
|
||||
private let queue: CMSimpleQueue
|
||||
private let format: CMFormatDescription
|
||||
private let pool: CVPixelBufferPool
|
||||
|
||||
init(width: Int, height: Int) throws {
|
||||
func address(_ selector: Int) -> CMIOObjectPropertyAddress {
|
||||
CMIOObjectPropertyAddress(mSelector: CMIOObjectPropertySelector(selector),
|
||||
mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
|
||||
mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain))
|
||||
}
|
||||
func objects(of object: CMIOObjectID, _ selector: Int) -> [CMIOObjectID] {
|
||||
var property = address(selector)
|
||||
var size: UInt32 = 0, used: UInt32 = 0
|
||||
CMIOObjectGetPropertyDataSize(object, &property, 0, nil, &size)
|
||||
var ids = [CMIOObjectID](repeating: 0, count: Int(size) / MemoryLayout<CMIOObjectID>.size)
|
||||
CMIOObjectGetPropertyData(object, &property, 0, nil, size, &used, &ids)
|
||||
return ids
|
||||
}
|
||||
func uid(of device: CMIOObjectID) -> String? {
|
||||
var property = address(kCMIODevicePropertyDeviceUID)
|
||||
var value: Unmanaged<CFString>?
|
||||
var used: UInt32 = 0
|
||||
CMIOObjectGetPropertyData(device, &property, 0, nil, UInt32(MemoryLayout<Unmanaged<CFString>?>.size), &used, &value)
|
||||
return value?.takeRetainedValue() as String?
|
||||
}
|
||||
|
||||
let devices = objects(of: CMIOObjectID(kCMIOObjectSystemObject), kCMIOHardwarePropertyDevices)
|
||||
guard let device = devices.first(where: { uid(of: $0) == VirtualCamera.obsDeviceUID }) else {
|
||||
throw VirtualCameraError(description: """
|
||||
OBS Virtual Camera not found. One-time setup: enable OBS under System Settings > General > \
|
||||
Login Items & Extensions > Camera Extensions, then in OBS click Start Virtual Camera, Stop Virtual Camera, and quit OBS.
|
||||
""")
|
||||
}
|
||||
// the camera has two streams: [0] is what apps watch, [1] is where frames are fed in
|
||||
let streams = objects(of: device, kCMIODevicePropertyStreams)
|
||||
guard streams.count >= 2 else { throw VirtualCameraError(description: "OBS Virtual Camera has no input stream.") }
|
||||
deviceID = device
|
||||
streamID = streams[1]
|
||||
|
||||
var createdQueue: Unmanaged<CMSimpleQueue>?
|
||||
CMIOStreamCopyBufferQueue(streamID, { _, _, _ in }, nil, &createdQueue)
|
||||
var createdFormat: CMFormatDescription?
|
||||
CMVideoFormatDescriptionCreate(allocator: nil, codecType: kCVPixelFormatType_422YpCbCr8,
|
||||
width: Int32(width), height: Int32(height), extensions: nil,
|
||||
formatDescriptionOut: &createdFormat)
|
||||
var createdPool: CVPixelBufferPool?
|
||||
CVPixelBufferPoolCreate(nil, nil, [
|
||||
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_422YpCbCr8,
|
||||
kCVPixelBufferWidthKey: width,
|
||||
kCVPixelBufferHeightKey: height,
|
||||
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary,
|
||||
] as CFDictionary, &createdPool)
|
||||
guard let createdQueue, let createdFormat, let createdPool else {
|
||||
throw VirtualCameraError(description: "Could not set up the OBS Virtual Camera stream.")
|
||||
}
|
||||
queue = createdQueue.takeRetainedValue()
|
||||
format = createdFormat
|
||||
pool = createdPool
|
||||
guard CMIODeviceStartStream(deviceID, streamID) == noErr else {
|
||||
throw VirtualCameraError(description: "Could not start the OBS Virtual Camera stream (is OBS using it?).")
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
CMIODeviceStopStream(deviceID, streamID)
|
||||
}
|
||||
|
||||
/// Sends one frame. The picture is flipped back left-to-right: the preview is a mirror,
|
||||
/// but the people on the call should see the real thing.
|
||||
func send(bgra pixels: [UInt8], width: Int, height: Int) {
|
||||
var created: CVPixelBuffer?
|
||||
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &created)
|
||||
guard let buffer = created else { return }
|
||||
|
||||
CVPixelBufferLockBaseAddress(buffer, [])
|
||||
if let base = CVPixelBufferGetBaseAddress(buffer)?.assumingMemoryBound(to: UInt8.self) {
|
||||
let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
|
||||
pixels.withUnsafeBufferPointer { source in
|
||||
// BGRA -> UYVY (BT.601 video range): two pixels share one U and one V
|
||||
for y in 0..<height {
|
||||
let row = base + y * rowBytes
|
||||
for pair in 0..<(width / 2) {
|
||||
let first = (y * width + (width - 1 - pair * 2)) * 4
|
||||
let second = first - 4
|
||||
let b0 = Int(source[first]), g0 = Int(source[first + 1]), r0 = Int(source[first + 2])
|
||||
let b1 = Int(source[second]), g1 = Int(source[second + 1]), r1 = Int(source[second + 2])
|
||||
row[pair * 4] = UInt8(clamping: ((-38 * r0 - 74 * g0 + 112 * b0 + 128) >> 8) + 128)
|
||||
row[pair * 4 + 1] = UInt8(clamping: ((66 * r0 + 129 * g0 + 25 * b0 + 128) >> 8) + 16)
|
||||
row[pair * 4 + 2] = UInt8(clamping: ((112 * r0 - 94 * g0 - 18 * b0 + 128) >> 8) + 128)
|
||||
row[pair * 4 + 3] = UInt8(clamping: ((66 * r1 + 129 * g1 + 25 * b1 + 128) >> 8) + 16)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
CVPixelBufferUnlockBaseAddress(buffer, [])
|
||||
|
||||
var timing = CMSampleTimingInfo(
|
||||
duration: .invalid,
|
||||
presentationTimeStamp: CMTime(value: CMTimeValue(clock_gettime_nsec_np(CLOCK_UPTIME_RAW)), timescale: 1_000_000_000),
|
||||
decodeTimeStamp: .invalid)
|
||||
var sample: CMSampleBuffer?
|
||||
CMSampleBufferCreateForImageBuffer(allocator: nil, imageBuffer: buffer, dataReady: true, makeDataReadyCallback: nil,
|
||||
refcon: nil, formatDescription: format, sampleTiming: &timing, sampleBufferOut: &sample)
|
||||
if let sample {
|
||||
// the queue takes ownership of the sample
|
||||
if CMSimpleQueueEnqueue(queue, element: Unmanaged.passRetained(sample).toOpaque()) != noErr {
|
||||
Unmanaged.passUnretained(sample).release()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
92
native/Sources/MotionLab/Panel.swift
Normal file
92
native/Sources/MotionLab/Panel.swift
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import AppKit
|
||||
|
||||
private final class TopAlignedView: NSView {
|
||||
override var isFlipped: Bool { true } // so the scroll view starts at the top
|
||||
}
|
||||
|
||||
/// Side panel: every effect with a checkbox, and the sliders of the ones that are switched on.
|
||||
final class Panel: NSObject {
|
||||
static let width: CGFloat = 230
|
||||
|
||||
let view = NSScrollView()
|
||||
private let stack = NSStackView()
|
||||
private let statusLabel = NSTextField(labelWithString: "")
|
||||
private let renderer: Renderer
|
||||
|
||||
init(renderer: Renderer) {
|
||||
self.renderer = renderer
|
||||
super.init()
|
||||
stack.orientation = .vertical
|
||||
stack.alignment = .leading
|
||||
stack.spacing = 4
|
||||
stack.translatesAutoresizingMaskIntoConstraints = false
|
||||
|
||||
let document = TopAlignedView()
|
||||
document.translatesAutoresizingMaskIntoConstraints = false
|
||||
document.addSubview(stack)
|
||||
view.documentView = document
|
||||
view.hasVerticalScroller = true
|
||||
view.drawsBackground = false
|
||||
NSLayoutConstraint.activate([
|
||||
view.widthAnchor.constraint(equalToConstant: Panel.width),
|
||||
document.widthAnchor.constraint(equalTo: view.contentView.widthAnchor),
|
||||
stack.topAnchor.constraint(equalTo: document.topAnchor, constant: 14),
|
||||
stack.leadingAnchor.constraint(equalTo: document.leadingAnchor, constant: 14),
|
||||
stack.trailingAnchor.constraint(equalTo: document.trailingAnchor, constant: -14),
|
||||
stack.bottomAnchor.constraint(equalTo: document.bottomAnchor, constant: -14),
|
||||
])
|
||||
statusLabel.font = .boldSystemFont(ofSize: 11)
|
||||
statusLabel.textColor = .systemRed
|
||||
rebuild()
|
||||
}
|
||||
|
||||
private func label(_ text: String, dim: Bool = true) -> NSTextField {
|
||||
let field = NSTextField(labelWithString: text)
|
||||
field.font = .systemFont(ofSize: 11)
|
||||
field.textColor = dim ? .secondaryLabelColor : .labelColor
|
||||
return field
|
||||
}
|
||||
|
||||
func rebuild() {
|
||||
stack.arrangedSubviews.forEach { $0.removeFromSuperview() }
|
||||
statusLabel.stringValue = renderer.status
|
||||
stack.addArrangedSubview(statusLabel)
|
||||
|
||||
for (layerIndex, layer) in renderer.layers.enumerated() {
|
||||
let title = layer.error == nil ? layer.name : "\(layer.name) (shader error)"
|
||||
let checkbox = NSButton(checkboxWithTitle: title, target: self, action: #selector(toggled(_:)))
|
||||
checkbox.state = layer.enabled ? .on : .off
|
||||
checkbox.tag = layerIndex
|
||||
checkbox.font = .systemFont(ofSize: 13, weight: .medium)
|
||||
stack.addArrangedSubview(checkbox)
|
||||
guard layer.enabled else { continue }
|
||||
|
||||
for (paramIndex, param) in layer.params.enumerated() {
|
||||
let slider = NSSlider(value: Double(param.value), minValue: Double(param.min), maxValue: Double(param.max),
|
||||
target: self, action: #selector(sliderMoved(_:)))
|
||||
slider.tag = layerIndex * maxParams + paramIndex
|
||||
slider.controlSize = .small
|
||||
stack.addArrangedSubview(label(param.name))
|
||||
stack.addArrangedSubview(slider)
|
||||
slider.widthAnchor.constraint(equalTo: stack.widthAnchor).isActive = true
|
||||
}
|
||||
stack.setCustomSpacing(12, after: stack.arrangedSubviews.last!)
|
||||
}
|
||||
if renderer.layers.isEmpty { stack.addArrangedSubview(label("no .metal files found")) }
|
||||
|
||||
stack.setCustomSpacing(16, after: stack.arrangedSubviews.last!)
|
||||
stack.addArrangedSubview(label("s screenshot\nr record\nq quit"))
|
||||
}
|
||||
|
||||
func refreshStatus() {
|
||||
statusLabel.stringValue = renderer.status
|
||||
}
|
||||
|
||||
@objc private func toggled(_ checkbox: NSButton) {
|
||||
renderer.setEnabled(checkbox.tag, checkbox.state == .on)
|
||||
}
|
||||
|
||||
@objc private func sliderMoved(_ slider: NSSlider) {
|
||||
renderer.setParam(layer: slider.tag / maxParams, param: slider.tag % maxParams, value: Float(slider.doubleValue))
|
||||
}
|
||||
}
|
||||
287
native/Sources/MotionLab/Renderer.swift
Normal file
287
native/Sources/MotionLab/Renderer.swift
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
import MetalKit
|
||||
|
||||
/// One effect file in the chain.
|
||||
final class Layer {
|
||||
let url: URL
|
||||
let name: String
|
||||
var enabled = false
|
||||
var order = 50
|
||||
var params: [EffectParam] = []
|
||||
var error: String?
|
||||
fileprivate var pipeline: MTLRenderPipelineState?
|
||||
fileprivate var modified: Date?
|
||||
fileprivate var usesParticles = false
|
||||
fileprivate var targets: [MTLTexture] = [] // [this frame, previous frame], swapped after every draw
|
||||
|
||||
init(url: URL) {
|
||||
self.url = url
|
||||
self.name = url.deletingPathExtension().lastPathComponent
|
||||
}
|
||||
}
|
||||
|
||||
private let passthrough = "float4 effect(float2 uv, Inputs in) { return in.image.sample(in.smp, uv); }"
|
||||
|
||||
/// Runs every enabled effect in order on each new camera frame, shows the result in the window,
|
||||
/// and hands it to whatever outputs are active (recording, virtual camera, screenshots).
|
||||
final class Renderer: NSObject, MTKViewDelegate {
|
||||
private let device: MTLDevice
|
||||
private let queue: MTLCommandQueue
|
||||
private let camera: Camera
|
||||
private let effectsDirectory: URL
|
||||
private let copyPipeline: MTLRenderPipelineState
|
||||
private var analysis: Analysis?
|
||||
private var plain: MTLTexture? // holds the camera picture when no effect is enabled
|
||||
private var result: MTLTexture? // the finished frame
|
||||
private var lastSerial = -1
|
||||
private let startTime = CACurrentMediaTime()
|
||||
private var frameCount = 0
|
||||
private var pixels: [UInt8] = []
|
||||
|
||||
private(set) var layers: [Layer] = []
|
||||
/// Called when layers were added, removed, reloaded or toggled.
|
||||
var onLayersChanged: (() -> Void)?
|
||||
/// Called when recording or the virtual camera starts or stops.
|
||||
var onStatusChanged: (() -> Void)?
|
||||
|
||||
/// When set, the app exits after this many camera frames, first saving that frame as a PNG if a url is given
|
||||
/// (used to check effects without watching).
|
||||
var snapshot: (frames: Int, url: URL?)?
|
||||
/// When set, recording into this file starts with the first frame.
|
||||
var recordURL: URL?
|
||||
/// When true, the OBS virtual camera is opened on the first frame, at the size of the output.
|
||||
var wantsVirtualCamera = false
|
||||
private var virtualCamera: VirtualCamera? { didSet { onStatusChanged?() } }
|
||||
private var recorder: Recorder? { didSet { onStatusChanged?() } }
|
||||
private var screenshotRequested = false
|
||||
|
||||
var status: String {
|
||||
[recorder != nil ? "REC" : nil, virtualCamera != nil ? "CAM" : nil].compactMap { $0 }.joined(separator: " ")
|
||||
}
|
||||
|
||||
init(device: MTLDevice, camera: Camera, effectsDirectory: URL) throws {
|
||||
self.device = device
|
||||
self.camera = camera
|
||||
self.effectsDirectory = effectsDirectory
|
||||
self.queue = device.makeCommandQueue()!
|
||||
self.copyPipeline = try makePipeline(device, source: effectShader(body: passthrough, params: []),
|
||||
fragment: "effectFragment", format: .bgra8Unorm)
|
||||
super.init()
|
||||
refreshFiles()
|
||||
}
|
||||
|
||||
// MARK: effect files
|
||||
|
||||
/// Picks up new, deleted and edited effect files. On a compile error the last working version keeps running.
|
||||
private func refreshFiles() {
|
||||
let files = ((try? FileManager.default.contentsOfDirectory(at: effectsDirectory, includingPropertiesForKeys: nil)) ?? [])
|
||||
.filter { $0.pathExtension == "metal" }
|
||||
var changed = false
|
||||
|
||||
let before = layers.count
|
||||
layers.removeAll { layer in !files.contains(layer.url) }
|
||||
changed = layers.count != before
|
||||
for url in files where !layers.contains(where: { $0.url == url }) {
|
||||
layers.append(Layer(url: url))
|
||||
changed = true
|
||||
}
|
||||
for layer in layers where reload(layer) { changed = true }
|
||||
|
||||
if changed {
|
||||
layers.sort { ($0.order, $0.name) < ($1.order, $1.name) }
|
||||
onLayersChanged?()
|
||||
}
|
||||
}
|
||||
|
||||
private func reload(_ layer: Layer) -> Bool {
|
||||
let modified = (try? layer.url.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate
|
||||
guard let modified, modified != layer.modified else { return false }
|
||||
let firstLoad = layer.modified == nil
|
||||
layer.modified = modified
|
||||
do {
|
||||
let source = try String(contentsOf: layer.url, encoding: .utf8)
|
||||
var info = parseEffect(source)
|
||||
layer.pipeline = try makePipeline(device, source: effectShader(body: source, params: info.params),
|
||||
fragment: "effectFragment", format: .bgra8Unorm)
|
||||
for i in info.params.indices { // keep slider positions across reloads
|
||||
if let old = layer.params.first(where: { $0.name == info.params[i].name }) {
|
||||
info.params[i].value = min(max(old.value, info.params[i].min), info.params[i].max)
|
||||
}
|
||||
}
|
||||
layer.params = info.params
|
||||
layer.order = info.order
|
||||
layer.usesParticles = source.contains("in.particles")
|
||||
layer.error = nil
|
||||
if firstLoad { layer.enabled = info.startsOn }
|
||||
print("[\(layer.name)] loaded")
|
||||
} catch {
|
||||
layer.error = error.localizedDescription
|
||||
print("[\(layer.name)] shader error:\n\(error.localizedDescription)")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var failedLayers: [String] { layers.filter { $0.error != nil }.map(\.name) }
|
||||
|
||||
/// Enables exactly the named layers. Returns the names that don't exist.
|
||||
func enableOnly(_ names: [String]) -> [String] {
|
||||
for layer in layers { layer.enabled = names.contains(layer.name) }
|
||||
return names.filter { name in !layers.contains { $0.name == name } }
|
||||
}
|
||||
|
||||
func setEnabled(_ index: Int, _ enabled: Bool) {
|
||||
guard layers.indices.contains(index) else { return }
|
||||
layers[index].enabled = enabled
|
||||
if !enabled { layers[index].targets = [] } // free its memory, and start clean next time
|
||||
onLayersChanged?()
|
||||
}
|
||||
|
||||
func setParam(layer: Int, param: Int, value: Float) {
|
||||
guard layers.indices.contains(layer), layers[layer].params.indices.contains(param) else { return }
|
||||
let p = layers[layer].params[param]
|
||||
layers[layer].params[param].value = min(max(value, p.min), p.max)
|
||||
}
|
||||
|
||||
// MARK: outputs
|
||||
|
||||
func requestScreenshot() { screenshotRequested = true }
|
||||
|
||||
func toggleRecording() {
|
||||
if let recorder {
|
||||
self.recorder = nil
|
||||
recorder.finish()
|
||||
print("saved \(recorder.url.path)")
|
||||
} else if let result {
|
||||
do {
|
||||
recorder = try Recorder(url: capturesURL(extension: "mp4"), width: result.width, height: result.height)
|
||||
print("recording \(recorder!.url.path)")
|
||||
} catch {
|
||||
print("could not start recording: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func finish() {
|
||||
if recorder != nil { toggleRecording() }
|
||||
}
|
||||
|
||||
private func capturesURL(extension ext: String) -> URL {
|
||||
let directory = URL(fileURLWithPath: "captures")
|
||||
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
||||
let stamp = DateFormatter()
|
||||
stamp.dateFormat = "yyyyMMdd-HHmmss"
|
||||
return directory.appendingPathComponent("motionlab-\(stamp.string(from: Date())).\(ext)")
|
||||
}
|
||||
|
||||
/// Runs once per camera frame, after the GPU work was submitted.
|
||||
private func deliver(_ commands: MTLCommandBuffer) {
|
||||
frameCount += 1
|
||||
let snapshotDue = snapshot.map { frameCount >= $0.frames } ?? false
|
||||
guard let result else { return }
|
||||
let width = result.width, height = result.height
|
||||
if let url = recordURL {
|
||||
recordURL = nil
|
||||
do { recorder = try Recorder(url: url, width: width, height: height) } catch { print("could not start recording: \(error.localizedDescription)") }
|
||||
}
|
||||
if wantsVirtualCamera {
|
||||
wantsVirtualCamera = false
|
||||
do {
|
||||
virtualCamera = try VirtualCamera(width: width, height: height)
|
||||
print("virtual camera on: pick 'OBS Virtual Camera' as the camera in your call or streaming app")
|
||||
} catch {
|
||||
print("virtual camera off: \(error)")
|
||||
}
|
||||
}
|
||||
guard recorder != nil || virtualCamera != nil || screenshotRequested || snapshotDue else { return }
|
||||
|
||||
commands.waitUntilCompleted()
|
||||
if pixels.count != width * height * 4 { pixels = [UInt8](repeating: 0, count: width * height * 4) }
|
||||
result.getBytes(&pixels, bytesPerRow: width * 4, from: MTLRegionMake2D(0, 0, width, height), mipmapLevel: 0)
|
||||
|
||||
recorder?.append(bgra: pixels, at: CACurrentMediaTime())
|
||||
virtualCamera?.send(bgra: pixels, width: width, height: height)
|
||||
if screenshotRequested {
|
||||
screenshotRequested = false
|
||||
let url = capturesURL(extension: "png")
|
||||
writePNG(bgra: &pixels, width: width, height: height, to: url)
|
||||
print("saved \(url.path)")
|
||||
}
|
||||
if snapshotDue {
|
||||
if let url = snapshot?.url {
|
||||
writePNG(bgra: &pixels, width: width, height: height, to: url)
|
||||
print("\(frameCount) frames -> \(url.path)")
|
||||
}
|
||||
finish()
|
||||
exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: drawing
|
||||
|
||||
private func process(_ frames: (current: MTLTexture, last: MTLTexture, serial: Int), _ commands: MTLCommandBuffer) throws {
|
||||
let width = frames.current.width, height = frames.current.height
|
||||
if analysis == nil { analysis = try Analysis(device: device, queue: queue, width: width, height: height) }
|
||||
guard let analysis else { return }
|
||||
analysis.update(commands, camera: frames.current, particles: layers.contains { $0.enabled && $0.usesParticles })
|
||||
|
||||
let time = Float(CACurrentMediaTime() - startTime)
|
||||
var image = frames.current
|
||||
for layer in layers where layer.enabled {
|
||||
guard let pipeline = layer.pipeline else { continue }
|
||||
if layer.targets.first?.width != width || layer.targets.first?.height != height {
|
||||
layer.targets = [makeTarget(device, width: width, height: height), makeTarget(device, width: width, height: height)]
|
||||
for target in layer.targets { commands.pass(into: target, clear: true) }
|
||||
}
|
||||
// must match `struct Uniforms` in the shader: time, padding, resolution.xy, params[16]
|
||||
var uniforms: [Float] = [time, 0, Float(width), Float(height)]
|
||||
uniforms += layer.params.map(\.value) + [Float](repeating: 0, count: maxParams - layer.params.count)
|
||||
let textures = [image, frames.current, frames.last, layer.targets[1], analysis.flow, analysis.particles]
|
||||
commands.pass(into: layer.targets[0]) { $0.fullscreen(pipeline, textures: textures, uniforms: uniforms) }
|
||||
image = layer.targets[0]
|
||||
layer.targets.swapAt(0, 1)
|
||||
}
|
||||
|
||||
if image === frames.current {
|
||||
if plain?.width != width || plain?.height != height { plain = makeTarget(device, width: width, height: height) }
|
||||
commands.pass(into: plain!) { $0.fullscreen(copyPipeline, textures: [image], uniforms: [0, 0, 0, 0]) }
|
||||
image = plain!
|
||||
}
|
||||
result = image
|
||||
}
|
||||
|
||||
/// The largest rectangle with the output's aspect ratio that fits the window, centred.
|
||||
private func fitted(_ output: MTLTexture, in size: CGSize) -> MTLViewport {
|
||||
let scale = min(size.width / Double(output.width), size.height / Double(output.height))
|
||||
let width = Double(output.width) * scale, height = Double(output.height) * scale
|
||||
return MTLViewport(originX: (size.width - width) / 2, originY: (size.height - height) / 2,
|
||||
width: width, height: height, znear: 0, zfar: 1)
|
||||
}
|
||||
|
||||
func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {}
|
||||
|
||||
func draw(in view: MTKView) {
|
||||
guard let frames = camera.latest, let commands = queue.makeCommandBuffer() else { return }
|
||||
|
||||
// effects advance once per camera frame; in between, the window just shows the last result again
|
||||
let isNewFrame = frames.serial != lastSerial
|
||||
if isNewFrame {
|
||||
lastSerial = frames.serial
|
||||
if frameCount % 15 == 0 { refreshFiles() }
|
||||
do {
|
||||
try process(frames, commands)
|
||||
} catch {
|
||||
print("render setup failed: \(error.localizedDescription)")
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
if let result, let drawable = view.currentDrawable, let windowPass = view.currentRenderPassDescriptor,
|
||||
let encoder = commands.makeRenderCommandEncoder(descriptor: windowPass) {
|
||||
encoder.setViewport(fitted(result, in: view.drawableSize))
|
||||
encoder.fullscreen(copyPipeline, textures: [result], uniforms: [0, 0, 0, 0])
|
||||
encoder.endEncoding()
|
||||
commands.present(drawable)
|
||||
}
|
||||
commands.commit()
|
||||
if isNewFrame { deliver(commands) }
|
||||
}
|
||||
}
|
||||
161
native/Sources/MotionLab/Shaders.swift
Normal file
161
native/Sources/MotionLab/Shaders.swift
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import Metal
|
||||
|
||||
/// A slider an effect file declares with a comment line: `// slider <name> <min> <max> <default>`
|
||||
struct EffectParam {
|
||||
let name: String
|
||||
let min: Float
|
||||
let max: Float
|
||||
var value: Float
|
||||
}
|
||||
|
||||
/// Everything an effect file says about itself in `//` comment lines:
|
||||
///
|
||||
/// // slider <name> <min> <max> <default> a slider, read in the shader as in.p.<name>
|
||||
/// // order <number> position in the chain, low runs first (default 50)
|
||||
/// // on enabled when the app starts
|
||||
struct EffectInfo {
|
||||
var params: [EffectParam] = []
|
||||
var order = 50
|
||||
var startsOn = false
|
||||
}
|
||||
|
||||
let maxParams = 16
|
||||
|
||||
func parseEffect(_ source: String) -> EffectInfo {
|
||||
var info = EffectInfo()
|
||||
for line in source.split(separator: "\n") {
|
||||
let parts = line.split(separator: " ")
|
||||
guard parts.count >= 2, parts[0] == "//" else { continue }
|
||||
switch parts[1] {
|
||||
case "slider" where parts.count == 6:
|
||||
if let min = Float(parts[3]), let max = Float(parts[4]), let value = Float(parts[5]) {
|
||||
info.params.append(EffectParam(name: String(parts[2]), min: min, max: max, value: value))
|
||||
}
|
||||
case "order" where parts.count == 3:
|
||||
info.order = Int(parts[2]) ?? info.order
|
||||
case "on" where parts.count == 2:
|
||||
info.startsOn = true
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
info.params = Array(info.params.prefix(maxParams))
|
||||
return info
|
||||
}
|
||||
|
||||
/// Shared by every shader in the app: a triangle that covers the whole target.
|
||||
let shaderPrelude = """
|
||||
#include <metal_stdlib>
|
||||
using namespace metal;
|
||||
|
||||
struct VertexOut {
|
||||
float4 position [[position]];
|
||||
float2 uv;
|
||||
};
|
||||
|
||||
vertex VertexOut fullscreenVertex(uint id [[vertex_id]]) {
|
||||
float2 p = float2((id << 1) & 2, id & 2);
|
||||
VertexOut out;
|
||||
out.position = float4(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0, 0.0, 1.0);
|
||||
out.uv = p;
|
||||
return out;
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
/// Wraps an effect file into a complete Metal shader. An effect file only defines:
|
||||
///
|
||||
/// float4 effect(float2 uv, Inputs in) { ... }
|
||||
///
|
||||
/// `uv` runs from (0,0) top-left to (1,1) bottom-right.
|
||||
func effectShader(body: String, params: [EffectParam]) -> String {
|
||||
let fields = params.isEmpty ? "float unused;" : params.map { "float \($0.name);" }.joined(separator: " ")
|
||||
let values = params.isEmpty ? "0.0" : params.indices.map { "u.params[\($0)]" }.joined(separator: ", ")
|
||||
return shaderPrelude + """
|
||||
struct Uniforms {
|
||||
float time;
|
||||
float2 resolution;
|
||||
float params[\(maxParams)];
|
||||
};
|
||||
|
||||
struct Params { \(fields) };
|
||||
|
||||
struct Inputs {
|
||||
texture2d<float> image; // the picture so far: output of the effect above, or the camera for the first one
|
||||
texture2d<float> camera; // the untouched camera frame
|
||||
texture2d<float> last; // the camera frame before this one (compare with camera to find motion)
|
||||
texture2d<float> prev; // this effect's own output from the previous frame (for trails)
|
||||
texture2d<float> flow; // .xy = which way and how far things moved since the last frame, in uv units
|
||||
texture2d<float> particles; // .r = brightness of particles that drift along with the motion
|
||||
sampler smp;
|
||||
float time; // seconds since start
|
||||
float2 resolution; // output size in pixels
|
||||
Params p; // this effect's sliders
|
||||
};
|
||||
|
||||
#line 1
|
||||
\(body)
|
||||
|
||||
fragment float4 effectFragment(VertexOut v [[stage_in]],
|
||||
texture2d<float> image [[texture(0)]],
|
||||
texture2d<float> camera [[texture(1)]],
|
||||
texture2d<float> last [[texture(2)]],
|
||||
texture2d<float> prev [[texture(3)]],
|
||||
texture2d<float> flow [[texture(4)]],
|
||||
texture2d<float> particles [[texture(5)]],
|
||||
constant Uniforms &u [[buffer(0)]]) {
|
||||
constexpr sampler smp(filter::linear, address::clamp_to_edge);
|
||||
Params p = { \(values) };
|
||||
Inputs in = { image, camera, last, prev, flow, particles, smp, u.time, u.resolution, p };
|
||||
return effect(v.uv, in);
|
||||
}
|
||||
"""
|
||||
}
|
||||
|
||||
func makePipeline(_ device: MTLDevice, source: String, vertex: String = "fullscreenVertex", fragment: String,
|
||||
format: MTLPixelFormat, additive: Bool = false) throws -> MTLRenderPipelineState {
|
||||
let library = try device.makeLibrary(source: source, options: nil)
|
||||
let descriptor = MTLRenderPipelineDescriptor()
|
||||
descriptor.vertexFunction = library.makeFunction(name: vertex)
|
||||
descriptor.fragmentFunction = library.makeFunction(name: fragment)
|
||||
let attachment = descriptor.colorAttachments[0]!
|
||||
attachment.pixelFormat = format
|
||||
if additive {
|
||||
attachment.isBlendingEnabled = true
|
||||
attachment.sourceRGBBlendFactor = .one
|
||||
attachment.destinationRGBBlendFactor = .one
|
||||
attachment.sourceAlphaBlendFactor = .one
|
||||
attachment.destinationAlphaBlendFactor = .one
|
||||
}
|
||||
return try device.makeRenderPipelineState(descriptor: descriptor)
|
||||
}
|
||||
|
||||
func makeTarget(_ device: MTLDevice, width: Int, height: Int, format: MTLPixelFormat = .bgra8Unorm) -> MTLTexture {
|
||||
let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width, height: height, mipmapped: false)
|
||||
descriptor.usage = [.renderTarget, .shaderRead]
|
||||
return device.makeTexture(descriptor: descriptor)!
|
||||
}
|
||||
|
||||
extension MTLCommandBuffer {
|
||||
/// One render pass into `target`. With `clear`, the target starts black; otherwise its old contents are ignored.
|
||||
func pass(into target: MTLTexture, clear: Bool = false, _ body: (MTLRenderCommandEncoder) -> Void = { _ in }) {
|
||||
let descriptor = MTLRenderPassDescriptor()
|
||||
descriptor.colorAttachments[0].texture = target
|
||||
descriptor.colorAttachments[0].loadAction = clear ? .clear : .dontCare
|
||||
descriptor.colorAttachments[0].storeAction = .store
|
||||
guard let encoder = makeRenderCommandEncoder(descriptor: descriptor) else { return }
|
||||
body(encoder)
|
||||
encoder.endEncoding()
|
||||
}
|
||||
}
|
||||
|
||||
extension MTLRenderCommandEncoder {
|
||||
func fullscreen(_ pipeline: MTLRenderPipelineState, textures: [MTLTexture], uniforms: [Float] = []) {
|
||||
setRenderPipelineState(pipeline)
|
||||
setFragmentTextures(textures, range: 0..<textures.count)
|
||||
if !uniforms.isEmpty {
|
||||
setFragmentBytes(uniforms, length: uniforms.count * MemoryLayout<Float>.stride, index: 0)
|
||||
}
|
||||
drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3)
|
||||
}
|
||||
}
|
||||
110
native/Sources/MotionLab/main.swift
Normal file
110
native/Sources/MotionLab/main.swift
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
// MotionLab: camera -> chain of Metal shader effects -> window, recording, virtual camera.
|
||||
//
|
||||
// swift run -c release MotionLab [options]
|
||||
//
|
||||
// --effects DIR folder of .metal effect files (default ./effects)
|
||||
// --only a,b,c start with exactly these effects enabled
|
||||
// --virtualcam also send the output to the OBS virtual camera (for Meet, Zoom, ...)
|
||||
// --snapshot OUT.png save one frame after --frames N camera frames (default 90) and exit
|
||||
// --record OUT.mp4 record --frames N camera frames (default 90) and exit
|
||||
//
|
||||
// In the window: s saves a screenshot, r starts/stops recording (both into ./captures), q or Esc quits.
|
||||
// Effect files are reloaded when saved.
|
||||
|
||||
import AppKit
|
||||
import MetalKit
|
||||
|
||||
func option(_ name: String) -> String? {
|
||||
let arguments = CommandLine.arguments
|
||||
guard let i = arguments.firstIndex(of: name), i + 1 < arguments.count else { return nil }
|
||||
return arguments[i + 1]
|
||||
}
|
||||
|
||||
func fail(_ message: String) -> Never {
|
||||
FileHandle.standardError.write((message + "\n").data(using: .utf8)!)
|
||||
exit(1)
|
||||
}
|
||||
|
||||
final class AppDelegate: NSObject, NSApplicationDelegate {
|
||||
var onQuit: (() -> Void)?
|
||||
func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
|
||||
func applicationWillTerminate(_ notification: Notification) { onQuit?() }
|
||||
}
|
||||
|
||||
setvbuf(stdout, nil, _IOLBF, 0) // show "loaded" / shader errors immediately, even when piped
|
||||
|
||||
guard let device = MTLCreateSystemDefaultDevice() else { fail("Metal is not available on this machine.") }
|
||||
let effectsDirectory = URL(fileURLWithPath: option("--effects") ?? "effects")
|
||||
|
||||
let renderer: Renderer
|
||||
do {
|
||||
renderer = try Renderer(device: device, camera: Camera(device: device), effectsDirectory: effectsDirectory)
|
||||
} catch {
|
||||
fail("\(error)")
|
||||
}
|
||||
if renderer.layers.isEmpty { print("No .metal files in \(effectsDirectory.path) - showing the plain camera.") }
|
||||
if let names = option("--only") {
|
||||
let unknown = renderer.enableOnly(names.split(separator: ",").map(String.init))
|
||||
if !unknown.isEmpty {
|
||||
fail("Unknown effect(s): \(unknown.joined(separator: ", ")). Available: \(renderer.layers.map(\.name).joined(separator: ", "))")
|
||||
}
|
||||
}
|
||||
if option("--snapshot") != nil || option("--record") != nil {
|
||||
if !renderer.failedLayers.isEmpty { fail("shader compile failed: \(renderer.failedLayers.joined(separator: ", "))") }
|
||||
renderer.snapshot = (Int(option("--frames") ?? "90") ?? 90, option("--snapshot").map { URL(fileURLWithPath: $0) })
|
||||
renderer.recordURL = option("--record").map { URL(fileURLWithPath: $0) }
|
||||
}
|
||||
renderer.wantsVirtualCamera = CommandLine.arguments.contains("--virtualcam")
|
||||
|
||||
let app = NSApplication.shared
|
||||
let delegate = AppDelegate()
|
||||
delegate.onQuit = { renderer.finish() }
|
||||
app.delegate = delegate
|
||||
app.setActivationPolicy(.regular)
|
||||
|
||||
let size = NSSize(width: 960 + Panel.width, height: 540)
|
||||
let window = NSWindow(contentRect: NSRect(origin: .zero, size: size),
|
||||
styleMask: [.titled, .closable, .miniaturizable, .resizable],
|
||||
backing: .buffered, defer: false)
|
||||
let view = MTKView(frame: .zero, device: device)
|
||||
view.colorPixelFormat = .bgra8Unorm
|
||||
view.delegate = renderer
|
||||
|
||||
let panel = Panel(renderer: renderer)
|
||||
renderer.onLayersChanged = { panel.rebuild() }
|
||||
renderer.onStatusChanged = { panel.refreshStatus() }
|
||||
|
||||
NSEvent.addLocalMonitorForEvents(matching: .keyDown) { event in
|
||||
if event.modifierFlags.contains(.command) { return event }
|
||||
switch event.charactersIgnoringModifiers {
|
||||
case "s": renderer.requestScreenshot()
|
||||
case "r": renderer.toggleRecording()
|
||||
case "q", "\u{1B}": app.terminate(nil)
|
||||
default: return event
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
let content = NSView(frame: NSRect(origin: .zero, size: size))
|
||||
for subview in [view, panel.view] {
|
||||
subview.translatesAutoresizingMaskIntoConstraints = false
|
||||
content.addSubview(subview)
|
||||
}
|
||||
NSLayoutConstraint.activate([
|
||||
view.leadingAnchor.constraint(equalTo: content.leadingAnchor),
|
||||
view.topAnchor.constraint(equalTo: content.topAnchor),
|
||||
view.bottomAnchor.constraint(equalTo: content.bottomAnchor),
|
||||
view.trailingAnchor.constraint(equalTo: panel.view.leadingAnchor),
|
||||
panel.view.trailingAnchor.constraint(equalTo: content.trailingAnchor),
|
||||
panel.view.topAnchor.constraint(equalTo: content.topAnchor),
|
||||
panel.view.bottomAnchor.constraint(equalTo: content.bottomAnchor),
|
||||
])
|
||||
|
||||
window.title = "MotionLab"
|
||||
window.contentMinSize = NSSize(width: 480 + Panel.width, height: 270)
|
||||
window.contentView = content
|
||||
window.setContentSize(size) // auto layout would otherwise pick its own window size
|
||||
window.center()
|
||||
window.makeKeyAndOrderFront(nil)
|
||||
app.activate(ignoringOtherApps: true)
|
||||
app.run()
|
||||
Loading…
Add table
Add a link
Reference in a new issue