import Metal /// Host-side motion analysis, shared by all effects. Every camera frame it produces: /// /// flow which way things moved (small 2-channel texture, uv units per frame) /// particles a picture of particles that spawn where something moves and drift along the flow /// /// Effects read these as `in.flow` and `in.particles`. final class Analysis { private static let flowWidth = 160, flowHeight = 90 private static let stateWidth = 128, stateHeight = 64 // one texel per particle private static let particleSpeed: Float = 1.5 private static let trailFade: Float = 0.88 // each pair is [this frame, previous frame], swapped at the start of every update private var luma: [MTLTexture] private var flowTextures: [MTLTexture] private var state: [MTLTexture] private var trail: [MTLTexture] private let lumaPipeline: MTLRenderPipelineState private let flowPipeline: MTLRenderPipelineState private let movePipeline: MTLRenderPipelineState private let fadePipeline: MTLRenderPipelineState private let dotPipeline: MTLRenderPipelineState var flow: MTLTexture { flowTextures[0] } var particles: MTLTexture { trail[0] } init(device: MTLDevice, queue: MTLCommandQueue, width: Int, height: Int) throws { func pair(_ w: Int, _ h: Int, _ format: MTLPixelFormat) -> [MTLTexture] { [makeTarget(device, width: w, height: h, format: format), makeTarget(device, width: w, height: h, format: format)] } luma = pair(Analysis.flowWidth, Analysis.flowHeight, .r16Float) flowTextures = pair(Analysis.flowWidth, Analysis.flowHeight, .rg16Float) state = pair(Analysis.stateWidth, Analysis.stateHeight, .rgba32Float) trail = pair(width, height, .r8Unorm) let source = shaderPrelude + analysisShaders lumaPipeline = try makePipeline(device, source: source, fragment: "lumaFragment", format: .r16Float) flowPipeline = try makePipeline(device, source: source, fragment: "flowFragment", format: .rg16Float) movePipeline = try makePipeline(device, source: source, fragment: "moveFragment", format: .rgba32Float) fadePipeline = try makePipeline(device, source: source, fragment: "fadeFragment", format: .r8Unorm) dotPipeline = try makePipeline(device, source: source, vertex: "dotVertex", fragment: "dotFragment", format: .r8Unorm, additive: true) // new textures hold garbage; start everything from zero if let commands = queue.makeCommandBuffer() { for texture in luma + flowTextures + state + trail { commands.pass(into: texture, clear: true) } commands.commit() } } func update(_ commands: MTLCommandBuffer, camera: MTLTexture, particles wantsParticles: Bool) { luma.swapAt(0, 1) commands.pass(into: luma[0]) { $0.fullscreen(lumaPipeline, textures: [camera]) } flowTextures.swapAt(0, 1) commands.pass(into: flowTextures[0]) { $0.fullscreen(flowPipeline, textures: [luma[0], luma[1], flowTextures[1]]) } guard wantsParticles else { return } state.swapAt(0, 1) commands.pass(into: state[0]) { $0.fullscreen(movePipeline, textures: [state[1], flowTextures[0], luma[0], luma[1]], uniforms: [Float.random(in: 0..<1000), Analysis.particleSpeed, 0, 0]) } trail.swapAt(0, 1) commands.pass(into: trail[0]) { encoder in encoder.fullscreen(fadePipeline, textures: [trail[1]], uniforms: [Analysis.trailFade, 0, 0, 0]) encoder.setRenderPipelineState(dotPipeline) encoder.setVertexTexture(state[0], index: 0) encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: Analysis.stateWidth * Analysis.stateHeight) } } } private let analysisShaders = """ static float brightness(float3 c) { return dot(c, float3(0.299, 0.587, 0.114)); } static float hash(float2 p) { return fract(sin(dot(p, float2(127.1, 311.7))) * 43758.5453); } // Camera -> small, blurred brightness image. Flow is measured on this. fragment float4 lumaFragment(VertexOut v [[stage_in]], texture2d camera [[texture(0)]]) { constexpr sampler s(filter::linear, address::clamp_to_edge); float2 texel = 1.0 / float2(camera.get_width(), camera.get_height()); float sum = 0.0; for (int y = 0; y < 4; y++) { for (int x = 0; x < 4; x++) { sum += brightness(camera.sample(s, v.uv + (float2(x, y) - 1.5) * 2.0 * texel).rgb); } } return float4(sum / 16.0); } // Lucas-Kanade optical flow: for each pixel, the shift that best explains how its 7x7 // neighbourhood changed between the previous and the current brightness image. fragment float4 flowFragment(VertexOut v [[stage_in]], texture2d now [[texture(0)]], texture2d before [[texture(1)]], texture2d flowBefore [[texture(2)]]) { constexpr sampler s(filter::linear, address::clamp_to_edge); float2 texel = 1.0 / float2(now.get_width(), now.get_height()); float a = 1e-3, b = 0.0, c = 1e-3, p = 0.0, q = 0.0; // the 1e-3 keeps flat areas at zero flow for (int y = -3; y <= 3; y++) { for (int x = -3; x <= 3; x++) { float2 uv = v.uv + float2(x, y) * texel; float ix = (now.sample(s, uv + float2(texel.x, 0.0)).r - now.sample(s, uv - float2(texel.x, 0.0)).r) * 0.5; float iy = (now.sample(s, uv + float2(0.0, texel.y)).r - now.sample(s, uv - float2(0.0, texel.y)).r) * 0.5; float it = now.sample(s, uv).r - before.sample(s, uv).r; a += ix * ix; b += ix * iy; c += iy * iy; p += ix * it; q += iy * it; } } float2 shift = -float2(c * p - b * q, a * q - b * p) / (a * c - b * b); // in small-image pixels float len = length(shift); if (len > 4.0) { shift *= 4.0 / len; } float2 smoothed = mix(flowBefore.sample(s, v.uv).xy, shift * texel, 0.6); return float4(smoothed, 0.0, 1.0); } // One texel per particle: xy = position (uv), z = frames left to live, w = 1 if it was born from motion. fragment float4 moveFragment(VertexOut v [[stage_in]], texture2d state [[texture(0)]], texture2d flow [[texture(1)]], texture2d now [[texture(2)]], texture2d before [[texture(3)]], constant float4 &u [[buffer(0)]]) { // x = random seed, y = speed constexpr sampler s(filter::linear, address::clamp_to_edge); float4 particle = state.read(uint2(v.position.xy)); float2 seed = v.position.xy * 0.37 + u.x; float2 pos = particle.xy + flow.sample(s, particle.xy).xy * u.y; pos += (float2(hash(seed), hash(seed + 7.3)) - 0.5) * 0.0015; float life = particle.z - 1.0; float born = particle.w; if (life <= 0.0 || pos.x < 0.0 || pos.x > 1.0 || pos.y < 0.0 || pos.y > 1.0) { // respawn: try a few random spots and take the first one where something is moving pos = float2(hash(seed + 1.1), hash(seed + 2.2)); born = 0.0; life = 2.0 + hash(seed + 5.5) * 4.0; // nothing moving: stay hidden and try again soon for (int i = 0; i < 6; i++) { float2 spot = float2(hash(seed + 3.3 + float(i)), hash(seed + 9.9 + float(i) * 1.7)); if (abs(now.sample(s, spot).r - before.sample(s, spot).r) > 0.04) { pos = spot; born = 1.0; life = 20.0 + hash(seed + 5.5) * 70.0; break; } } } return float4(pos, life, born); } // Particle trails: fade what was there, then add a dot per particle on top. fragment float4 fadeFragment(VertexOut v [[stage_in]], texture2d trail [[texture(0)]], constant float4 &u [[buffer(0)]]) { // x = fade constexpr sampler s(filter::linear, address::clamp_to_edge); return trail.sample(s, v.uv) * u.x; } struct DotOut { float4 position [[position]]; float size [[point_size]]; }; vertex DotOut dotVertex(uint id [[vertex_id]], texture2d state [[texture(0)]]) { uint width = state.get_width(); float4 particle = state.read(uint2(id % width, id / width)); DotOut out; out.position = float4(particle.x * 2.0 - 1.0, 1.0 - particle.y * 2.0, 0.0, 1.0); if (particle.w < 0.5) { out.position.x = 4.0; } // hidden particles are moved off screen out.size = 3.0; return out; } fragment float4 dotFragment(DotOut in [[stage_in]], float2 coord [[point_coord]]) { return float4(smoothstep(0.5, 0.1, length(coord - 0.5)) * 0.9); } """