motionlab/native/Sources/MotionLab/Camera.swift

import AVFoundation
import CoreVideo
import Metal

enum CameraError: Error, CustomStringConvertible {
    case noPermission
    case noCamera

    var description: String {
        switch self {
        case .noPermission:
            return "No camera permission. Allow your terminal under System Settings > Privacy & Security > Camera, then restart it."
        case .noCamera:
            return "No camera found."
        }
    }
}

/// Streams the built-in camera as Metal textures; `latest` is the newest frame and the one before it.
final class Camera: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
    private let session = AVCaptureSession()
    private var cache: CVMetalTextureCache?
    private let lock = NSLock()
    // each entry keeps its CVMetalTexture, which keeps the pixel buffer behind the Metal texture alive
    private var current: (texture: MTLTexture, backing: CVMetalTexture)?
    private var previous: (texture: MTLTexture, backing: CVMetalTexture)?

    private var serial = 0  // counts frames, so the renderer can tell a new frame from one it already saw

    var latest: (current: MTLTexture, last: MTLTexture, serial: Int)? {
        lock.lock()
        defer { lock.unlock() }
        guard let current else { return nil }
        return (current.texture, (previous ?? current).texture, serial)
    }

    init(device: MTLDevice) throws {
        super.init()
        guard Camera.requestAccess() else { throw CameraError.noPermission }
        // prefer the built-in camera over an iPhone that happens to be nearby
        guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .unspecified)
            ?? AVCaptureDevice.default(for: .video)
        else { throw CameraError.noCamera }

        CVMetalTextureCacheCreate(nil, nil, device, nil, &cache)
        let output = AVCaptureVideoDataOutput()
        output.videoSettings = [
            kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
            kCVPixelBufferMetalCompatibilityKey as String: true,
        ]
        output.alwaysDiscardsLateVideoFrames = true
        output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "motionlab.camera"))

        session.beginConfiguration()
        session.addInput(try AVCaptureDeviceInput(device: camera))
        session.addOutput(output)
        if session.canSetSessionPreset(.hd1280x720) { session.sessionPreset = .hd1280x720 }
        print("camera: \(camera.localizedName)")
        if let connection = output.connection(with: .video), connection.isVideoMirroringSupported {
            connection.automaticallyAdjustsVideoMirroring = false
            connection.isVideoMirrored = true  // behave like a mirror
        }
        session.commitConfiguration()
        session.startRunning()
    }

    private static func requestAccess() -> Bool {
        switch AVCaptureDevice.authorizationStatus(for: .video) {
        case .authorized:
            return true
        case .notDetermined:
            let done = DispatchSemaphore(value: 0)
            var granted = false
            AVCaptureDevice.requestAccess(for: .video) { granted = $0; done.signal() }
            done.wait()
            return granted
        default:
            return false
        }
    }

    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        guard let cache, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
        var created: CVMetalTexture?
        CVMetalTextureCacheCreateTextureFromImage(
            nil, cache, buffer, nil, .bgra8Unorm,
            CVPixelBufferGetWidth(buffer), CVPixelBufferGetHeight(buffer), 0, &created)
        guard let created, let frame = CVMetalTextureGetTexture(created) else { return }
        lock.lock()
        previous = current
        current = (frame, created)
        serial += 1
        lock.unlock()
    }
}