๐Ÿ— KeyzHub
19Keys ยท community archive
4850 bytes raw
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
// personmatte.swift โ€” Apple Vision person-matte extractor for masked luxury grading.
//
//   Usage: personmatte <video> <outW> <outH> [fast|balanced|accurate]
//
// Decodes the video SEQUENTIALLY (AVAssetReader โ€” never per-frame seeks), runs ONE
// reused VNGeneratePersonSegmentationRequest (Apple's documented pattern: the request
// keeps temporal state across frames, which stabilises the matte), scales each matte
// to <outW>x<outH> and writes raw 8-bit gray frames to stdout (outW*outH bytes per
// frame, full range 0-255, white = person). Pipe into:
//   ffmpeg -f rawvideo -pix_fmt gray -s WxH -r FPS -i - -c:v libx264 -crf 6 \
//          -pix_fmt gray -color_range pc matte.mp4
// Prints "personmatte: frames=N coverage=F" to stderr on completion.
//
//   Build: swiftc -O -o personmatte personmatte.swift

import AVFoundation
import CoreImage
import Foundation
import Vision

func fail(_ msg: String) -> Never {
    FileHandle.standardError.write(Data(("personmatte: " + msg + "\n").utf8))
    exit(1)
}

let args = CommandLine.arguments
guard args.count >= 4, let outW = Int(args[2]), let outH = Int(args[3]), outW > 0, outH > 0 else {
    fail("usage: personmatte <video> <outW> <outH> [fast|balanced|accurate]")
}
let url = URL(fileURLWithPath: args[1])
let quality = args.count > 4 ? args[4] : "accurate"

let asset = AVURLAsset(url: url)
// loadTracks(withMediaType:) is the non-deprecated async API โ€” bridge to sync.
var loadedTrack: AVAssetTrack?
let sem = DispatchSemaphore(value: 0)
Task {
    loadedTrack = try? await asset.loadTracks(withMediaType: .video).first
    sem.signal()
}
sem.wait()
guard let track = loadedTrack else { fail("no video track in \(args[1])") }

guard let reader = try? AVAssetReader(asset: asset) else { fail("cannot open AVAssetReader") }
let output = AVAssetReaderTrackOutput(
    track: track,
    outputSettings: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA])
output.alwaysCopiesSampleData = false
reader.add(output)
guard reader.startReading() else { fail("reader failed: \(String(describing: reader.error))") }

// ONE request instance reused across every frame (temporal smoothing).
let request = VNGeneratePersonSegmentationRequest()
switch quality {
case "fast": request.qualityLevel = .fast
case "balanced": request.qualityLevel = .balanced
default: request.qualityLevel = .accurate
}
request.outputPixelFormat = kCVPixelFormatType_OneComponent8

// No color management โ€” matte values pass through untouched.
let ciContext = CIContext(options: [.workingColorSpace: NSNull(), .outputColorSpace: NSNull()])
var scaledPB: CVPixelBuffer?
CVPixelBufferCreate(kCFAllocatorDefault, outW, outH, kCVPixelFormatType_OneComponent8, nil, &scaledPB)
guard let outBuf = scaledPB else { fail("cannot allocate \(outW)x\(outH) matte buffer") }

let out = FileHandle.standardOutput
let black = Data(count: outW * outH)
var frames = 0
var coverage = 0.0

while let sample = output.copyNextSampleBuffer() {
    guard let frame = CMSampleBufferGetImageBuffer(sample) else { continue }
    do { try VNImageRequestHandler(cvPixelBuffer: frame, options: [:]).perform([request]) }
    catch { fail("vision inference failed at frame \(frames): \(error)") }

    var data: Data
    if let mask = request.results?.first?.pixelBuffer {
        // Vision's matte resolution depends on qualityLevel โ€” scale to output dims.
        let mw = CVPixelBufferGetWidth(mask), mh = CVPixelBufferGetHeight(mask)
        let scaled = CIImage(cvPixelBuffer: mask).transformed(
            by: CGAffineTransform(scaleX: CGFloat(outW) / CGFloat(mw),
                                  y: CGFloat(outH) / CGFloat(mh)))
        ciContext.render(scaled, to: outBuf,
                         bounds: CGRect(x: 0, y: 0, width: outW, height: outH), colorSpace: nil)
        CVPixelBufferLockBaseAddress(outBuf, .readOnly)
        let base = CVPixelBufferGetBaseAddress(outBuf)!
        let stride = CVPixelBufferGetBytesPerRow(outBuf)
        data = Data(capacity: outW * outH)
        for y in 0..<outH { data.append(Data(bytes: base + y * stride, count: outW)) }
        CVPixelBufferUnlockBaseAddress(outBuf, .readOnly)
    } else {
        data = black  // no person found this frame
    }

    // Sampled mean for the no-person fallback signal (~every 251st pixel).
    var sum = 0, n = 0
    data.withUnsafeBytes { (p: UnsafeRawBufferPointer) in
        var i = 0
        while i < p.count { sum += Int(p[i]); n += 1; i += 251 }
    }
    coverage += Double(sum) / (255.0 * Double(max(n, 1)))
    out.write(data)
    frames += 1
}
if reader.status == .failed { fail("decode failed: \(String(describing: reader.error))") }
let mean = frames > 0 ? coverage / Double(frames) : 0
FileHandle.standardError.write(
    Data("personmatte: frames=\(frames) coverage=\(String(format: "%.4f", mean))\n".utf8))