From 11b0ff3f1ecbbe8cf0d8510c9377ec43998afcad Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 22:08:40 -0400 Subject: [PATCH 01/14] Add SigLIP 2 image sorting: Core ML manager, Pets check, and demo app SigLIP2Manager loads the mobius-converted image and text encoders, embeds labels once, and scores images with Core ML's async API. SigLIP2Tokenizer ports Gemma BPE (identical ids to the Python tokenizer on 1,018 prompts incl. accents, CJK, emoji); the preprocessor reproduces PIL's antialiased bilinear resize. ImageSortCheck: zero-shot Oxford-IIIT Pets, 37 breed prompts, all 3,669 test photos: 94.85% (Python Core ML 94.77%), 195 photos/s with 4 in flight on an M5 Pro. ImageSortDemo animates photos into breed buckets (Show / Turbo, IMAGE_SORT_AUTOPLAY); Turbo sorts 1,000 photos in 5.2 s. Models load from SIGLIP2_MODEL_DIR until the Hugging Face repo exists. --- Package.swift | 3 + .../SigLIP2/SigLIP2ImagePreprocessor.swift | 95 ++++++ Sources/FluidUse/SigLIP2/SigLIP2Manager.swift | 106 +++++++ .../FluidUse/SigLIP2/SigLIP2Tokenizer.swift | 82 +++++ Sources/FluidUse/SigLIP2/SigLIP2Types.swift | 55 ++++ Sources/ImageSort/ImageSorter.swift | 55 ++++ Sources/ImageSort/PetsSample.swift | 165 ++++++++++ Sources/ImageSortCheck/main.swift | 75 +++++ Sources/ImageSortDemo/ContentView.swift | 249 +++++++++++++++ Sources/ImageSortDemo/ImageSortDemoApp.swift | 28 ++ Sources/ImageSortDemo/ImageSortModel.swift | 295 ++++++++++++++++++ Sources/ImageSortDemo/README.md | 19 ++ 12 files changed, 1227 insertions(+) create mode 100644 Sources/FluidUse/SigLIP2/SigLIP2ImagePreprocessor.swift create mode 100644 Sources/FluidUse/SigLIP2/SigLIP2Manager.swift create mode 100644 Sources/FluidUse/SigLIP2/SigLIP2Tokenizer.swift create mode 100644 Sources/FluidUse/SigLIP2/SigLIP2Types.swift create mode 100644 Sources/ImageSort/ImageSorter.swift create mode 100644 Sources/ImageSort/PetsSample.swift create mode 100644 Sources/ImageSortCheck/main.swift create mode 100644 Sources/ImageSortDemo/ContentView.swift create mode 100644 Sources/ImageSortDemo/ImageSortDemoApp.swift create mode 100644 Sources/ImageSortDemo/ImageSortModel.swift create mode 100644 Sources/ImageSortDemo/README.md diff --git a/Package.swift b/Package.swift index 1b032aa..9fc5e4f 100644 --- a/Package.swift +++ b/Package.swift @@ -50,6 +50,9 @@ let package = Package( .executableTarget(name: "SortDecisionsCheck", dependencies: ["SortAnything"]), .executableTarget(name: "SortDecisionsDemo", dependencies: ["SortAnything"], exclude: ["README.md"]), .executableTarget(name: "SortAnythingDemo", dependencies: ["SortAnything"], exclude: ["README.md"]), + .target(name: "ImageSort", dependencies: ["FluidUse"]), + .executableTarget(name: "ImageSortCheck", dependencies: ["ImageSort", "FluidUse"]), + .executableTarget(name: "ImageSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]), .testTarget( name: "FluidUseTests", dependencies: ["FluidUse", "LayaTetris"], resources: [.copy("Fixtures")] diff --git a/Sources/FluidUse/SigLIP2/SigLIP2ImagePreprocessor.swift b/Sources/FluidUse/SigLIP2/SigLIP2ImagePreprocessor.swift new file mode 100644 index 0000000..95123cd --- /dev/null +++ b/Sources/FluidUse/SigLIP2/SigLIP2ImagePreprocessor.swift @@ -0,0 +1,95 @@ +import CoreGraphics +import Foundation + +/// Matches the Hugging Face SigLIP image processor: PIL bilinear resize (antialiased when shrinking) to a square, +/// rescale to [0, 1], normalize per channel. Output is planar float32 `[3, size, size]`. +public enum SigLIP2ImagePreprocessor { + public static func pixels(from image: CGImage, config: SigLIP2Config) throws -> [Float] { + let width = image.width + let height = image.height + let size = config.imageSize + var rgba = [UInt8](repeating: 0, count: width * height * 4) + guard + let space = CGColorSpace(name: CGColorSpace.sRGB), + let context = CGContext( + data: &rgba, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4, + space: space, bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) + else { + throw SigLIP2Error.invalidInput("Could not decode a \(width)×\(height) image") + } + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + // Horizontal pass, then vertical, each rounded to 8 bits like PIL. Layout stays 4 bytes per pixel. + let horizontal = resample( + rgba, lines: height, inLength: width, outLength: size, sampleStride: 4, lineStride: width * 4, + outSampleStride: 4, outLineStride: size * 4, outCount: height * size * 4) + let resized = resample( + horizontal, lines: size, inLength: height, outLength: size, sampleStride: size * 4, lineStride: 4, + outSampleStride: size * 4, outLineStride: 4, outCount: size * size * 4) + var planar = [Float](repeating: 0, count: 3 * size * size) + for channel in 0..<3 { + let scale = 1 / (255 * config.imageStd[channel]) + let offset = config.imageMean[channel] / config.imageStd[channel] + let base = channel * size * size + for pixel in 0..<(size * size) { + planar[base + pixel] = Float(resized[pixel * 4 + channel]) * scale - offset + } + } + return planar + } + + /// One separable pass of PIL's `ImagingResample` with the triangle filter, rounding to 8 bits like PIL. + static func resample( + _ input: [UInt8], lines: Int, inLength: Int, outLength: Int, sampleStride: Int, lineStride: Int, + outSampleStride: Int, outLineStride: Int, outCount: Int + ) -> [UInt8] { + let scale = Double(inLength) / Double(outLength) + let filterScale = max(scale, 1) + var starts = [Int](repeating: 0, count: outLength) + var counts = [Int](repeating: 0, count: outLength) + let taps = Int((filterScale * 2).rounded(.up)) + 2 + var weights = [Float](repeating: 0, count: outLength * taps) + for out in 0.. 0 ? weight / total : 0) + } + starts[out] = low + counts[out] = min(high - low, taps) + } + var output = [UInt8](repeating: 0, count: outCount) + input.withUnsafeBufferPointer { source in + output.withUnsafeMutableBufferPointer { destination in + weights.withUnsafeBufferPointer { weight in + for line in 0.. SigLIP2Manager + { + let configURL = directory.appendingPathComponent("config.json") + guard let configData = try? Data(contentsOf: configURL) else { + throw SigLIP2Error.invalidAsset("Missing config.json in \(directory.path)") + } + let config = try JSONDecoder().decode(SigLIP2Config.self, from: configData) + let tokenizer = try SigLIP2Tokenizer( + tokenizerJsonURL: directory.appendingPathComponent("tokenizer.json"), length: config.textLength) + let configuration = MLModelConfiguration() + configuration.computeUnits = computeUnits + async let image = loadModel(named: "\(config.name)-image-\(config.precision)", in: directory, configuration) + async let text = loadModel(named: "\(config.name)-text-\(config.precision)", in: directory, configuration) + return try await SigLIP2Manager(config: config, tokenizer: tokenizer, imageModel: image, textModel: text) + } + + private static func loadModel( + named name: String, in directory: URL, _ configuration: MLModelConfiguration + ) async throws -> MLModel { + let compiled = directory.appendingPathComponent("\(name).mlmodelc") + let package = directory.appendingPathComponent("\(name).mlpackage") + let url: URL + if FileManager.default.fileExists(atPath: compiled.path) { + url = compiled + } else if FileManager.default.fileExists(atPath: package.path) { + url = try await MLModel.compileModel(at: package) + } else { + throw SigLIP2Error.invalidAsset("Missing \(name).mlmodelc or .mlpackage in \(directory.path)") + } + return try await MLModel.load(contentsOf: url, configuration: configuration) + } + + /// L2-normalized text embedding per label. Compute once per label set and reuse. + public func embed(labels: [String]) async throws -> [[Float]] { + var embeddings: [[Float]] = [] + for label in labels { + let ids = try tokenizer.encode(label) + let input = try MLMultiArray(shape: [1, NSNumber(value: ids.count)], dataType: .int32) + let pointer = input.dataPointer.assumingMemoryBound(to: Int32.self) + for (index, id) in ids.enumerated() { pointer[index] = id } + let output = try await textModel.prediction( + from: MLDictionaryFeatureProvider(dictionary: ["input_ids": MLFeatureValue(multiArray: input)])) + embeddings.append(try Self.vector(output, name: "text_embeds")) + } + return embeddings + } + + /// L2-normalized image embedding. + public func embed(image: CGImage) async throws -> [Float] { + let pixels = try SigLIP2ImagePreprocessor.pixels(from: image, config: config) + let size = NSNumber(value: config.imageSize) + let input = try MLMultiArray(shape: [1, 3, size, size], dataType: .float32) + pixels.withUnsafeBufferPointer { source in + input.dataPointer.assumingMemoryBound(to: Float.self).update(from: source.baseAddress!, count: pixels.count) + } + let output = try await imageModel.prediction( + from: MLDictionaryFeatureProvider(dictionary: ["pixel_values": MLFeatureValue(multiArray: input)])) + return try Self.vector(output, name: "image_embeds") + } + + /// Scores `image` against label embeddings from `embed(labels:)`. + public func classify(image: CGImage, labels: [String], labelEmbeddings: [[Float]]) async throws -> SigLIP2Answer { + guard labels.count == labelEmbeddings.count, !labels.isEmpty else { + throw SigLIP2Error.invalidInput("Expected one embedding per label") + } + return score(imageEmbedding: try await embed(image: image), labels: labels, labelEmbeddings: labelEmbeddings) + } + + public func score(imageEmbedding: [Float], labels: [String], labelEmbeddings: [[Float]]) -> SigLIP2Answer { + let similarities = labelEmbeddings.map { label in zip(label, imageEmbedding).reduce(0) { $0 + $1.0 * $1.1 } } + let probabilities = similarities.map { 1 / (1 + exp(-(config.logitScale * $0 + config.logitBias))) } + return SigLIP2Answer(labels: labels, similarities: similarities, probabilities: probabilities) + } + + private static func vector(_ output: MLFeatureProvider, name: String) throws -> [Float] { + guard let array = output.featureValue(for: name)?.multiArrayValue else { + throw SigLIP2Error.predictionFailed("Missing \(name)") + } + return (0..` appended, `` to a fixed length. +/// Text is lowercased first, matching SigLIP 2's training. +public final class SigLIP2Tokenizer: Sendable { + public let length: Int + public let padId: Int + public let eosId: Int + + private let vocab: [String: Int] + private let ranks: [String: Int] + + public convenience init(tokenizerJsonURL: URL, length: Int = 64) throws { + try self.init(data: Data(contentsOf: tokenizerJsonURL), length: length) + } + + public init(data: Data, length: Int = 64) throws { + guard + let root = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let model = root["model"] as? [String: Any], + let vocab = model["vocab"] as? [String: Int], + let merges = model["merges"] as? [[String]] + else { + throw SigLIP2Error.invalidAsset("tokenizer.json is not a Gemma BPE tokenizer") + } + guard let padId = vocab[""], let eosId = vocab[""] else { + throw SigLIP2Error.invalidAsset("tokenizer.json has no or token") + } + var ranks: [String: Int] = [:] + ranks.reserveCapacity(merges.count) + for (rank, pair) in merges.enumerated() where pair.count == 2 { + ranks[Self.key(pair[0], pair[1])] = rank + } + self.vocab = vocab + self.ranks = ranks + self.length = length + self.padId = padId + self.eosId = eosId + } + + /// Token ids for `text`, ``-terminated and padded to `length`. + public func encode(_ text: String) throws -> [Int32] { + var ids = tokenize(text.lowercased()) + ids.append(eosId) + guard ids.count <= length else { + throw SigLIP2Error.invalidInput("Label needs \(ids.count) tokens; the text encoder takes \(length)") + } + ids.append(contentsOf: repeatElement(padId, count: length - ids.count)) + return ids.map(Int32.init) + } + + /// Token ids without `` or padding. + public func tokenize(_ text: String) -> [Int] { + let normalized = text.replacingOccurrences(of: " ", with: "\u{2581}") + var symbols = normalized.unicodeScalars.map { String($0) } + while symbols.count > 1 { + var best: (rank: Int, index: Int)? + for index in 0..<(symbols.count - 1) { + if let rank = ranks[Self.key(symbols[index], symbols[index + 1])], rank < (best?.rank ?? .max) { + best = (rank, index) + } + } + guard let best else { break } + symbols[best.index] += symbols[best.index + 1] + symbols.remove(at: best.index + 1) + } + var ids: [Int] = [] + for symbol in symbols { + if let id = vocab[symbol] { + ids.append(id) + } else { + for byte in symbol.utf8 { + ids.append(vocab[String(format: "<0x%02X>", byte)] ?? vocab[""] ?? 3) + } + } + } + return ids + } + + private static func key(_ left: String, _ right: String) -> String { left + "\u{0}" + right } +} diff --git a/Sources/FluidUse/SigLIP2/SigLIP2Types.swift b/Sources/FluidUse/SigLIP2/SigLIP2Types.swift new file mode 100644 index 0000000..1bb6986 --- /dev/null +++ b/Sources/FluidUse/SigLIP2/SigLIP2Types.swift @@ -0,0 +1,55 @@ +import Foundation + +public enum SigLIP2Error: Error, LocalizedError { + case invalidAsset(String) + case invalidInput(String) + case predictionFailed(String) + + public var errorDescription: String? { + switch self { + case .invalidAsset(let reason): "Invalid SigLIP 2 asset: \(reason)" + case .invalidInput(let reason): "Invalid SigLIP 2 input: \(reason)" + case .predictionFailed(let reason): "SigLIP 2 prediction failed: \(reason)" + } + } +} + +/// Preprocessing and scoring constants written by the converter (`config.json`). +public struct SigLIP2Config: Codable, Sendable { + public let modelId: String + public let imageSize: Int + public let imageMean: [Float] + public let imageStd: [Float] + public let textLength: Int + public let logitScale: Float + public let logitBias: Float + public let embeddingDim: Int + public let precision: String + + enum CodingKeys: String, CodingKey { + case modelId = "model_id" + case imageSize = "image_size" + case imageMean = "image_mean" + case imageStd = "image_std" + case textLength = "text_length" + case logitScale = "logit_scale" + case logitBias = "logit_bias" + case embeddingDim = "embedding_dim" + case precision + } + + /// Package base name, e.g. `siglip2-base-patch16-256`. + public var name: String { modelId.split(separator: "/").last.map(String.init) ?? modelId } +} + +/// One image scored against a label set. +public struct SigLIP2Answer: Sendable { + public let labels: [String] + /// Cosine similarity per label. + public let similarities: [Float] + /// Independent `sigmoid(scale · cos + bias)` per label, as SigLIP was trained. + public let probabilities: [Float] + + public var selectedIndex: Int { similarities.indices.max { similarities[$0] < similarities[$1] } ?? 0 } + public var selectedLabel: String { labels[selectedIndex] } +} diff --git a/Sources/ImageSort/ImageSorter.swift b/Sources/ImageSort/ImageSorter.swift new file mode 100644 index 0000000..baec462 --- /dev/null +++ b/Sources/ImageSort/ImageSorter.swift @@ -0,0 +1,55 @@ +import CoreGraphics +import FluidUse +import Foundation +import ImageIO + +/// Sorts photos into breeds with SigLIP 2 on Core ML. Calls are not serialized: several `sort` calls may be in flight. +public final class ImageSorter: Sendable { + public struct Result: Sendable { + public let breed: String + /// SigLIP's own sigmoid probability for the chosen label. + public let probability: Float + public let milliseconds: Double + } + + public let breeds: [String] + private let manager: SigLIP2Manager + private let embeddings: [[Float]] + + private init(manager: SigLIP2Manager, breeds: [String], embeddings: [[Float]]) { + self.manager = manager + self.breeds = breeds + self.embeddings = embeddings + } + + /// Loads the encoders from `SIGLIP2_MODEL_DIR` and embeds the breed prompts once. + public static func load(breeds: [String] = PetsSample.breeds) async throws -> ImageSorter { + guard let path = ProcessInfo.processInfo.environment["SIGLIP2_MODEL_DIR"], !path.isEmpty else { + throw SigLIP2Error.invalidAsset("Set SIGLIP2_MODEL_DIR to the converted siglip2-base-patch16-256 folder") + } + let manager = try await SigLIP2Manager.load(from: URL(fileURLWithPath: path)) + let embeddings = try await manager.embed(labels: breeds.map(PetsSample.prompt(for:))) + return ImageSorter(manager: manager, breeds: breeds, embeddings: embeddings) + } + + public var modelName: String { manager.config.name } + + public func sort(_ item: PetItem) async throws -> Result { + let image = try Self.decode(item.file) + let start = DispatchTime.now().uptimeNanoseconds + let answer = try await manager.classify(image: image, labels: breeds, labelEmbeddings: embeddings) + let milliseconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e6 + return Result( + breed: answer.selectedLabel, probability: answer.probabilities[answer.selectedIndex], + milliseconds: milliseconds) + } + + public static func decode(_ file: URL) throws -> CGImage { + guard let source = CGImageSourceCreateWithURL(file as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { + throw SigLIP2Error.invalidInput("Could not decode \(file.lastPathComponent)") + } + return image + } +} diff --git a/Sources/ImageSort/PetsSample.swift b/Sources/ImageSort/PetsSample.swift new file mode 100644 index 0000000..24161bc --- /dev/null +++ b/Sources/ImageSort/PetsSample.swift @@ -0,0 +1,165 @@ +import Foundation + +/// One photo from the Oxford-IIIT Pets test split with its gold breed. +public struct PetItem: Codable, Sendable, Identifiable, Hashable { + public let id: Int + public let breed: String + /// Cached JPEG on disk. + public let file: URL +} + +/// Seeded sample of the Oxford-IIIT Pets test split (CC BY-SA 4.0), fetched from the Hugging Face dataset viewer +/// API on first use and cached locally. Nothing is bundled. +public enum PetsSample { + public static let attribution = "Oxford-IIIT Pets test split (Parkhi et al., 2012) · CC BY-SA 4.0" + public static let testCount = 3669 + + /// The 37 breeds in dataset label order. + public static let breeds = [ + "abyssinian", "american bulldog", "american pit bull terrier", "basset hound", "beagle", "bengal", "birman", + "bombay", "boxer", "british shorthair", "chihuahua", "egyptian mau", "english cocker spaniel", + "english setter", "german shorthaired", "great pyrenees", "havanese", "japanese chin", "keeshond", + "leonberger", "maine coon", "miniature pinscher", "newfoundland", "persian", "pomeranian", "pug", "ragdoll", + "russian blue", "saint bernard", "samoyed", "scottish terrier", "shiba inu", "siamese", "sphynx", + "staffordshire bull terrier", "wheaten terrier", "yorkshire terrier", + ] + + /// Prompt per breed, as scored in the mobius Pets check. + public static func prompt(for breed: String) -> String { "a photo of a \(breed), a type of pet." } + + static let endpoint = "https://datasets-server.huggingface.co/rows" + static let pageSize = 100 + + private struct Page: Decodable { + struct Entry: Decodable { + struct Row: Decodable { + struct Image: Decodable { let src: String } + let image: Image + let label: Int + } + let rowIndex: Int + let row: Row + + enum CodingKeys: String, CodingKey { + case row + case rowIndex = "row_idx" + } + } + let rows: [Entry] + } + + public static func cacheDirectory() -> URL { + FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + .appendingPathComponent("FluidUse/image-sort/oxford-pets-test") + } + + /// `count` photos in a seeded shuffled order (all 3,669 when `count` is nil). `progress` gets photos cached so far. + public static func load( + count: Int? = 1000, seed: UInt64 = 0, progress: (@Sendable (Int, Int) -> Void)? = nil + ) async throws -> [PetItem] { + let directory = cacheDirectory() + let manifestURL = directory.appendingPathComponent("manifest.json") + let manager = FileManager.default + try manager.createDirectory(at: directory, withIntermediateDirectories: true) + + var labels: [Int: Int] = [:] + if let data = try? Data(contentsOf: manifestURL), + let saved = try? JSONDecoder().decode([Int: Int].self, from: data), + saved.count == testCount + { + labels = saved + } + var sources: [Int: String] = [:] + if labels.count != testCount { + for offset in stride(from: 0, to: testCount, by: pageSize) { + for entry in try await page(offset: offset) { + labels[entry.rowIndex] = entry.row.label + sources[entry.rowIndex] = entry.row.image.src + } + } + try JSONEncoder().encode(labels).write(to: manifestURL) + } + + var generator = SeededGenerator(seed: seed) + let chosen = Array(labels.keys.sorted().shuffled(using: &generator).prefix(count ?? testCount)) + let missing = chosen.filter { !manager.fileExists(atPath: file(for: $0).path) } + if !missing.isEmpty { + if sources.isEmpty { + for offset in stride(from: 0, to: testCount, by: pageSize) { + for entry in try await page(offset: offset) { sources[entry.rowIndex] = entry.row.image.src } + } + } + try await download( + missing, sources: sources, done: chosen.count - missing.count, total: chosen.count, progress) + } + return chosen.map { PetItem(id: $0, breed: breeds[labels[$0]!], file: file(for: $0)) } + } + + static func file(for row: Int) -> URL { cacheDirectory().appendingPathComponent("\(row).jpg") } + + private static func page(offset: Int) async throws -> [Page.Entry] { + var components = URLComponents(string: endpoint)! + components.queryItems = [ + URLQueryItem(name: "dataset", value: "timm/oxford-iiit-pet"), + URLQueryItem(name: "config", value: "default"), + URLQueryItem(name: "split", value: "test"), + URLQueryItem(name: "offset", value: String(offset)), + URLQueryItem(name: "length", value: String(pageSize)), + ] + let data = try await fetch(components.url!) + return try JSONDecoder().decode(Page.self, from: data).rows + } + + /// GET with backoff: the dataset viewer answers bursts with 429 and a `Retry-After`. + static func fetch(_ url: URL, attempts: Int = 8) async throws -> Data { + var delay = 2.0 + for attempt in 1...attempts { + let (data, response) = try await URLSession.shared.data(from: url) + let http = response as? HTTPURLResponse + if http?.statusCode == 200 { return data } + guard attempt < attempts, http?.statusCode == 429 || (http?.statusCode ?? 0) >= 500 else { break } + let wait = (http?.value(forHTTPHeaderField: "Retry-After")).flatMap(Double.init) ?? delay + try await Task.sleep(for: .seconds(min(wait, 60))) + delay *= 2 + } + throw URLError(.badServerResponse) + } + + private static func download( + _ rows: [Int], sources: [Int: String], done: Int, total: Int, _ progress: (@Sendable (Int, Int) -> Void)? + ) async throws { + try await withThrowingTaskGroup(of: Void.self) { group in + var next = 0 + var finished = done + func launch() { + guard next < rows.count else { return } + let row = rows[next] + next += 1 + group.addTask { + guard let source = sources[row], let url = URL(string: source) else { throw URLError(.badURL) } + try await fetch(url).write(to: file(for: row), options: .atomic) + } + } + for _ in 0..<8 { launch() } + while try await group.next() != nil { + finished += 1 + progress?(finished, total) + launch() + } + } + } +} + +/// xorshift64*, so a seed gives the same sample on every machine. +struct SeededGenerator: RandomNumberGenerator { + private var state: UInt64 + + init(seed: UInt64) { state = seed &+ 0x9E37_79B9_7F4A_7C15 } + + mutating func next() -> UInt64 { + state ^= state >> 12 + state ^= state << 25 + state ^= state >> 27 + return state &* 0x2545_F491_4F6C_DD1D + } +} diff --git a/Sources/ImageSortCheck/main.swift b/Sources/ImageSortCheck/main.swift new file mode 100644 index 0000000..72dcc8a --- /dev/null +++ b/Sources/ImageSortCheck/main.swift @@ -0,0 +1,75 @@ +import FluidUse +import Foundation +import ImageSort + +// Headless checks for the SigLIP 2 image sorter. +// ImageSortCheck tokenizer token ids must equal the Python tokenizer's +// ImageSortCheck [--count=N] [--inflight=N] [--predictions=out.json] zero-shot Pets accuracy and speed +setvbuf(stdout, nil, _IOLBF, 0) +let arguments = CommandLine.arguments.dropFirst() + +func option(_ name: String) -> String? { + arguments.first { $0.hasPrefix("--\(name)=") }.map { String($0.dropFirst(name.count + 3)) } +} + +if arguments.first == "tokenizer", let path = arguments.dropFirst().first { + struct Case: Decodable { + let text: String + let ids: [Int32] + } + guard let directory = ProcessInfo.processInfo.environment["SIGLIP2_MODEL_DIR"] else { + fatalError("Set SIGLIP2_MODEL_DIR") + } + let tokenizer = try SigLIP2Tokenizer( + tokenizerJsonURL: URL(fileURLWithPath: directory).appendingPathComponent("tokenizer.json")) + let cases = try JSONDecoder().decode([Case].self, from: Data(contentsOf: URL(fileURLWithPath: path))) + var mismatches = 0 + for item in cases { + let ids = try tokenizer.encode(item.text) + if ids != item.ids { + mismatches += 1 + if mismatches <= 5 { + print("mismatch: \(item.text.debugDescription)\n swift \(ids)\n python \(item.ids)") + } + } + } + print("tokenizer: \(cases.count - mismatches)/\(cases.count) identical") + exit(mismatches == 0 ? 0 : 1) +} + +let count = option("count").flatMap(Int.init) +let inFlight = option("inflight").flatMap(Int.init) ?? 4 +let items = try await PetsSample.load(count: count ?? PetsSample.testCount) { done, total in + if done % 250 == 0 || done == total { print("cached \(done)/\(total) photos") } +} +let sorter = try await ImageSorter.load() +_ = try await sorter.sort(items[0]) + +let start = Date() +let results = try await withThrowingTaskGroup(of: (PetItem, ImageSorter.Result).self) { group in + var next = 0 + var results: [(PetItem, ImageSorter.Result)] = [] + func launch() { + guard next < items.count else { return } + let item = items[next] + next += 1 + group.addTask { (item, try await sorter.sort(item)) } + } + for _ in 0.. [String: CGRect]) { + value.merge(nextValue()) { $1 } + } +} + +extension View { + fileprivate func reportFrame(_ key: String) -> some View { + background( + GeometryReader { proxy in + Color.clear.preference(key: FramesKey.self, value: [key: proxy.frame(in: .named("board"))]) + }) + } +} + +private let incomingKey = "__incoming" + +struct ContentView: View { + @EnvironmentObject private var model: ImageSortModel + @State private var frames: [String: CGRect] = [:] + + var body: some View { + VStack(spacing: 0) { + header + Divider() + switch model.phase { + case .loading(let message): + status(message, spinning: true) + case .failed(let message): + status("Failed: \(message)", spinning: false) + default: + board + } + Divider() + footer + } + .background(Color(nsColor: .windowBackgroundColor)) + } + + private var header: some View { + HStack(alignment: .center, spacing: 18) { + VStack(alignment: .leading, spacing: 2) { + Text("Sort photos").font(.system(size: 26, weight: .bold)) + Text("SigLIP 2 · Core ML on the Neural Engine · 37 breeds, zero-shot, nothing trained on these photos") + .font(.callout).foregroundStyle(.secondary).lineLimit(1) + } + Spacer(minLength: 12) + HStack(spacing: 16) { + stat("Sorted", "\(model.sorted) / \(model.total)") + stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–") + stat("Elapsed", String(format: "%.1f s", model.elapsed)) + stat("ms / photo", millisecondsPerPhoto) + stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–") + } + controls + } + .padding(.horizontal, 20) + .padding(.vertical, 12) + } + + /// Show: one model call, pre- and post-processing included. Turbo: wall time per photo with calls overlapping. + private var millisecondsPerPhoto: String { + guard model.sorted > 0 else { return "–" } + if model.mode == .turbo { return String(format: "%.1f", 1000 / model.photosPerSecond) } + return model.medianMilliseconds.map { String(format: "%.1f", $0) } ?? "–" + } + + private func stat(_ title: String, _ value: String) -> some View { + VStack(alignment: .trailing, spacing: 2) { + Text(title.uppercased()).font(.caption2.weight(.semibold)).foregroundStyle(.secondary) + Text(value).font(.system(size: 20, weight: .semibold, design: .rounded)).monospacedDigit() + .contentTransition(.numericText()).fixedSize() + } + } + + private var controls: some View { + VStack(alignment: .leading, spacing: 6) { + HStack { + Button(model.phase == .running ? "Pause" : "Start") { model.toggleRun() } + .keyboardShortcut(.space, modifiers: []) + .disabled(!(model.phase == .ready || model.phase == .running || model.phase == .paused)) + .buttonStyle(.borderedProminent) + Button("Reset") { model.reset() }.disabled(model.phase == .running) + } + Picker("Mode", selection: $model.mode) { + ForEach(ImageSortModel.Mode.allCases) { Text($0.rawValue).tag($0) } + } + .pickerStyle(.segmented).labelsHidden().frame(width: 150) + if model.mode == .show { + HStack(spacing: 6) { + Text("Pace").font(.caption).fixedSize() + Slider(value: $model.pace, in: 2...30).frame(width: 90) + Text(String(format: "%.0f/s", model.pace)).font(.caption).monospacedDigit().fixedSize() + } + } else { + Text("\(ImageSortModel.turboInFlight) calls in flight").font(.caption).foregroundStyle(.secondary) + } + } + } + + private var board: some View { + GeometryReader { outer in + let incomingWidth = min(260, max(190, outer.size.width * 0.17)) + HStack(alignment: .top, spacing: 14) { + incoming.frame(width: incomingWidth) + GeometryReader { proxy in + let columns = max(4, min(8, Int(proxy.size.width / 150))) + let rows = (model.breeds.count + columns - 1) / columns + let height = max(78, (proxy.size.height - CGFloat(rows - 1) * 8) / CGFloat(rows)) + ScrollView { + LazyVGrid( + columns: Array(repeating: GridItem(.flexible(), spacing: 8), count: columns), spacing: 8 + ) { + ForEach(model.breeds, id: \.self) { breed in + BucketView( + name: breed, placed: model.buckets[breed] ?? [], count: model.counts[breed] ?? 0, + color: color(for: breed), height: height + ) + .reportFrame(breed) + } + } + } + } + } + } + .padding(14) + .coordinateSpace(name: "board") + .onPreferenceChange(FramesKey.self) { frames = $0 } + .overlay(alignment: .topLeading) { flying } + } + + private var incoming: some View { + VStack(alignment: .leading, spacing: 8) { + Text("INCOMING · \(model.queue.count) left").font(.caption.weight(.semibold)).foregroundStyle(.secondary) + ZStack { + RoundedRectangle(cornerRadius: 12).fill(Color.secondary.opacity(0.06)) + if model.incomingImage != nil { PhotoView(image: model.incomingImage).padding(8) } + } + .aspectRatio(1, contentMode: .fit) + .reportFrame(incomingKey) + Text("Labels are just the 37 breed names, typed once:\n\"a photo of a {breed}, a type of pet.\"") + .font(.caption).foregroundStyle(.secondary) + Spacer(minLength: 0) + } + } + + private var flying: some View { + ZStack(alignment: .topLeading) { + ForEach(model.flights) { flight in + if let from = frames[incomingKey], let to = frames[flight.placed.result.breed] { + let target = flight.arrived ? to : from + VStack(spacing: 4) { + PhotoView(image: flight.placed.thumbnail) + Text("→ \(flight.placed.result.breed)").font(.headline) + .foregroundStyle(flight.placed.matchesGold ? Color.green : Color.red) + } + .padding(6) + .background(RoundedRectangle(cornerRadius: 10).fill(Color(nsColor: .controlBackgroundColor))) + .frame(width: model.mode == .turbo ? 130 : from.width - 16) + .scaleEffect(flight.arrived ? 0.3 : 1) + .opacity(flight.arrived ? 0.2 : 1) + .position(x: target.midX, y: target.midY) + } + } + } + .allowsHitTesting(false) + } + + private func status(_ message: String, spinning: Bool) -> some View { + VStack(spacing: 12) { + if spinning { ProgressView() } + Text(message).foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + + private var footer: some View { + HStack { + Text(PetsSample.attribution) + Spacer() + Text("Model: google/siglip2-base-patch16-256 (Apache-2.0), converted to Core ML by FluidInference") + } + .font(.caption).foregroundStyle(.secondary) + .padding(.horizontal, 20).padding(.vertical, 8) + } + + private func color(for breed: String) -> Color { ImageSortModel.cats.contains(breed) ? .orange : .blue } +} + +private struct PhotoView: View { + let image: CGImage? + + var body: some View { + Color.secondary.opacity(0.1) + .aspectRatio(1, contentMode: .fit) + .overlay { + if let image { Image(decorative: image, scale: 1).resizable().aspectRatio(contentMode: .fill) } + } + .clipShape(RoundedRectangle(cornerRadius: 8)) + } +} + +private struct BucketView: View { + let name: String + let placed: [ImageSortModel.Placed] + let count: Int + let color: Color + let height: CGFloat + + var body: some View { + VStack(alignment: .leading, spacing: 4) { + HStack(alignment: .firstTextBaseline) { + Text(name).font(.callout.weight(.semibold)).lineLimit(1).minimumScaleFactor(0.7) + Spacer(minLength: 2) + Text("\(count)").font(.headline).monospacedDigit().contentTransition(.numericText()) + } + GeometryReader { proxy in + let side = max(20, min(proxy.size.height, (proxy.size.width - 8) / 3)) + HStack(spacing: 4) { + ForEach(placed.prefix(3)) { entry in + Image(decorative: entry.thumbnail ?? blank, scale: 1).resizable() + .aspectRatio(contentMode: .fill) + .frame(width: side, height: side).clipped() + .clipShape(RoundedRectangle(cornerRadius: 5)) + .overlay( + RoundedRectangle(cornerRadius: 5) + .stroke(entry.matchesGold ? Color.green : Color.red, lineWidth: 2) + ) + .help("Gold breed: \(entry.item.breed)") + } + } + } + } + .padding(8) + .frame(height: height) + .background(RoundedRectangle(cornerRadius: 10).fill(color.opacity(0.10))) + .overlay(RoundedRectangle(cornerRadius: 10).stroke(color.opacity(0.35), lineWidth: 1)) + } + + private var blank: CGImage { + CGContext( + data: nil, width: 1, height: 1, bitsPerComponent: 8, bytesPerRow: 4, + space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)!.makeImage()! + } +} diff --git a/Sources/ImageSortDemo/ImageSortDemoApp.swift b/Sources/ImageSortDemo/ImageSortDemoApp.swift new file mode 100644 index 0000000..a4f68cf --- /dev/null +++ b/Sources/ImageSortDemo/ImageSortDemoApp.swift @@ -0,0 +1,28 @@ +import AppKit +import SwiftUI + +@main +struct ImageSortDemoApp: App { + @StateObject private var model = ImageSortModel() + + init() { + setvbuf(stdout, nil, _IOLBF, 0) + // Bare SwiftPM executables start as background processes; make this one a regular windowed app. + for key in UserDefaults.standard.dictionaryRepresentation().keys where key.hasPrefix("NSWindow Frame") { + UserDefaults.standard.removeObject(forKey: key) + } + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate(ignoringOtherApps: true) + } + + var body: some Scene { + WindowGroup("Sort photos — SigLIP 2 on-device") { + ContentView() + .environmentObject(model) + .frame(minWidth: 900, minHeight: 620) + .task { await model.prepare() } + } + .defaultSize(width: 1560, height: 980) + .windowResizability(.contentMinSize) + } +} diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift new file mode 100644 index 0000000..c9aa67c --- /dev/null +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -0,0 +1,295 @@ +import CoreGraphics +import Foundation +import ImageIO +import ImageSort +import SwiftUI + +/// Drives the stream: loads Pets photos, sorts them with SigLIP 2, and keeps the live statistics. +@MainActor +final class ImageSortModel: ObservableObject { + enum Phase: Equatable { + case loading(String) + case ready + case running + case paused + case finished + case failed(String) + } + + enum Mode: String, CaseIterable, Identifiable { + /// One photo at a time, animated into its bucket. + case show = "Show" + /// Several calls in flight, as fast as the model goes. + case turbo = "Turbo" + var id: String { rawValue } + } + + struct Placed: Identifiable { + let item: PetItem + let result: ImageSorter.Result + let thumbnail: CGImage? + var id: Int { item.id } + var matchesGold: Bool { result.breed == item.breed } + } + + struct Flight: Identifiable { + let placed: Placed + var arrived = false + var id: Int { placed.id } + } + + @Published private(set) var phase: Phase = .loading("Starting…") + @Published var mode: Mode = .show + /// Photos per second in Show mode. + @Published var pace: Double = 10 + @Published private(set) var buckets: [String: [Placed]] = [:] + @Published private(set) var queue: [PetItem] = [] { + didSet { refreshIncoming() } + } + /// Thumbnail of the next photo, decoded once rather than on every redraw. + @Published private(set) var incomingImage: CGImage? + private var incomingID: Int? + @Published private(set) var flights: [Flight] = [] + @Published private(set) var sorted = 0 + @Published private(set) var correct = 0 + @Published private(set) var elapsed: Double = 0 + + let breeds = PetsSample.breeds + let total: Int + static let turboInFlight = 4 + static let turboFlush = 0.05 + static let turboFlightsPerFlush = 2 + static let travel = 0.45 + static let cats: Set = [ + "abyssinian", "bengal", "birman", "bombay", "british shorthair", "egyptian mau", "maine coon", "persian", + "ragdoll", "russian blue", "siamese", "sphynx", + ] + + private let environment = ProcessInfo.processInfo.environment + private var landed: Set = [] + private var items: [PetItem] = [] + private var sorter: ImageSorter? + private var runner: Task? + private var modelMilliseconds: [Double] = [] + private var runStart: Date? + private var elapsedBeforePause: Double = 0 + private var shownInShow = 0 + private var preparing = false + + init() { + total = ProcessInfo.processInfo.environment["IMAGE_SORT_COUNT"].flatMap(Int.init) ?? 1000 + } + + var photosPerSecond: Double { elapsed > 0 ? Double(sorted) / elapsed : 0 } + var accuracy: Double? { sorted > 0 ? Double(correct) / Double(sorted) : nil } + var medianMilliseconds: Double? { + guard !modelMilliseconds.isEmpty else { return nil } + return modelMilliseconds.sorted()[modelMilliseconds.count / 2] + } + var modelName: String { sorter?.modelName ?? "SigLIP 2" } + + func prepare() async { + guard !preparing else { return } + preparing = true + do { + phase = .loading("Fetching \(total) Oxford-IIIT Pets photos…") + items = try await PetsSample.load(count: total) { [weak self] done, all in + Task { @MainActor in self?.phase = .loading("Cached \(done) / \(all) photos") } + } + phase = .loading("Loading SigLIP 2 and embedding 37 breed names…") + sorter = try await ImageSorter.load() + _ = try await sorter?.sort(items[0]) + reset() + print("ready at \(Date().timeIntervalSince1970)") + // IMAGE_SORT_AUTOSTART=show|turbo starts without a click; IMAGE_SORT_AUTOPLAY=N flies N photos in Show + // mode and then switches to Turbo (for recordings). + if environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init) != nil { + try? await Task.sleep(for: .seconds(1.5)) + mode = .show + toggleRun() + } else if let start = environment["IMAGE_SORT_AUTOSTART"], let mode = Mode(rawValue: start.capitalized) { + self.mode = mode + toggleRun() + } + } catch { + phase = .failed(error.localizedDescription) + } + } + + func reset() { + runner?.cancel() + runner = nil + buckets = Dictionary(uniqueKeysWithValues: breeds.map { ($0, []) }) + queue = items + flights = [] + landed = [] + counts = [:] + sorted = 0 + correct = 0 + elapsed = 0 + elapsedBeforePause = 0 + modelMilliseconds = [] + shownInShow = 0 + phase = .ready + } + + func toggleRun() { + switch phase { + case .running: + runner?.cancel() + runner = nil + elapsedBeforePause = elapsed + phase = .paused + case .ready, .paused: + phase = .running + runStart = Date() + runner = Task { [weak self] in await self?.run() } + default: + break + } + } + + private func run() async { + guard let sorter else { return } + while !Task.isCancelled, !queue.isEmpty { + if mode == .turbo { + await runTurbo(sorter) + } else { + await runShow(sorter) + } + } + if !Task.isCancelled, queue.isEmpty { + tick() + phase = .finished + print( + "finished \(sorted) photos in \(String(format: "%.2f", elapsed)) s " + + "(\(String(format: "%.0f", photosPerSecond)) photos/s), " + + "correct \(String(format: "%.1f", (accuracy ?? 0) * 100))% (\(mode.rawValue))") + } + } + + private func runShow(_ sorter: ImageSorter) async { + let stream = Self.stream(sorter, items: queue, inFlight: 2) + for await placed in stream { + if Task.isCancelled || mode != .show { break } + launch(placed) + let started = Date() + try? await Task.sleep(for: .seconds(1 / pace)) + if Date().timeIntervalSince(started) < 1 / pace { break } + } + while !Task.isCancelled, !flights.isEmpty { try? await Task.sleep(for: .milliseconds(20)) } + } + + private func launch(_ placed: Placed, decorative: Bool = false) { + guard !flights.contains(where: { $0.id == placed.id }) else { return } + flights.append(Flight(placed: placed)) + Task { @MainActor [weak self] in + try? await Task.sleep(for: .milliseconds(20)) + withAnimation(.easeInOut(duration: Self.travel)) { + if let index = self?.flights.firstIndex(where: { $0.id == placed.id }) { + self?.flights[index].arrived = true + } + } + try? await Task.sleep(for: .seconds(Self.travel)) + guard let self else { return } + flights.removeAll { $0.id == placed.id } + guard !decorative else { return } + land([placed]) + shownInShow += 1 + if let count = environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init), shownInShow >= count { mode = .turbo } + } + } + + /// Sorts `items` with `inFlight` calls always running off the main thread and streams each result. + private nonisolated static func stream( + _ sorter: ImageSorter, items: [PetItem], inFlight: Int + ) + -> AsyncStream + { + AsyncStream { continuation in + let producer = Task.detached { + await withTaskGroup(of: Placed?.self) { group in + var next = 0 + func launch() { + guard next < items.count, !Task.isCancelled else { return } + let item = items[next] + next += 1 + group.addTask { + guard let result = try? await sorter.sort(item) else { return nil } + return Placed(item: item, result: result, thumbnail: thumbnail(item.file)) + } + } + for _ in 0.. CGImage? { + guard let source = CGImageSourceCreateWithURL(file as CFURL, nil) else { return nil } + return CGImageSourceCreateThumbnailAtIndex( + source, 0, + [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceThumbnailMaxPixelSize: size, + kCGImageSourceCreateThumbnailWithTransform: true, + ] as CFDictionary) + } + + private func runTurbo(_ sorter: ImageSorter) async { + let stream = Self.stream(sorter, items: queue, inFlight: Self.turboInFlight) + var pending: [Placed] = [] + var lastFlush = Date() + for await placed in stream { + pending.append(placed) + if Date().timeIntervalSince(lastFlush) >= Self.turboFlush { + for placed in pending.suffix(Self.turboFlightsPerFlush) where flights.count < 30 { + launch(placed, decorative: true) + } + land(pending) + pending.removeAll(keepingCapacity: true) + lastFlush = Date() + } + if Task.isCancelled || mode != .turbo { break } + } + land(pending) + } + + private func land(_ incoming: [Placed]) { + let batch = incoming.filter { self.landed.insert($0.id).inserted } + guard !batch.isEmpty else { return } + var updated = buckets + for placed in batch { + updated[placed.result.breed, default: []].insert(placed, at: 0) + if updated[placed.result.breed]!.count > 6 { updated[placed.result.breed]!.removeLast() } + correct += placed.matchesGold ? 1 : 0 + modelMilliseconds.append(placed.result.milliseconds) + counts[placed.result.breed, default: 0] += 1 + } + buckets = updated + let ids = Set(batch.map(\.id)) + queue.removeAll { ids.contains($0.id) } + sorted += batch.count + tick() + } + + /// Photos per bucket (the bucket itself only keeps the latest few for display). + @Published private(set) var counts: [String: Int] = [:] + + private func refreshIncoming() { + let next = queue.first { item in !flights.contains { $0.id == item.id } } + guard next?.id != incomingID else { return } + incomingID = next?.id + incomingImage = next.flatMap { Self.thumbnail($0.file, size: 400) } + } + + private func tick() { + if let runStart { elapsed = elapsedBeforePause + Date().timeIntervalSince(runStart) } + } +} diff --git a/Sources/ImageSortDemo/README.md b/Sources/ImageSortDemo/README.md new file mode 100644 index 0000000..2d637a7 --- /dev/null +++ b/Sources/ImageSortDemo/README.md @@ -0,0 +1,19 @@ +# Sort photos + +Sorts Oxford-IIIT Pets test photos into 37 breeds with SigLIP 2 (base, 256 px) on Core ML. The only hint the model +gets is each breed's name, in the prompt `a photo of a {breed}, a type of pet.`; nothing is trained on these photos. + +```bash +SIGLIP2_MODEL_DIR=/path/to/siglip2-base-patch16-256 IMAGE_SORT_AUTOPLAY=12 swift run -c release ImageSortDemo +``` + +- `IMAGE_SORT_AUTOPLAY=N` flies N photos in Show mode, then switches to Turbo; `IMAGE_SORT_AUTOSTART=show|turbo` + starts a run in that mode. Without either, press Start. +- `IMAGE_SORT_COUNT` sets the sample size (default 1,000; the test split has 3,669). +- Orange buckets are cat breeds, blue are dog breeds; a green frame means the gold label agrees. + +Headless: `swift run -c release ImageSortCheck --inflight=4` (M5 Pro, macOS 27: 94.85% on all 3,669 test photos, +193 photos/s). + +Data: Oxford-IIIT Pets test split (Parkhi et al., 2012), CC BY-SA 4.0. Model: google/siglip2-base-patch16-256, +Apache-2.0. From e7ff298d1a979e7d71a4a1fe8faf9b78cf6856d6 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 22:16:00 -0400 Subject: [PATCH 02/14] ImageSortDemo: IMAGE_SORT_LOG=1 per-photo terminal log Prints each decision with the label's softmax share among the 37 breeds and the model call time, matching SortAnythingDemo's SORT_LOG. --- Sources/ImageSort/ImageSorter.swift | 7 ++++++- Sources/ImageSortDemo/ImageSortModel.swift | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Sources/ImageSort/ImageSorter.swift b/Sources/ImageSort/ImageSorter.swift index baec462..63fd30a 100644 --- a/Sources/ImageSort/ImageSorter.swift +++ b/Sources/ImageSort/ImageSorter.swift @@ -9,6 +9,8 @@ public final class ImageSorter: Sendable { public let breed: String /// SigLIP's own sigmoid probability for the chosen label. public let probability: Float + /// Softmax of the scaled similarities across all labels: the chosen label's share among the candidates. + public let share: Float public let milliseconds: Double } @@ -39,8 +41,11 @@ public final class ImageSorter: Sendable { let start = DispatchTime.now().uptimeNanoseconds let answer = try await manager.classify(image: image, labels: breeds, labelEmbeddings: embeddings) let milliseconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e6 + let scale = manager.config.logitScale + let top = answer.similarities[answer.selectedIndex] + let total = answer.similarities.reduce(Float(0)) { $0 + exp(scale * ($1 - top)) } return Result( - breed: answer.selectedLabel, probability: answer.probabilities[answer.selectedIndex], + breed: answer.selectedLabel, probability: answer.probabilities[answer.selectedIndex], share: 1 / total, milliseconds: milliseconds) } diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index c9aa67c..713a3e1 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -66,6 +66,8 @@ final class ImageSortModel: ObservableObject { ] private let environment = ProcessInfo.processInfo.environment + /// IMAGE_SORT_LOG=1 prints every decision to stdout (for a terminal next to the window). + private let logDecisions = ProcessInfo.processInfo.environment["IMAGE_SORT_LOG"] == "1" private var landed: Set = [] private var items: [PetItem] = [] private var sorter: ImageSorter? @@ -277,6 +279,23 @@ final class ImageSortModel: ObservableObject { queue.removeAll { ids.contains($0.id) } sorted += batch.count tick() + if logDecisions { log(batch) } + } + + private func log(_ batch: [Placed]) { + let (cyan, yellow, red, green, dim, reset) = + ("\u{1B}[1;36m", "\u{1B}[33m", "\u{1B}[31m", "\u{1B}[32m", "\u{1B}[2m", "\u{1B}[0m") + var lines = "" + for (offset, placed) in batch.enumerated() { + let number = sorted - batch.count + offset + 1 + let mark = placed.matchesGold ? "\(green)✓\(reset)" : "\(red)✗ label: \(placed.item.breed)\(reset)" + lines += "\(cyan)▶ #\(number) photo \(placed.item.id).jpg\(reset)\n" + lines += + " \(yellow)→ \(placed.result.breed)\(reset) · \(Int(placed.result.share * 100))% of 37 · " + + "\(red)model call \(String(format: "%.1f", placed.result.milliseconds)) ms\(reset) \(mark)\n" + } + lines += "\(dim) sorted \(sorted)/\(total) · \(String(format: "%.1f", elapsed)) s\(reset)\n" + print(lines, terminator: "") } /// Photos per bucket (the bucket itself only keeps the latest few for display). From 3ae15d736f6073fbceb2de94266cd228bd81a010 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 22:34:14 -0400 Subject: [PATCH 03/14] ImageSortDemo: photo bar chart, top-5 panel, narrow layout, train split Each sorted photo becomes a tile in its breed's row (one bitmap, so thousands of tiles stay cheap); wrong calls get a red frame. The left panel shows the latest photo with its five most likely breeds. Header and board adapt to narrow windows. A cache holding the train split samples up to 7,349 photos (Turbo: 38 s, 192 photos/s, 94.3% on an M5 Pro). IMAGE_SORT_WAIT=1 waits for Start. --- Sources/ImageSort/ImageSorter.swift | 12 +- Sources/ImageSort/PetsSample.swift | 11 +- Sources/ImageSortDemo/ContentView.swift | 242 ++++++++----------- Sources/ImageSortDemo/ImageSortDemoApp.swift | 2 +- Sources/ImageSortDemo/ImageSortModel.swift | 149 +++++------- Sources/ImageSortDemo/PhotoChart.swift | 68 ++++++ Sources/ImageSortDemo/README.md | 10 +- 7 files changed, 257 insertions(+), 237 deletions(-) create mode 100644 Sources/ImageSortDemo/PhotoChart.swift diff --git a/Sources/ImageSort/ImageSorter.swift b/Sources/ImageSort/ImageSorter.swift index 63fd30a..ee37890 100644 --- a/Sources/ImageSort/ImageSorter.swift +++ b/Sources/ImageSort/ImageSorter.swift @@ -11,6 +11,8 @@ public final class ImageSorter: Sendable { public let probability: Float /// Softmax of the scaled similarities across all labels: the chosen label's share among the candidates. public let share: Float + /// The five most likely breeds with their shares, best first. + public let top: [(breed: String, share: Float)] public let milliseconds: Double } @@ -42,11 +44,15 @@ public final class ImageSorter: Sendable { let answer = try await manager.classify(image: image, labels: breeds, labelEmbeddings: embeddings) let milliseconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e6 let scale = manager.config.logitScale - let top = answer.similarities[answer.selectedIndex] - let total = answer.similarities.reduce(Float(0)) { $0 + exp(scale * ($1 - top)) } + let best = answer.similarities[answer.selectedIndex] + let weights = answer.similarities.map { exp(scale * ($0 - best)) } + let total = weights.reduce(0, +) + let ranked = weights.indices.sorted { weights[$0] > weights[$1] }.prefix(5).map { + (breed: breeds[$0], share: weights[$0] / total) + } return Result( breed: answer.selectedLabel, probability: answer.probabilities[answer.selectedIndex], share: 1 / total, - milliseconds: milliseconds) + top: ranked, milliseconds: milliseconds) } public static func decode(_ file: URL) throws -> CGImage { diff --git a/Sources/ImageSort/PetsSample.swift b/Sources/ImageSort/PetsSample.swift index 24161bc..993e163 100644 --- a/Sources/ImageSort/PetsSample.swift +++ b/Sources/ImageSort/PetsSample.swift @@ -11,7 +11,7 @@ public struct PetItem: Codable, Sendable, Identifiable, Hashable { /// Seeded sample of the Oxford-IIIT Pets test split (CC BY-SA 4.0), fetched from the Hugging Face dataset viewer /// API on first use and cached locally. Nothing is bundled. public enum PetsSample { - public static let attribution = "Oxford-IIIT Pets test split (Parkhi et al., 2012) · CC BY-SA 4.0" + public static let attribution = "Oxford-IIIT Pets (Parkhi et al., 2012) · CC BY-SA 4.0" public static let testCount = 3669 /// The 37 breeds in dataset label order. @@ -53,7 +53,8 @@ public enum PetsSample { .appendingPathComponent("FluidUse/image-sort/oxford-pets-test") } - /// `count` photos in a seeded shuffled order (all 3,669 when `count` is nil). `progress` gets photos cached so far. + /// `count` photos in a seeded shuffled order (every cached photo when `count` is nil). The viewer API fetches the + /// 3,669 test photos; a cache that also holds the train split (ids from 3,669) samples from both. public static func load( count: Int? = 1000, seed: UInt64 = 0, progress: (@Sendable (Int, Int) -> Void)? = nil ) async throws -> [PetItem] { @@ -65,12 +66,12 @@ public enum PetsSample { var labels: [Int: Int] = [:] if let data = try? Data(contentsOf: manifestURL), let saved = try? JSONDecoder().decode([Int: Int].self, from: data), - saved.count == testCount + saved.count >= testCount { labels = saved } var sources: [Int: String] = [:] - if labels.count != testCount { + if labels.count < testCount { for offset in stride(from: 0, to: testCount, by: pageSize) { for entry in try await page(offset: offset) { labels[entry.rowIndex] = entry.row.label @@ -81,7 +82,7 @@ public enum PetsSample { } var generator = SeededGenerator(seed: seed) - let chosen = Array(labels.keys.sorted().shuffled(using: &generator).prefix(count ?? testCount)) + let chosen = Array(labels.keys.sorted().shuffled(using: &generator).prefix(count ?? labels.count)) let missing = chosen.filter { !manager.fileExists(atPath: file(for: $0).path) } if !missing.isEmpty { if sources.isEmpty { diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index 06373af..c8bf083 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -1,27 +1,8 @@ import ImageSort import SwiftUI -private struct FramesKey: PreferenceKey { - static let defaultValue: [String: CGRect] = [:] - static func reduce(value: inout [String: CGRect], nextValue: () -> [String: CGRect]) { - value.merge(nextValue()) { $1 } - } -} - -extension View { - fileprivate func reportFrame(_ key: String) -> some View { - background( - GeometryReader { proxy in - Color.clear.preference(key: FramesKey.self, value: [key: proxy.frame(in: .named("board"))]) - }) - } -} - -private let incomingKey = "__incoming" - struct ContentView: View { @EnvironmentObject private var model: ImageSortModel - @State private var frames: [String: CGRect] = [:] var body: some View { VStack(spacing: 0) { @@ -42,26 +23,44 @@ struct ContentView: View { } private var header: some View { - HStack(alignment: .center, spacing: 18) { - VStack(alignment: .leading, spacing: 2) { - Text("Sort photos").font(.system(size: 26, weight: .bold)) - Text("SigLIP 2 · Core ML on the Neural Engine · 37 breeds, zero-shot, nothing trained on these photos") - .font(.callout).foregroundStyle(.secondary).lineLimit(1) + ViewThatFits(in: .horizontal) { + HStack(alignment: .center, spacing: 18) { + titleBlock + Spacer(minLength: 12) + stats + controls } - Spacer(minLength: 12) - HStack(spacing: 16) { - stat("Sorted", "\(model.sorted) / \(model.total)") - stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–") - stat("Elapsed", String(format: "%.1f s", model.elapsed)) - stat("ms / photo", millisecondsPerPhoto) - stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–") + VStack(alignment: .leading, spacing: 10) { + titleBlock + HStack(alignment: .center, spacing: 14) { + stats + Spacer(minLength: 8) + controls + } } - controls } .padding(.horizontal, 20) .padding(.vertical, 12) } + private var titleBlock: some View { + VStack(alignment: .leading, spacing: 2) { + Text("Sort photos").font(.system(size: 26, weight: .bold)) + Text("SigLIP 2 · Core ML on the Neural Engine · 37 breeds, zero-shot") + .font(.callout).foregroundStyle(.secondary).lineLimit(1).fixedSize() + } + } + + private var stats: some View { + HStack(spacing: 16) { + stat("Sorted", "\(model.sorted) / \(model.total)") + stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–") + stat("Elapsed", String(format: "%.1f s", model.elapsed)) + stat("ms / photo", millisecondsPerPhoto) + stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–") + } + } + /// Show: one model call, pre- and post-processing included. Turbo: wall time per photo with calls overlapping. private var millisecondsPerPhoto: String { guard model.sorted > 0 else { return "–" } @@ -72,6 +71,7 @@ struct ContentView: View { private func stat(_ title: String, _ value: String) -> some View { VStack(alignment: .trailing, spacing: 2) { Text(title.uppercased()).font(.caption2.weight(.semibold)).foregroundStyle(.secondary) + .lineLimit(1).fixedSize() Text(value).font(.system(size: 20, weight: .semibold, design: .rounded)).monospacedDigit() .contentTransition(.numericText()).fixedSize() } @@ -103,71 +103,90 @@ struct ContentView: View { } private var board: some View { - GeometryReader { outer in - let incomingWidth = min(260, max(190, outer.size.width * 0.17)) - HStack(alignment: .top, spacing: 14) { - incoming.frame(width: incomingWidth) - GeometryReader { proxy in - let columns = max(4, min(8, Int(proxy.size.width / 150))) - let rows = (model.breeds.count + columns - 1) / columns - let height = max(78, (proxy.size.height - CGFloat(rows - 1) * 8) / CGFloat(rows)) - ScrollView { - LazyVGrid( - columns: Array(repeating: GridItem(.flexible(), spacing: 8), count: columns), spacing: 8 - ) { - ForEach(model.breeds, id: \.self) { breed in - BucketView( - name: breed, placed: model.buckets[breed] ?? [], count: model.counts[breed] ?? 0, - color: color(for: breed), height: height - ) - .reportFrame(breed) - } - } + HStack(alignment: .top, spacing: 18) { + nowSorting.frame(width: 250) + chart + } + .padding(14) + } + + private var nowSorting: some View { + VStack(alignment: .leading, spacing: 10) { + Text("NOW SORTING · \(model.remaining) left").font(.caption.weight(.semibold)).foregroundStyle(.secondary) + Color.secondary.opacity(0.08) + .aspectRatio(1, contentMode: .fit) + .overlay { + if let image = model.currentImage { + Image(decorative: image, scale: 1).resizable().aspectRatio(contentMode: .fill) + } + } + .clipShape(RoundedRectangle(cornerRadius: 12)) + .overlay( + RoundedRectangle(cornerRadius: 12) + .stroke(model.current.map { $0.matchesGold ? Color.green : Color.red } ?? .clear, lineWidth: 3)) + if let current = model.current { + VStack(alignment: .leading, spacing: 5) { + ForEach(Array(current.result.top.enumerated()), id: \.offset) { rank, entry in + topRow(entry.breed, share: entry.share, first: rank == 0, gold: current.item.breed) + } + if !current.matchesGold { + Text("label: \(current.item.breed)").font(.caption.weight(.semibold)).foregroundStyle(.red) } } } + Spacer(minLength: 0) + Text("Labels are the 37 breed names, typed once: \"a photo of a {breed}, a type of pet.\"") + .font(.caption).foregroundStyle(.secondary) } - .padding(14) - .coordinateSpace(name: "board") - .onPreferenceChange(FramesKey.self) { frames = $0 } - .overlay(alignment: .topLeading) { flying } } - private var incoming: some View { - VStack(alignment: .leading, spacing: 8) { - Text("INCOMING · \(model.queue.count) left").font(.caption.weight(.semibold)).foregroundStyle(.secondary) - ZStack { - RoundedRectangle(cornerRadius: 12).fill(Color.secondary.opacity(0.06)) - if model.incomingImage != nil { PhotoView(image: model.incomingImage).padding(8) } + private func topRow(_ breed: String, share: Float, first: Bool, gold: String) -> some View { + HStack(spacing: 6) { + Text(breed).font(first ? .callout.weight(.bold) : .caption).lineLimit(1) + .frame(width: 128, alignment: .leading) + GeometryReader { proxy in + Capsule().fill(breed == gold ? Color.green : color(for: breed)) + .frame(width: max(2, proxy.size.width * CGFloat(share))) } - .aspectRatio(1, contentMode: .fit) - .reportFrame(incomingKey) - Text("Labels are just the 37 breed names, typed once:\n\"a photo of a {breed}, a type of pet.\"") - .font(.caption).foregroundStyle(.secondary) - Spacer(minLength: 0) + .frame(height: first ? 10 : 6) + Text("\(Int(share * 100))%").font(.caption).monospacedDigit().frame(width: 34, alignment: .trailing) } } - private var flying: some View { - ZStack(alignment: .topLeading) { - ForEach(model.flights) { flight in - if let from = frames[incomingKey], let to = frames[flight.placed.result.breed] { - let target = flight.arrived ? to : from - VStack(spacing: 4) { - PhotoView(image: flight.placed.thumbnail) - Text("→ \(flight.placed.result.breed)").font(.headline) - .foregroundStyle(flight.placed.matchesGold ? Color.green : Color.red) + private var chart: some View { + GeometryReader { proxy in + let labelWidth: CGFloat = 190 + let width = CGFloat(PhotoChart.columns * PhotoChart.tile) + let height = CGFloat(model.breeds.count * PhotoChart.rowHeight) + let scale = min((proxy.size.width - labelWidth) / width, proxy.size.height / height) + let rowHeight = CGFloat(PhotoChart.rowHeight) * scale + HStack(alignment: .top, spacing: 0) { + VStack(alignment: .trailing, spacing: 0) { + ForEach(model.breeds, id: \.self) { breed in + HStack(spacing: 6) { + Text(breed).lineLimit(1).foregroundStyle(color(for: breed)) + Text("\(model.counts[breed] ?? 0)").monospacedDigit().foregroundStyle(.secondary) + .frame(width: 34, alignment: .trailing) + } + .font(.system(size: max(9, min(13, rowHeight * 0.55)), weight: .semibold)) + .frame(width: labelWidth - 8, height: rowHeight, alignment: .trailing) + .padding(.trailing, 8) + } + } + ZStack(alignment: .topLeading) { + if let image = model.chartImage { + Image(decorative: image, scale: 1).resizable().interpolation(.medium) + .frame(width: width * scale, height: height * scale) + } + if let slot = model.lastSlot { + RoundedRectangle(cornerRadius: 3).stroke(Color.yellow, lineWidth: 2) + .frame(width: slot.width * scale + 6, height: slot.height * scale + 6) + .offset(x: slot.minX * scale - 3, y: slot.minY * scale - 3) } - .padding(6) - .background(RoundedRectangle(cornerRadius: 10).fill(Color(nsColor: .controlBackgroundColor))) - .frame(width: model.mode == .turbo ? 130 : from.width - 16) - .scaleEffect(flight.arrived ? 0.3 : 1) - .opacity(flight.arrived ? 0.2 : 1) - .position(x: target.midX, y: target.midY) } + .frame(width: width * scale, height: height * scale, alignment: .topLeading) } } - .allowsHitTesting(false) } private func status(_ message: String, spinning: Bool) -> some View { @@ -190,60 +209,3 @@ struct ContentView: View { private func color(for breed: String) -> Color { ImageSortModel.cats.contains(breed) ? .orange : .blue } } - -private struct PhotoView: View { - let image: CGImage? - - var body: some View { - Color.secondary.opacity(0.1) - .aspectRatio(1, contentMode: .fit) - .overlay { - if let image { Image(decorative: image, scale: 1).resizable().aspectRatio(contentMode: .fill) } - } - .clipShape(RoundedRectangle(cornerRadius: 8)) - } -} - -private struct BucketView: View { - let name: String - let placed: [ImageSortModel.Placed] - let count: Int - let color: Color - let height: CGFloat - - var body: some View { - VStack(alignment: .leading, spacing: 4) { - HStack(alignment: .firstTextBaseline) { - Text(name).font(.callout.weight(.semibold)).lineLimit(1).minimumScaleFactor(0.7) - Spacer(minLength: 2) - Text("\(count)").font(.headline).monospacedDigit().contentTransition(.numericText()) - } - GeometryReader { proxy in - let side = max(20, min(proxy.size.height, (proxy.size.width - 8) / 3)) - HStack(spacing: 4) { - ForEach(placed.prefix(3)) { entry in - Image(decorative: entry.thumbnail ?? blank, scale: 1).resizable() - .aspectRatio(contentMode: .fill) - .frame(width: side, height: side).clipped() - .clipShape(RoundedRectangle(cornerRadius: 5)) - .overlay( - RoundedRectangle(cornerRadius: 5) - .stroke(entry.matchesGold ? Color.green : Color.red, lineWidth: 2) - ) - .help("Gold breed: \(entry.item.breed)") - } - } - } - } - .padding(8) - .frame(height: height) - .background(RoundedRectangle(cornerRadius: 10).fill(color.opacity(0.10))) - .overlay(RoundedRectangle(cornerRadius: 10).stroke(color.opacity(0.35), lineWidth: 1)) - } - - private var blank: CGImage { - CGContext( - data: nil, width: 1, height: 1, bitsPerComponent: 8, bytesPerRow: 4, - space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)!.makeImage()! - } -} diff --git a/Sources/ImageSortDemo/ImageSortDemoApp.swift b/Sources/ImageSortDemo/ImageSortDemoApp.swift index a4f68cf..a8bb5e2 100644 --- a/Sources/ImageSortDemo/ImageSortDemoApp.swift +++ b/Sources/ImageSortDemo/ImageSortDemoApp.swift @@ -19,7 +19,7 @@ struct ImageSortDemoApp: App { WindowGroup("Sort photos — SigLIP 2 on-device") { ContentView() .environmentObject(model) - .frame(minWidth: 900, minHeight: 620) + .frame(minWidth: 640, minHeight: 560) .task { await model.prepare() } } .defaultSize(width: 1560, height: 980) diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index 713a3e1..afa9e79 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -4,7 +4,7 @@ import ImageIO import ImageSort import SwiftUI -/// Drives the stream: loads Pets photos, sorts them with SigLIP 2, and keeps the live statistics. +/// Drives the stream: loads Pets photos, sorts them with SigLIP 2, and grows the photo chart. @MainActor final class ImageSortModel: ObservableObject { enum Phase: Equatable { @@ -17,49 +17,41 @@ final class ImageSortModel: ObservableObject { } enum Mode: String, CaseIterable, Identifiable { - /// One photo at a time, animated into its bucket. + /// One photo at a time, with its top-5 breeds. case show = "Show" /// Several calls in flight, as fast as the model goes. case turbo = "Turbo" var id: String { rawValue } } - struct Placed: Identifiable { + struct Placed: Identifiable, Sendable { let item: PetItem let result: ImageSorter.Result - let thumbnail: CGImage? + let tile: CGImage? var id: Int { item.id } var matchesGold: Bool { result.breed == item.breed } } - struct Flight: Identifiable { - let placed: Placed - var arrived = false - var id: Int { placed.id } - } - @Published private(set) var phase: Phase = .loading("Starting…") @Published var mode: Mode = .show /// Photos per second in Show mode. - @Published var pace: Double = 10 - @Published private(set) var buckets: [String: [Placed]] = [:] - @Published private(set) var queue: [PetItem] = [] { - didSet { refreshIncoming() } - } - /// Thumbnail of the next photo, decoded once rather than on every redraw. - @Published private(set) var incomingImage: CGImage? - private var incomingID: Int? - @Published private(set) var flights: [Flight] = [] + @Published var pace: Double = 8 + @Published private(set) var remaining = 0 @Published private(set) var sorted = 0 @Published private(set) var correct = 0 @Published private(set) var elapsed: Double = 0 + @Published private(set) var counts: [String: Int] = [:] + @Published private(set) var chartImage: CGImage? + /// The photo just sorted, shown large with its top-5 breeds. + @Published private(set) var current: Placed? + @Published private(set) var currentImage: CGImage? + /// Where the latest tile landed, for the highlight in Show mode. + @Published private(set) var lastSlot: CGRect? let breeds = PetsSample.breeds let total: Int static let turboInFlight = 4 static let turboFlush = 0.05 - static let turboFlightsPerFlush = 2 - static let travel = 0.45 static let cats: Set = [ "abyssinian", "bengal", "birman", "bombay", "british shorthair", "egyptian mau", "maine coon", "persian", "ragdoll", "russian blue", "siamese", "sphynx", @@ -68,6 +60,8 @@ final class ImageSortModel: ObservableObject { private let environment = ProcessInfo.processInfo.environment /// IMAGE_SORT_LOG=1 prints every decision to stdout (for a terminal next to the window). private let logDecisions = ProcessInfo.processInfo.environment["IMAGE_SORT_LOG"] == "1" + private var chart: PhotoChart? + private var queue: [PetItem] = [] private var landed: Set = [] private var items: [PetItem] = [] private var sorter: ImageSorter? @@ -88,7 +82,6 @@ final class ImageSortModel: ObservableObject { guard !modelMilliseconds.isEmpty else { return nil } return modelMilliseconds.sorted()[modelMilliseconds.count / 2] } - var modelName: String { sorter?.modelName ?? "SigLIP 2" } func prepare() async { guard !preparing else { return } @@ -103,9 +96,11 @@ final class ImageSortModel: ObservableObject { _ = try await sorter?.sort(items[0]) reset() print("ready at \(Date().timeIntervalSince1970)") - // IMAGE_SORT_AUTOSTART=show|turbo starts without a click; IMAGE_SORT_AUTOPLAY=N flies N photos in Show - // mode and then switches to Turbo (for recordings). - if environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init) != nil { + // IMAGE_SORT_WAIT=1 keeps the chart empty until Start (Space). IMAGE_SORT_AUTOPLAY=N sorts N photos in + // Show mode and then switches to Turbo; IMAGE_SORT_AUTOSTART=show|turbo starts in that mode. + if environment["IMAGE_SORT_WAIT"] == "1" { + return + } else if environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init) != nil { try? await Task.sleep(for: .seconds(1.5)) mode = .show toggleRun() @@ -121,11 +116,19 @@ final class ImageSortModel: ObservableObject { func reset() { runner?.cancel() runner = nil - buckets = Dictionary(uniqueKeysWithValues: breeds.map { ($0, []) }) + chart = PhotoChart(rows: breeds.count) { [breeds] row in + Self.cats.contains(breeds[row]) + ? CGColor(red: 1, green: 0.6, blue: 0.2, alpha: 0.10) + : CGColor(red: 0.3, green: 0.55, blue: 1, alpha: 0.10) + } + chartImage = chart?.snapshot() queue = items - flights = [] + remaining = items.count landed = [] counts = [:] + current = nil + currentImage = nil + lastSlot = nil sorted = 0 correct = 0 elapsed = 0 @@ -171,35 +174,32 @@ final class ImageSortModel: ObservableObject { } private func runShow(_ sorter: ImageSorter) async { - let stream = Self.stream(sorter, items: queue, inFlight: 2) - for await placed in stream { + for await placed in Self.stream(sorter, items: queue, inFlight: 2) { if Task.isCancelled || mode != .show { break } - launch(placed) + land([placed], highlight: true) + shownInShow += 1 + if let count = environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init), shownInShow >= count { + mode = .turbo + } let started = Date() try? await Task.sleep(for: .seconds(1 / pace)) if Date().timeIntervalSince(started) < 1 / pace { break } } - while !Task.isCancelled, !flights.isEmpty { try? await Task.sleep(for: .milliseconds(20)) } } - private func launch(_ placed: Placed, decorative: Bool = false) { - guard !flights.contains(where: { $0.id == placed.id }) else { return } - flights.append(Flight(placed: placed)) - Task { @MainActor [weak self] in - try? await Task.sleep(for: .milliseconds(20)) - withAnimation(.easeInOut(duration: Self.travel)) { - if let index = self?.flights.firstIndex(where: { $0.id == placed.id }) { - self?.flights[index].arrived = true - } + private func runTurbo(_ sorter: ImageSorter) async { + var pending: [Placed] = [] + var lastFlush = Date() + for await placed in Self.stream(sorter, items: queue, inFlight: Self.turboInFlight) { + pending.append(placed) + if Date().timeIntervalSince(lastFlush) >= Self.turboFlush { + land(pending, highlight: false) + pending.removeAll(keepingCapacity: true) + lastFlush = Date() } - try? await Task.sleep(for: .seconds(Self.travel)) - guard let self else { return } - flights.removeAll { $0.id == placed.id } - guard !decorative else { return } - land([placed]) - shownInShow += 1 - if let count = environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init), shownInShow >= count { mode = .turbo } + if Task.isCancelled || mode != .turbo { break } } + land(pending, highlight: false) } /// Sorts `items` with `inFlight` calls always running off the main thread and streams each result. @@ -218,7 +218,7 @@ final class ImageSortModel: ObservableObject { next += 1 group.addTask { guard let result = try? await sorter.sort(item) else { return nil } - return Placed(item: item, result: result, thumbnail: thumbnail(item.file)) + return Placed(item: item, result: result, tile: thumbnail(item.file, size: 48)) } } for _ in 0.. CGImage? { + nonisolated static func thumbnail(_ file: URL, size: Int) -> CGImage? { guard let source = CGImageSourceCreateWithURL(file as CFURL, nil) else { return nil } return CGImageSourceCreateThumbnailAtIndex( source, 0, @@ -244,39 +244,28 @@ final class ImageSortModel: ObservableObject { ] as CFDictionary) } - private func runTurbo(_ sorter: ImageSorter) async { - let stream = Self.stream(sorter, items: queue, inFlight: Self.turboInFlight) - var pending: [Placed] = [] - var lastFlush = Date() - for await placed in stream { - pending.append(placed) - if Date().timeIntervalSince(lastFlush) >= Self.turboFlush { - for placed in pending.suffix(Self.turboFlightsPerFlush) where flights.count < 30 { - launch(placed, decorative: true) - } - land(pending) - pending.removeAll(keepingCapacity: true) - lastFlush = Date() - } - if Task.isCancelled || mode != .turbo { break } - } - land(pending) - } - - private func land(_ incoming: [Placed]) { + private func land(_ incoming: [Placed], highlight: Bool) { let batch = incoming.filter { self.landed.insert($0.id).inserted } - guard !batch.isEmpty else { return } - var updated = buckets + guard let last = batch.last, let chart else { return } + var updated = counts + var slot: CGRect? for placed in batch { - updated[placed.result.breed, default: []].insert(placed, at: 0) - if updated[placed.result.breed]!.count > 6 { updated[placed.result.breed]!.removeLast() } + let index = updated[placed.result.breed, default: 0] + let row = breeds.firstIndex(of: placed.result.breed) ?? 0 + chart.draw(placed.tile, row: row, index: index, wrong: !placed.matchesGold) + slot = PhotoChart.slot(row: row, index: index) + updated[placed.result.breed] = index + 1 correct += placed.matchesGold ? 1 : 0 modelMilliseconds.append(placed.result.milliseconds) - counts[placed.result.breed, default: 0] += 1 } - buckets = updated + counts = updated + chartImage = chart.snapshot() + lastSlot = highlight ? slot : nil + current = last + currentImage = Self.thumbnail(last.item.file, size: 480) let ids = Set(batch.map(\.id)) queue.removeAll { ids.contains($0.id) } + remaining = queue.count sorted += batch.count tick() if logDecisions { log(batch) } @@ -298,16 +287,6 @@ final class ImageSortModel: ObservableObject { print(lines, terminator: "") } - /// Photos per bucket (the bucket itself only keeps the latest few for display). - @Published private(set) var counts: [String: Int] = [:] - - private func refreshIncoming() { - let next = queue.first { item in !flights.contains { $0.id == item.id } } - guard next?.id != incomingID else { return } - incomingID = next?.id - incomingImage = next.flatMap { Self.thumbnail($0.file, size: 400) } - } - private func tick() { if let runStart { elapsed = elapsedBeforePause + Date().timeIntervalSince(runStart) } } diff --git a/Sources/ImageSortDemo/PhotoChart.swift b/Sources/ImageSortDemo/PhotoChart.swift new file mode 100644 index 0000000..fec16bf --- /dev/null +++ b/Sources/ImageSortDemo/PhotoChart.swift @@ -0,0 +1,68 @@ +import CoreGraphics +import Foundation + +/// A bar chart made of photos: one row per breed, each sorted photo drawn as a tile at the end of its row. +/// Tiles go into one bitmap, so thousands of them cost one image on screen. +@MainActor +final class PhotoChart { + static let tile = 16 + static let tilesPerColumn = 2 + static let columns = 130 + static let rowGap = 6 + static var rowHeight: Int { tile * tilesPerColumn + rowGap } + static var capacity: Int { tilesPerColumn * columns } + + let rows: Int + let width: Int + let height: Int + private let context: CGContext + + init(rows: Int, tint: (Int) -> CGColor) { + self.rows = rows + width = Self.columns * Self.tile + height = rows * Self.rowHeight + context = CGContext( + data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpace(name: CGColorSpace.sRGB)!, bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + for row in 0.. CGRect? { + guard index < capacity else { return nil } + let column = index / tilesPerColumn + let level = index % tilesPerColumn + return CGRect(x: column * tile, y: row * rowHeight + level * tile, width: tile, height: tile) + } + + func draw(_ image: CGImage?, row: Int, index: Int, wrong: Bool) { + guard let slot = Self.slot(row: row, index: index) else { return } + let target = rect( + row: row, x: Int(slot.minX), width: Self.tile, height: Self.tile, level: index % Self.tilesPerColumn) + if let image { + context.draw(Self.squareCrop(image), in: target.insetBy(dx: 0.5, dy: 0.5)) + } + if wrong { + context.setStrokeColor(CGColor(red: 1, green: 0.2, blue: 0.2, alpha: 1)) + context.setLineWidth(2.5) + context.stroke(target.insetBy(dx: 1.25, dy: 1.25)) + } + } + + func snapshot() -> CGImage? { context.makeImage() } + + /// Converts a top-left rectangle to the context's bottom-left coordinates. + private func rect(row: Int, x: Int, width: Int, height: Int, level: Int = 0) -> CGRect { + let top = row * Self.rowHeight + level * Self.tile + return CGRect(x: x, y: self.height - top - height, width: width, height: height) + } + + private static func squareCrop(_ image: CGImage) -> CGImage { + let side = min(image.width, image.height) + let crop = CGRect(x: (image.width - side) / 2, y: (image.height - side) / 2, width: side, height: side) + return image.cropping(to: crop) ?? image + } +} diff --git a/Sources/ImageSortDemo/README.md b/Sources/ImageSortDemo/README.md index 2d637a7..7540b81 100644 --- a/Sources/ImageSortDemo/README.md +++ b/Sources/ImageSortDemo/README.md @@ -1,6 +1,6 @@ # Sort photos -Sorts Oxford-IIIT Pets test photos into 37 breeds with SigLIP 2 (base, 256 px) on Core ML. The only hint the model +Sorts Oxford-IIIT Pets photos into 37 breeds with SigLIP 2 (base, 256 px) on Core ML. The only hint the model gets is each breed's name, in the prompt `a photo of a {breed}, a type of pet.`; nothing is trained on these photos. ```bash @@ -9,8 +9,12 @@ SIGLIP2_MODEL_DIR=/path/to/siglip2-base-patch16-256 IMAGE_SORT_AUTOPLAY=12 swift - `IMAGE_SORT_AUTOPLAY=N` flies N photos in Show mode, then switches to Turbo; `IMAGE_SORT_AUTOSTART=show|turbo` starts a run in that mode. Without either, press Start. -- `IMAGE_SORT_COUNT` sets the sample size (default 1,000; the test split has 3,669). -- Orange buckets are cat breeds, blue are dog breeds; a green frame means the gold label agrees. +- `IMAGE_SORT_COUNT` sets the sample size (default 1,000; the test split has 3,669, and a cache that also holds + the train split allows up to 7,349). `IMAGE_SORT_WAIT=1` waits for Start (Space); `IMAGE_SORT_LOG=1` prints + each decision. +- Each sorted photo becomes a tile in its breed's row, so the rows grow into a bar chart made of photos; a red + frame marks a photo whose gold breed differs. Orange rows are cat breeds, blue are dog breeds. The left panel + shows the latest photo with its five most likely breeds. Headless: `swift run -c release ImageSortCheck --inflight=4` (M5 Pro, macOS 27: 94.85% on all 3,669 test photos, 193 photos/s). From 0d5d61ee976268a8ea46d9f74b0924fe527c4368 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 22:54:31 -0400 Subject: [PATCH 04/14] Add VideoSortDemo: SigLIP 2 labels video frames in real time Name the animal (default): 26 Wikimedia Commons animal clips, every frame scored against the 26 names; caption, top-5 panel, and a Spotted strip; correct-frame rate from per-clip boundaries. M5 Pro: 31 fps, 9.5 ms per frame, 94.2% of frames, 26/26 species. Potato scene: 6x4 grid over a USDA sorting video, about 8 grids/s. GridClassifier gains top-5 and a prompt template. --- Package.swift | 1 + Sources/ImageSort/GridClassifier.swift | 94 ++++++ Sources/VideoSortDemo/ContentView.swift | 285 +++++++++++++++++++ Sources/VideoSortDemo/README.md | 21 ++ Sources/VideoSortDemo/VideoFrames.swift | 59 ++++ Sources/VideoSortDemo/VideoSortDemoApp.swift | 28 ++ Sources/VideoSortDemo/VideoSortModel.swift | 277 ++++++++++++++++++ 7 files changed, 765 insertions(+) create mode 100644 Sources/ImageSort/GridClassifier.swift create mode 100644 Sources/VideoSortDemo/ContentView.swift create mode 100644 Sources/VideoSortDemo/README.md create mode 100644 Sources/VideoSortDemo/VideoFrames.swift create mode 100644 Sources/VideoSortDemo/VideoSortDemoApp.swift create mode 100644 Sources/VideoSortDemo/VideoSortModel.swift diff --git a/Package.swift b/Package.swift index 9fc5e4f..71e2d2a 100644 --- a/Package.swift +++ b/Package.swift @@ -53,6 +53,7 @@ let package = Package( .target(name: "ImageSort", dependencies: ["FluidUse"]), .executableTarget(name: "ImageSortCheck", dependencies: ["ImageSort", "FluidUse"]), .executableTarget(name: "ImageSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]), + .executableTarget(name: "VideoSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]), .testTarget( name: "FluidUseTests", dependencies: ["FluidUse", "LayaTetris"], resources: [.copy("Fixtures")] diff --git a/Sources/ImageSort/GridClassifier.swift b/Sources/ImageSort/GridClassifier.swift new file mode 100644 index 0000000..e8d8abd --- /dev/null +++ b/Sources/ImageSort/GridClassifier.swift @@ -0,0 +1,94 @@ +import CoreGraphics +import FluidUse +import Foundation + +/// Labels every cell of a `columns × rows` grid over a frame with SigLIP 2 on Core ML. +/// Cells are scored concurrently; one call handles one cell. +public final class GridClassifier: Sendable { + public struct Cell: Sendable { + public let label: Int + /// Softmax share of the chosen label among all labels. + public let share: Float + /// The five most likely labels with their shares, best first. + public let top: [Ranked] + } + + public struct Ranked: Sendable { + public let label: Int + public let share: Float + } + + public struct Grid: Sendable { + public let cells: [Cell] + public let milliseconds: Double + } + + public let labels: [String] + public let columns: Int + public let rows: Int + private let manager: SigLIP2Manager + private let embeddings: [[Float]] + + private init(manager: SigLIP2Manager, labels: [String], embeddings: [[Float]], columns: Int, rows: Int) { + self.manager = manager + self.labels = labels + self.embeddings = embeddings + self.columns = columns + self.rows = rows + } + + /// Loads the encoders from `SIGLIP2_MODEL_DIR` and embeds `template` (with `{}` replaced) once per label. + public static func load( + labels: [String], template: String = "a photo of {}.", columns: Int, rows: Int + ) async throws -> GridClassifier { + guard let path = ProcessInfo.processInfo.environment["SIGLIP2_MODEL_DIR"], !path.isEmpty else { + throw SigLIP2Error.invalidAsset("Set SIGLIP2_MODEL_DIR to the converted siglip2-base-patch16-256 folder") + } + let manager = try await SigLIP2Manager.load(from: URL(fileURLWithPath: path)) + let embeddings = try await manager.embed( + labels: labels.map { template.replacingOccurrences(of: "{}", with: $0) }) + return GridClassifier(manager: manager, labels: labels, embeddings: embeddings, columns: columns, rows: rows) + } + + /// Classifies every cell of `frame`, keeping `inFlight` calls running. + public func classify(_ frame: CGImage, inFlight: Int = 6) async throws -> Grid { + let start = DispatchTime.now().uptimeNanoseconds + let width = frame.width / columns + let height = frame.height / rows + let crops = (0..<(columns * rows)).map { index in + frame.cropping( + to: CGRect(x: (index % columns) * width, y: (index / columns) * height, width: width, height: height)) + } + var cells = [Cell?](repeating: nil, count: crops.count) + try await withThrowingTaskGroup(of: (Int, Cell).self) { group in + var next = 0 + func launch() { + guard next < crops.count else { return } + let index = next + next += 1 + group.addTask { [self] in (index, try await cell(crops[index])) } + } + for _ in 0.. Cell { + guard let crop else { throw SigLIP2Error.invalidInput("Empty cell") } + let answer = manager.score( + imageEmbedding: try await manager.embed(image: crop), labels: labels, labelEmbeddings: embeddings) + let scale = manager.config.logitScale + let best = answer.similarities[answer.selectedIndex] + let weights = answer.similarities.map { exp(scale * ($0 - best)) } + let total = weights.reduce(0, +) + let top = weights.indices.sorted { weights[$0] > weights[$1] }.prefix(5).map { + Ranked(label: $0, share: weights[$0] / total) + } + return Cell(label: answer.selectedIndex, share: 1 / total, top: top) + } +} diff --git a/Sources/VideoSortDemo/ContentView.swift b/Sources/VideoSortDemo/ContentView.swift new file mode 100644 index 0000000..4ec6ee2 --- /dev/null +++ b/Sources/VideoSortDemo/ContentView.swift @@ -0,0 +1,285 @@ +import ImageSort +import SwiftUI + +struct ContentView: View { + @EnvironmentObject private var model: VideoSortModel + + private var scene: VideoSortModel.Scene { model.scene } + + var body: some View { + VStack(spacing: 0) { + header + Divider() + switch model.phase { + case .loading(let message): + status(message, spinning: true) + case .failed(let message): + status("Failed: \(message)", spinning: false) + default: + if scene.isGrid { + HStack(alignment: .top, spacing: 16) { + video + gridLegend.frame(width: 240) + } + .padding(14) + } else { + VStack(spacing: 12) { + HStack(alignment: .top, spacing: 16) { + video + topFive.frame(width: 290) + } + spotted + } + .padding(14) + } + } + Divider() + footer + } + .background(Color(nsColor: .windowBackgroundColor)) + } + + // MARK: Header + + private var header: some View { + ViewThatFits(in: .horizontal) { + HStack(alignment: .center, spacing: 18) { + titleBlock + Spacer(minLength: 12) + stats + button + } + VStack(alignment: .leading, spacing: 10) { + titleBlock + HStack(spacing: 14) { + stats + Spacer(minLength: 8) + button + } + } + } + .padding(.horizontal, 20) + .padding(.vertical, 12) + } + + private var titleBlock: some View { + VStack(alignment: .leading, spacing: 2) { + Text(scene.title).font(.system(size: 26, weight: .bold)) + Text(subtitle).font(.callout).foregroundStyle(.secondary).lineLimit(1).fixedSize() + } + } + + private var subtitle: String { + scene.isGrid + ? "SigLIP 2 · Core ML on the Neural Engine · \(scene.columns * scene.rows) cells per frame, zero-shot" + : "SigLIP 2 · Core ML on the Neural Engine · \(scene.labels.count) animals, zero-shot, every frame" + } + + private var stats: some View { + HStack(spacing: 16) { + stat( + scene.isGrid ? "Frames / s" : "Frames / s", + model.phase == .running ? String(format: "%.0f", model.framesPerSecond) : "–") + stat( + scene.isGrid ? "ms / frame" : "ms / frame", + model.grid.map { String(format: "%.1f", $0.milliseconds) } ?? "–") + stat("Frames labeled", "\(model.framesLabeled)") + if !scene.isGrid { + stat("Correct frames", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–") + stat("Species", "\(model.speciesSpotted) / \(scene.labels.count)") + } + } + } + + private func stat(_ title: String, _ value: String) -> some View { + VStack(alignment: .trailing, spacing: 2) { + Text(title.uppercased()).font(.caption2.weight(.semibold)).foregroundStyle(.secondary) + .lineLimit(1).fixedSize() + Text(value).font(.system(size: 20, weight: .semibold, design: .rounded)).monospacedDigit() + .contentTransition(.numericText()).fixedSize() + } + } + + private var button: some View { + Button(model.phase == .running ? "Stop" : "Start") { model.toggle() } + .keyboardShortcut(.space, modifiers: []) + .buttonStyle(.borderedProminent) + .disabled(!(model.phase == .ready || model.phase == .running)) + } + + // MARK: Video + + private var video: some View { + Color.black + .aspectRatio(16 / 9, contentMode: .fit) + .overlay { + if let frame = model.frame { + Image(decorative: frame, scale: 1).resizable().aspectRatio(contentMode: .fit) + } + } + .overlay { if model.phase == .running { overlay } } + .clipShape(RoundedRectangle(cornerRadius: 10)) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) + } + + @ViewBuilder + private var overlay: some View { + if let grid = model.grid { + if scene.isGrid { + GeometryReader { proxy in + let width = proxy.size.width / CGFloat(scene.columns) + let height = proxy.size.height / CGFloat(scene.rows) + ForEach(grid.cells.indices, id: \.self) { index in + let cell = grid.cells[index] + let label = scene.labels[cell.label] + ZStack(alignment: .topLeading) { + Rectangle().fill(label.color.opacity(0.12)) + Rectangle().stroke(label.color, lineWidth: 2) + Text("\(label.name) \(Int(cell.share * 100))%") + .font(.system(size: max(10, min(15, width / 13)), weight: .bold)) + .foregroundStyle(label.color).lineLimit(1) + .padding(.horizontal, 5).padding(.vertical, 2) + .background(Color.black.opacity(0.6)) + } + .frame(width: width, height: height) + .offset(x: CGFloat(index % scene.columns) * width, y: CGFloat(index / scene.columns) * height) + } + } + } else if let cell = grid.cells.first { + let label = scene.labels[cell.label] + let correct = model.truth.map { $0 == cell.label } + VStack { + Spacer() + HStack(alignment: .firstTextBaseline, spacing: 10) { + Text(label.name).font(.system(size: 44, weight: .heavy, design: .rounded)) + Text("\(Int(cell.share * 100))%").font(.system(size: 28, weight: .bold, design: .rounded)) + .monospacedDigit() + if correct == false { + Image(systemName: "xmark.circle.fill").foregroundStyle(.red).font(.system(size: 26)) + } + } + .foregroundStyle(label.color) + .padding(.horizontal, 18).padding(.vertical, 8) + .background(Capsule().fill(.black.opacity(0.65))) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(18) + } + } + } + } + + // MARK: Side panels + + private var topFive: some View { + VStack(alignment: .leading, spacing: 10) { + Text("TOP 5 · THIS FRAME").font(.caption.weight(.semibold)).foregroundStyle(.secondary) + if let cell = model.grid?.cells.first, model.phase == .running { + ForEach(Array(cell.top.enumerated()), id: \.offset) { rank, entry in + let label = scene.labels[entry.label] + HStack(spacing: 8) { + Text(label.name).font(rank == 0 ? .title3.weight(.bold) : .callout) + .foregroundStyle(rank == 0 ? label.color : .primary).lineLimit(1) + .frame(width: 120, alignment: .leading) + GeometryReader { proxy in + Capsule().fill(label.color.opacity(rank == 0 ? 1 : 0.6)) + .frame(width: max(2, proxy.size.width * CGFloat(entry.share))) + } + .frame(height: rank == 0 ? 12 : 7) + Text("\(Int(entry.share * 100))%").font(.callout).monospacedDigit() + .frame(width: 42, alignment: .trailing) + } + } + } + Spacer(minLength: 0) + Text( + "Labels are \(scene.labels.count) animal names, typed once: \"a photo of a {animal}.\" Nothing is trained on this video." + ) + .font(.caption).foregroundStyle(.secondary) + } + } + + private var spotted: some View { + VStack(alignment: .leading, spacing: 6) { + Text("SPOTTED · \(model.spots.count)").font(.caption.weight(.semibold)).foregroundStyle(.secondary) + ScrollViewReader { reader in + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 8) { + ForEach(model.spots) { spot in + let label = scene.labels[spot.label] + VStack(spacing: 3) { + Color.secondary.opacity(0.1) + .frame(width: 92, height: 92) + .overlay { + if let image = spot.thumbnail { + Image(decorative: image, scale: 1).resizable().aspectRatio( + contentMode: .fill) + } + } + .clipShape(RoundedRectangle(cornerRadius: 8)) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke( + spot.correct.map { $0 ? Color.green : Color.red } ?? label.color, + lineWidth: 2.5)) + Text(label.name).font(.caption.weight(.semibold)).foregroundStyle(label.color) + .lineLimit(1) + Text(String(format: "%.1f s · %d%%", spot.seconds, Int(spot.share * 100))) + .font(.caption2).foregroundStyle(.secondary).monospacedDigit() + } + .frame(width: 96) + .id(spot.id) + .transition(.scale.combined(with: .opacity)) + } + } + .animation(.spring(duration: 0.35), value: model.spots.count) + } + .onChange(of: model.spots.count) { _, _ in + if let last = model.spots.last { withAnimation { reader.scrollTo(last.id, anchor: .trailing) } } + } + } + .frame(height: 132) + } + } + + private var gridLegend: some View { + VStack(alignment: .leading, spacing: 8) { + Text("CELLS IN THIS FRAME").font(.caption.weight(.semibold)).foregroundStyle(.secondary) + let counts = model.counts + let total = max(1, counts.reduce(0, +)) + ForEach(Array(scene.labels.enumerated()), id: \.offset) { index, label in + HStack(spacing: 6) { + Text(label.name).font(.callout.weight(counts[index] > 0 ? .semibold : .regular)) + .foregroundStyle(counts[index] > 0 ? label.color : .secondary).lineLimit(1) + .frame(width: 130, alignment: .leading) + GeometryReader { proxy in + Capsule().fill(label.color) + .frame(width: max(2, proxy.size.width * CGFloat(counts[index]) / CGFloat(total))) + } + .frame(height: 8) + Text("\(counts[index])").font(.callout).monospacedDigit().frame(width: 24, alignment: .trailing) + } + } + Spacer(minLength: 0) + Text("Labels are plain phrases, typed once: \"a photo of {label}.\" Nothing is trained on this video.") + .font(.caption).foregroundStyle(.secondary) + } + } + + private func status(_ message: String, spinning: Bool) -> some View { + VStack(spacing: 12) { + if spinning { ProgressView() } + Text(message).foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + + private var footer: some View { + HStack { + Text(scene.credit) + Spacer() + Text("Model: google/siglip2-base-patch16-256 (Apache-2.0), converted to Core ML by FluidInference") + } + .font(.caption).foregroundStyle(.secondary).lineLimit(1) + .padding(.horizontal, 20).padding(.vertical, 8) + } +} diff --git a/Sources/VideoSortDemo/README.md b/Sources/VideoSortDemo/README.md new file mode 100644 index 0000000..d3f864c --- /dev/null +++ b/Sources/VideoSortDemo/README.md @@ -0,0 +1,21 @@ +# SigLIP 2 on video + +Plays a video in real time and labels the newest frame with SigLIP 2 (base, 256 px) on Core ML, as fast as the +model allows. + +```bash +SIGLIP2_MODEL_DIR=/path/to/siglip2-base-patch16-256 VIDEO_SORT_WAIT=1 swift run -c release VideoSortDemo +``` + +- Default scene, **Name the animal**: `animals.mp4`, 26 real clips of different animals from Wikimedia Commons, + about 4 s each. Every frame is scored against the 26 animal names (`a photo of a {animal}.`); the caption shows + the top label, the side panel the top five, and the strip below adds a card each time a new animal holds for six + labeled frames. "Correct frames" compares each frame with the animal its clip shows (`animals-credits.json`). + M5 Pro: every frame at 31 fps, 9.5 ms per frame, 94.2% of frames correct, 26 of 26 species spotted. +- `VIDEO_SORT_SCENE=potatoes`: a USDA potato-sorting video (public domain) with a 6 × 4 grid; each cell is labeled + from ten phrases (potatoes, a gloved hand, a truck, the sky, …), about 8 grids per second. +- `VIDEO_SORT_WAIT=1` waits on the first frame for Start (Space); `VIDEO_SORT_LOG=1` prints each labeled frame; + `VIDEO_SORT_FILE` / `VIDEO_SORT_START` play another file. + +Videos are read from `~/Library/Caches/FluidUse/video-sort/`; they are not bundled. Clip sources, licenses, and +authors are listed in `animals-credits.json`. diff --git a/Sources/VideoSortDemo/VideoFrames.swift b/Sources/VideoSortDemo/VideoFrames.swift new file mode 100644 index 0000000..1bc736a --- /dev/null +++ b/Sources/VideoSortDemo/VideoFrames.swift @@ -0,0 +1,59 @@ +import AVFoundation +import CoreGraphics +import VideoToolbox + +/// Decodes a video file at playback speed, looping from `start` seconds, scaled to `width × height`. +enum VideoFrames { + struct Frame: Sendable { + let image: CGImage + let seconds: Double + let number: Int + } + + static func stream(url: URL, start: Double, width: Int, height: Int) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + let task = Task.detached { + do { + var number = 0 + while !Task.isCancelled { + let asset = AVURLAsset(url: url) + guard let track = try await asset.loadTracks(withMediaType: .video).first else { + throw CocoaError(.fileReadCorruptFile) + } + let duration = try await asset.load(.duration) + let reader = try AVAssetReader(asset: asset) + reader.timeRange = CMTimeRange( + start: CMTime(seconds: start, preferredTimescale: 600), end: duration) + let output = AVAssetReaderTrackOutput( + track: track, + outputSettings: [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: width, + kCVPixelBufferHeightKey as String: height, + ]) + reader.add(output) + reader.startReading() + let clock = ContinuousClock() + let began = clock.now + while !Task.isCancelled, let sample = output.copyNextSampleBuffer() { + let seconds = sample.presentationTimeStamp.seconds + try await clock.sleep(until: began + .seconds(seconds - start)) + guard let buffer = sample.imageBuffer else { continue } + var image: CGImage? + VTCreateCGImageFromCVPixelBuffer(buffer, options: nil, imageOut: &image) + if let image { + number += 1 + continuation.yield(Frame(image: image, seconds: seconds, number: number)) + } + } + reader.cancelReading() + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + } +} diff --git a/Sources/VideoSortDemo/VideoSortDemoApp.swift b/Sources/VideoSortDemo/VideoSortDemoApp.swift new file mode 100644 index 0000000..60247ed --- /dev/null +++ b/Sources/VideoSortDemo/VideoSortDemoApp.swift @@ -0,0 +1,28 @@ +import AppKit +import SwiftUI + +@main +struct VideoSortDemoApp: App { + @StateObject private var model = VideoSortModel() + + init() { + setvbuf(stdout, nil, _IOLBF, 0) + // Bare SwiftPM executables start as background processes; make this one a regular windowed app. + for key in UserDefaults.standard.dictionaryRepresentation().keys where key.hasPrefix("NSWindow Frame") { + UserDefaults.standard.removeObject(forKey: key) + } + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate(ignoringOtherApps: true) + } + + var body: some Scene { + WindowGroup("SigLIP 2 on-device — video") { + ContentView() + .environmentObject(model) + .frame(minWidth: 720, minHeight: 520) + .task { await model.prepare() } + } + .defaultSize(width: 1560, height: 980) + .windowResizability(.contentMinSize) + } +} diff --git a/Sources/VideoSortDemo/VideoSortModel.swift b/Sources/VideoSortDemo/VideoSortModel.swift new file mode 100644 index 0000000..f0620ec --- /dev/null +++ b/Sources/VideoSortDemo/VideoSortModel.swift @@ -0,0 +1,277 @@ +import CoreGraphics +import Foundation +import ImageIO +import ImageSort +import SwiftUI + +/// Plays a video in real time and keeps labeling the newest frame as fast as the model allows. +@MainActor +final class VideoSortModel: ObservableObject { + enum Phase: Equatable { + case loading(String) + case ready + case running + case failed(String) + } + + struct Label: Identifiable { + let name: String + let color: Color + var id: String { name } + } + + /// What to play and how to label it. + struct Scene { + let title: String + let file: String + let start: Double + let labels: [Label] + let template: String + let columns: Int + let rows: Int + let credit: String + /// JSON next to the video listing each single-label clip's `animal`, `clip_start` and `clip_end` seconds. + let clipsFile: String? + var isGrid: Bool { columns * rows > 1 } + } + + /// An animal the stream settled on, with the frame it was first seen in. + struct Spot: Identifiable { + let id: Int + let label: Int + let share: Float + let seconds: Double + let thumbnail: CGImage? + let correct: Bool? + } + + static let directory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + .appendingPathComponent("FluidUse/video-sort") + + static let animals: Scene = { + let names = [ + "elephant", "giraffe", "zebra", "tiger", "penguin", "flamingo", "brown bear", "wolf", "deer", "horse", + "goat", + "duck", "owl", "eagle", "monkey", "hippopotamus", "rhinoceros", "seal", "bison", "moose", "fox", "pelican", + "swan", "camel", "cat", "dog", + ] + let labels = names.enumerated().map { index, name in + Label( + name: name, color: Color(hue: Double(index) / Double(names.count), saturation: 0.65, brightness: 0.95)) + } + return Scene( + title: "Name the animal", file: "animals.mp4", start: 0, labels: labels, template: "a photo of a {}.", + columns: 1, rows: 1, + credit: + "Video: 26 clips from Wikimedia Commons (CC BY / CC BY-SA / CC0 / public domain; animals-credits.json)", + clipsFile: "animals-credits.json") + }() + + static let potatoes = Scene( + title: "Label every frame", file: "potatoes.mp4", start: 9.5, + labels: [ + Label(name: "potatoes", color: Color(red: 0.95, green: 0.78, blue: 0.35)), + Label(name: "a gloved hand", color: .red), Label(name: "a person", color: .purple), + Label(name: "a conveyor belt", color: .blue), Label(name: "a truck", color: .orange), + Label(name: "the sky", color: .cyan), Label(name: "a metal machine", color: .gray), + Label(name: "gravel ground", color: .brown), Label(name: "a wire basket", color: .green), + Label(name: "a car", color: .pink), + ], + template: "a photo of {}.", columns: 6, rows: 4, + credit: "Video: \"Potatoes Sorting Montana2026\", USDA (Brien Aho), public domain, via Wikimedia Commons", + clipsFile: nil) + + let scene: Scene + @Published private(set) var phase: Phase = .loading("Starting…") + @Published private(set) var frame: CGImage? + @Published private(set) var grid: GridClassifier.Grid? + @Published private(set) var framesPerSecond: Double = 0 + @Published private(set) var framesLabeled = 0 + @Published private(set) var framesCorrect = 0 + @Published private(set) var spots: [Spot] = [] + /// Label the current clip really shows, when the scene knows it. + @Published private(set) var truth: Int? + + private let environment = ProcessInfo.processInfo.environment + private let logFrames = ProcessInfo.processInfo.environment["VIDEO_SORT_LOG"] == "1" + private var classifier: GridClassifier? + private var player: Task? + private var labeler: Task? + private var latest: VideoFrames.Frame? + private var labelTimes: [Date] = [] + private var streak: (label: Int, count: Int) = (-1, 0) + /// (start, end, label) per clip, when the scene has single-label clips. + private var clips: [(start: Double, end: Double, label: Int)] = [] + private var preparing = false + + init() { + scene = ProcessInfo.processInfo.environment["VIDEO_SORT_SCENE"] == "potatoes" ? Self.potatoes : Self.animals + } + + var counts: [Int] { + var counts = [Int](repeating: 0, count: scene.labels.count) + for cell in grid?.cells ?? [] { counts[cell.label] += 1 } + return counts + } + + var accuracy: Double? { + framesLabeled > 0 && !clips.isEmpty ? Double(framesCorrect) / Double(framesLabeled) : nil + } + + private func expectedLabel(at seconds: Double) -> Int? { + clips.first { seconds >= $0.start && seconds < $0.end }?.label + } + + private func loadClips() { + struct Clip: Decodable { + let animal: String + let clipStart: Double + let clipEnd: Double + enum CodingKeys: String, CodingKey { + case animal + case clipStart = "clip_start" + case clipEnd = "clip_end" + } + } + guard let file = scene.clipsFile, + let data = try? Data(contentsOf: Self.directory.appendingPathComponent(file)), + let decoded = try? JSONDecoder().decode([Clip].self, from: data) + else { return } + let names = scene.labels.map(\.name) + clips = decoded.compactMap { clip in + names.firstIndex(of: clip.animal).map { (clip.clipStart, clip.clipEnd, $0) } + } + } + + var speciesSpotted: Int { Set(spots.filter { $0.correct != false }.map(\.label)).count } + + func prepare() async { + guard !preparing else { return } + preparing = true + do { + loadClips() + phase = .loading("Loading SigLIP 2 and embedding \(scene.labels.count) labels…") + classifier = try await GridClassifier.load( + labels: scene.labels.map(\.name), template: scene.template, columns: scene.columns, rows: scene.rows) + phase = .ready + startPlayback() + print("ready at \(Date().timeIntervalSince1970)") + // VIDEO_SORT_WAIT=1 shows the first frame and waits for Start (Space). + if environment["VIDEO_SORT_WAIT"] != "1" { toggle() } + } catch { + phase = .failed(error.localizedDescription) + } + } + + private func startPlayback() { + let path = environment["VIDEO_SORT_FILE"] ?? Self.directory.appendingPathComponent(scene.file).path + let start = environment["VIDEO_SORT_START"].flatMap(Double.init) ?? scene.start + player?.cancel() + latest = nil + player = Task { [weak self] in + do { + for try await frame in VideoFrames.stream( + url: URL(fileURLWithPath: path), start: start, width: 1280, height: 720) + { + guard let self else { return } + self.frame = frame.image + self.latest = frame + truth = expectedLabel(at: frame.seconds) + // Hold the first frame until Start. + if phase == .ready, environment["VIDEO_SORT_WAIT"] == "1" { break } + } + } catch { + self?.phase = .failed("Video: \(error.localizedDescription)") + } + } + } + + func toggle() { + if phase == .running { + labeler?.cancel() + player?.cancel() + phase = .ready + return + } + guard phase == .ready, let classifier else { return } + phase = .running + framesLabeled = 0 + framesCorrect = 0 + spots = [] + streak = (-1, 0) + labelTimes = [] + startPlayback() + let inFlight = scene.isGrid ? 6 : 1 + labeler = Task { [weak self] in + var lastNumber = -1 + while !Task.isCancelled { + guard let frame = self?.latest, frame.number != lastNumber else { + try? await Task.sleep(for: .milliseconds(3)) + continue + } + lastNumber = frame.number + guard let grid = try? await Self.label(frame.image, with: classifier, inFlight: inFlight) else { + continue + } + self?.apply(grid, frame: frame) + } + } + } + + private nonisolated static func label( + _ image: CGImage, with classifier: GridClassifier, inFlight: Int + ) + async throws -> GridClassifier.Grid + { + try await Task.detached { try await classifier.classify(image, inFlight: inFlight) }.value + } + + private func apply(_ grid: GridClassifier.Grid, frame: VideoFrames.Frame) { + self.grid = grid + framesLabeled += 1 + let clipTruth = expectedLabel(at: frame.seconds) + if let clipTruth, grid.cells.first?.label == clipTruth { framesCorrect += 1 } + let now = Date() + labelTimes.append(now) + labelTimes.removeAll { now.timeIntervalSince($0) > 1 } + framesPerSecond = Double(labelTimes.count) + if !scene.isGrid, let cell = grid.cells.first { track(cell, frame: frame, truth: clipTruth) } + if logFrames { log(grid, frame: frame, truth: clipTruth) } + } + + /// Adds a spot once the same confident label holds for several labeled frames in a row. + private func track(_ cell: GridClassifier.Cell, frame: VideoFrames.Frame, truth: Int?) { + streak = cell.label == streak.label ? (cell.label, streak.count + 1) : (cell.label, 1) + guard streak.count == 6, cell.share >= 0.5, spots.last?.label != cell.label else { return } + let thumbnail = frame.image.cropping( + to: CGRect( + x: (frame.image.width - frame.image.height) / 2, y: 0, width: frame.image.height, + height: frame.image.height)) + spots.append( + Spot( + id: spots.count, label: cell.label, share: cell.share, seconds: frame.seconds, thumbnail: thumbnail, + correct: truth.map { $0 == cell.label })) + } + + private func log(_ grid: GridClassifier.Grid, frame: VideoFrames.Frame, truth: Int?) { + let (cyan, yellow, red, green, dim, reset) = + ("\u{1B}[1;36m", "\u{1B}[33m", "\u{1B}[31m", "\u{1B}[32m", "\u{1B}[2m", "\u{1B}[0m") + let time = String(format: "%.2f", frame.seconds) + if scene.isGrid { + let summary = counts.enumerated().filter { $0.element > 0 }.sorted { $0.element > $1.element } + .map { "\(scene.labels[$0.offset].name) \($0.element)" }.joined(separator: " · ") + print( + "\(cyan)▶ frame \(frame.number) @ \(time) s\(reset) \(yellow)\(summary)\(reset) " + + "\(red)\(grid.cells.count) cells in \(String(format: "%.0f", grid.milliseconds)) ms\(reset)") + return + } + guard let cell = grid.cells.first else { return } + let mark: String = + truth.map { $0 == cell.label ? "\(green)✓\(reset)" : "\(red)✗ clip: \(scene.labels[$0].name)\(reset)" } + ?? "" + print( + "\(cyan)▶ frame \(frame.number) @ \(time) s\(reset) \(yellow)→ \(scene.labels[cell.label].name)\(reset) · " + + "\(Int(cell.share * 100))% · \(red)model call \(String(format: "%.1f", grid.milliseconds)) ms\(reset) " + + "\(mark)\(dim)\(reset)") + } +} From 93d25f9b39c895bc66df2f3f84a235b1f5311164 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:13:49 -0400 Subject: [PATCH 05/14] ImageSortDemo: photos fly from Now Sorting into their chart slot Show mode flies every photo and lands its tile on arrival; Turbo flies two photos per 50 ms flush while the chart updates at full speed (7,349 photos in 39 s, 187 photos/s, 94.3%). --- Sources/ImageSortDemo/ContentView.swift | 53 +++++++++++++ Sources/ImageSortDemo/ImageSortModel.swift | 87 +++++++++++++++++++--- 2 files changed, 129 insertions(+), 11 deletions(-) diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index c8bf083..1712977 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -1,8 +1,25 @@ import ImageSort import SwiftUI +private struct FramesKey: PreferenceKey { + static let defaultValue: [String: CGRect] = [:] + static func reduce(value: inout [String: CGRect], nextValue: () -> [String: CGRect]) { + value.merge(nextValue()) { $1 } + } +} + +extension View { + fileprivate func reportFrame(_ key: String) -> some View { + background( + GeometryReader { proxy in + Color.clear.preference(key: FramesKey.self, value: [key: proxy.frame(in: .named("board"))]) + }) + } +} + struct ContentView: View { @EnvironmentObject private var model: ImageSortModel + @State private var frames: [String: CGRect] = [:] var body: some View { VStack(spacing: 0) { @@ -108,6 +125,40 @@ struct ContentView: View { chart } .padding(14) + .coordinateSpace(name: "board") + .onPreferenceChange(FramesKey.self) { frames = $0 } + .overlay(alignment: .topLeading) { flying } + } + + /// Photos travelling from the Now Sorting panel to their tile; they shrink to tile size on arrival. + private var flying: some View { + ZStack(alignment: .topLeading) { + if let from = frames["photo"], let chart = frames["chart"] { + let scale = chart.width / CGFloat(PhotoChart.columns * PhotoChart.tile) + ForEach(model.flights) { flight in + let target = CGRect( + x: chart.minX + flight.slot.minX * scale, y: chart.minY + flight.slot.minY * scale, + width: flight.slot.width * scale, height: flight.slot.height * scale) + let side = flight.arrived ? max(target.width, 4) : from.width * 0.55 + Color.clear + .frame(width: side, height: side) + .overlay { + if let image = flight.image { + Image(decorative: image, scale: 1).resizable().aspectRatio(contentMode: .fill) + } + } + .clipShape(RoundedRectangle(cornerRadius: flight.arrived ? 1 : 8)) + .overlay( + RoundedRectangle(cornerRadius: flight.arrived ? 1 : 8) + .stroke(flight.wrong ? Color.red : Color.green, lineWidth: flight.arrived ? 1 : 3) + ) + .shadow(color: .black.opacity(0.4), radius: flight.arrived ? 0 : 6) + .position( + x: flight.arrived ? target.midX : from.midX, y: flight.arrived ? target.midY : from.midY) + } + } + } + .allowsHitTesting(false) } private var nowSorting: some View { @@ -121,6 +172,7 @@ struct ContentView: View { } } .clipShape(RoundedRectangle(cornerRadius: 12)) + .reportFrame("photo") .overlay( RoundedRectangle(cornerRadius: 12) .stroke(model.current.map { $0.matchesGold ? Color.green : Color.red } ?? .clear, lineWidth: 3)) @@ -185,6 +237,7 @@ struct ContentView: View { } } .frame(width: width * scale, height: height * scale, alignment: .topLeading) + .reportFrame("chart") } } } diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index afa9e79..d32fcc4 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -47,6 +47,21 @@ final class ImageSortModel: ObservableObject { @Published private(set) var currentImage: CGImage? /// Where the latest tile landed, for the highlight in Show mode. @Published private(set) var lastSlot: CGRect? + /// Photos travelling from the Now Sorting panel to their chart slot. + @Published private(set) var flights: [Flight] = [] + + struct Flight: Identifiable { + let id: Int + let image: CGImage? + /// Target tile in chart pixels. + let slot: CGRect + let wrong: Bool + var arrived = false + } + + static let travel = 0.55 + static let turboFlightsPerFlush = 2 + private var flightCounter = 0 let breeds = PetsSample.breeds let total: Int @@ -129,6 +144,7 @@ final class ImageSortModel: ObservableObject { current = nil currentImage = nil lastSlot = nil + flights = [] sorted = 0 correct = 0 elapsed = 0 @@ -176,7 +192,21 @@ final class ImageSortModel: ObservableObject { private func runShow(_ sorter: ImageSorter) async { for await placed in Self.stream(sorter, items: queue, inFlight: 2) { if Task.isCancelled || mode != .show { break } - land([placed], highlight: true) + current = placed + currentImage = Self.thumbnail(placed.item.file, size: 480) + let breed = placed.result.breed + let row = breeds.firstIndex(of: breed) ?? 0 + let inAir = flights.filter { + $0.slot.minY >= CGFloat(row * PhotoChart.rowHeight) + && $0.slot.minY < CGFloat((row + 1) * PhotoChart.rowHeight) + }.count + if let slot = PhotoChart.slot(row: row, index: (counts[breed] ?? 0) + inAir) { + fly(image: placed.tile ?? currentImage, to: slot, wrong: !placed.matchesGold) { [weak self] in + self?.land([placed], highlight: true, updateCurrent: false) + } + } else { + land([placed], highlight: true, updateCurrent: false) + } shownInShow += 1 if let count = environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init), shownInShow >= count { mode = .turbo @@ -185,6 +215,8 @@ final class ImageSortModel: ObservableObject { try? await Task.sleep(for: .seconds(1 / pace)) if Date().timeIntervalSince(started) < 1 / pace { break } } + // Let photos already in the air land before Turbo takes over the queue. + while !Task.isCancelled, !flights.isEmpty { try? await Task.sleep(for: .milliseconds(20)) } } private func runTurbo(_ sorter: ImageSorter) async { @@ -193,13 +225,41 @@ final class ImageSortModel: ObservableObject { for await placed in Self.stream(sorter, items: queue, inFlight: Self.turboInFlight) { pending.append(placed) if Date().timeIntervalSince(lastFlush) >= Self.turboFlush { - land(pending, highlight: false) + flush(pending) pending.removeAll(keepingCapacity: true) lastFlush = Date() } if Task.isCancelled || mode != .turbo { break } } - land(pending, highlight: false) + flush(pending) + } + + /// Lands a Turbo batch at once; a couple of its photos also fly so the motion stays visible. + private func flush(_ batch: [Placed]) { + let slots = land(batch, highlight: false, updateCurrent: true) + for (placed, slot) in slots.suffix(Self.turboFlightsPerFlush) where flights.count < 24 { + fly(image: placed.tile, to: slot, wrong: !placed.matchesGold, decorative: true) {} + } + } + + /// Animates `image` from the Now Sorting panel to `slot`, then runs `arrival`. + private func fly( + image: CGImage?, to slot: CGRect, wrong: Bool, decorative: Bool = false, + arrival: @escaping @MainActor () -> Void + ) { + flightCounter += 1 + let id = flightCounter + flights.append(Flight(id: id, image: image, slot: slot, wrong: wrong)) + let travel = decorative ? Self.travel * 0.7 : Self.travel + Task { @MainActor [weak self] in + try? await Task.sleep(for: .milliseconds(20)) + withAnimation(.easeIn(duration: travel)) { + if let index = self?.flights.firstIndex(where: { $0.id == id }) { self?.flights[index].arrived = true } + } + try? await Task.sleep(for: .seconds(travel)) + self?.flights.removeAll { $0.id == id } + arrival() + } } /// Sorts `items` with `inFlight` calls always running off the main thread and streams each result. @@ -218,7 +278,7 @@ final class ImageSortModel: ObservableObject { next += 1 group.addTask { guard let result = try? await sorter.sort(item) else { return nil } - return Placed(item: item, result: result, tile: thumbnail(item.file, size: 48)) + return Placed(item: item, result: result, tile: thumbnail(item.file, size: 160)) } } for _ in 0.. [(Placed, CGRect)] { let batch = incoming.filter { self.landed.insert($0.id).inserted } - guard let last = batch.last, let chart else { return } + guard let last = batch.last, let chart else { return [] } var updated = counts - var slot: CGRect? + var slots: [(Placed, CGRect)] = [] for placed in batch { let index = updated[placed.result.breed, default: 0] let row = breeds.firstIndex(of: placed.result.breed) ?? 0 chart.draw(placed.tile, row: row, index: index, wrong: !placed.matchesGold) - slot = PhotoChart.slot(row: row, index: index) + if let slot = PhotoChart.slot(row: row, index: index) { slots.append((placed, slot)) } updated[placed.result.breed] = index + 1 correct += placed.matchesGold ? 1 : 0 modelMilliseconds.append(placed.result.milliseconds) } counts = updated chartImage = chart.snapshot() - lastSlot = highlight ? slot : nil - current = last - currentImage = Self.thumbnail(last.item.file, size: 480) + lastSlot = highlight ? slots.last?.1 : nil + if updateCurrent { + current = last + currentImage = Self.thumbnail(last.item.file, size: 480) + } let ids = Set(batch.map(\.id)) queue.removeAll { ids.contains($0.id) } remaining = queue.count sorted += batch.count tick() if logDecisions { log(batch) } + return slots } private func log(_ batch: [Placed]) { From d401da47464c7803dff380f515d22c32c7b87cb9 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:15:56 -0400 Subject: [PATCH 06/14] ImageSortDemo: chart reshapes to fill the window; stacked layout when narrow The empty chart picks how many tiles to stack per breed row (1-10) so the bitmap fills the available space; windows under 980 pt put the current photo and its top five above the chart. --- Sources/ImageSortDemo/ContentView.swift | 108 ++++++++++++++------- Sources/ImageSortDemo/ImageSortModel.swift | 43 +++++--- Sources/ImageSortDemo/PhotoChart.swift | 58 +++++++---- 3 files changed, 143 insertions(+), 66 deletions(-) diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index 1712977..a4c530c 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -120,9 +120,19 @@ struct ContentView: View { } private var board: some View { - HStack(alignment: .top, spacing: 18) { - nowSorting.frame(width: 250) - chart + GeometryReader { proxy in + // Narrow windows stack the current photo above the chart so the chart gets the full width. + if proxy.size.width < 980 { + VStack(alignment: .leading, spacing: 12) { + compactNowSorting.frame(height: min(200, proxy.size.height * 0.28)) + chart(labelWidth: 150) + } + } else { + HStack(alignment: .top, spacing: 18) { + nowSorting.frame(width: 250) + chart(labelWidth: 190) + } + } } .padding(14) .coordinateSpace(name: "board") @@ -134,7 +144,7 @@ struct ContentView: View { private var flying: some View { ZStack(alignment: .topLeading) { if let from = frames["photo"], let chart = frames["chart"] { - let scale = chart.width / CGFloat(PhotoChart.columns * PhotoChart.tile) + let scale = chart.width / CGFloat(model.chartGeometry.width) ForEach(model.flights) { flight in let target = CGRect( x: chart.minX + flight.slot.minX * scale, y: chart.minY + flight.slot.minY * scale, @@ -161,37 +171,62 @@ struct ContentView: View { .allowsHitTesting(false) } - private var nowSorting: some View { - VStack(alignment: .leading, spacing: 10) { - Text("NOW SORTING · \(model.remaining) left").font(.caption.weight(.semibold)).foregroundStyle(.secondary) - Color.secondary.opacity(0.08) - .aspectRatio(1, contentMode: .fit) - .overlay { - if let image = model.currentImage { - Image(decorative: image, scale: 1).resizable().aspectRatio(contentMode: .fill) - } + private var photo: some View { + Color.secondary.opacity(0.08) + .aspectRatio(1, contentMode: .fit) + .overlay { + if let image = model.currentImage { + Image(decorative: image, scale: 1).resizable().aspectRatio(contentMode: .fill) } - .clipShape(RoundedRectangle(cornerRadius: 12)) - .reportFrame("photo") - .overlay( - RoundedRectangle(cornerRadius: 12) - .stroke(model.current.map { $0.matchesGold ? Color.green : Color.red } ?? .clear, lineWidth: 3)) - if let current = model.current { - VStack(alignment: .leading, spacing: 5) { - ForEach(Array(current.result.top.enumerated()), id: \.offset) { rank, entry in - topRow(entry.breed, share: entry.share, first: rank == 0, gold: current.item.breed) - } - if !current.matchesGold { - Text("label: \(current.item.breed)").font(.caption.weight(.semibold)).foregroundStyle(.red) - } + } + .clipShape(RoundedRectangle(cornerRadius: 12)) + .reportFrame("photo") + .overlay( + RoundedRectangle(cornerRadius: 12) + .stroke(model.current.map { $0.matchesGold ? Color.green : Color.red } ?? .clear, lineWidth: 3)) + } + + @ViewBuilder + private var topFive: some View { + if let current = model.current { + VStack(alignment: .leading, spacing: 5) { + ForEach(Array(current.result.top.enumerated()), id: \.offset) { rank, entry in + topRow(entry.breed, share: entry.share, first: rank == 0, gold: current.item.breed) + } + if !current.matchesGold { + Text("label: \(current.item.breed)").font(.caption.weight(.semibold)).foregroundStyle(.red) } } + } + } + + private var nowSorting: some View { + VStack(alignment: .leading, spacing: 10) { + Text("NOW SORTING · \(model.remaining) left").font(.caption.weight(.semibold)).foregroundStyle(.secondary) + photo + topFive Spacer(minLength: 0) Text("Labels are the 37 breed names, typed once: \"a photo of a {breed}, a type of pet.\"") .font(.caption).foregroundStyle(.secondary) } } + private var compactNowSorting: some View { + HStack(alignment: .top, spacing: 14) { + photo + VStack(alignment: .leading, spacing: 8) { + Text("NOW SORTING · \(model.remaining) left").font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + topFive + Spacer(minLength: 0) + Text("Labels are the 37 breed names, typed once: \"a photo of a {breed}, a type of pet.\"") + .font(.caption).foregroundStyle(.secondary) + } + .frame(maxWidth: 420, alignment: .leading) + Spacer(minLength: 0) + } + } + private func topRow(_ breed: String, share: Float, first: Bool, gold: String) -> some View { HStack(spacing: 6) { Text(breed).font(first ? .callout.weight(.bold) : .caption).lineLimit(1) @@ -205,22 +240,23 @@ struct ContentView: View { } } - private var chart: some View { + private func chart(labelWidth: CGFloat) -> some View { GeometryReader { proxy in - let labelWidth: CGFloat = 190 - let width = CGFloat(PhotoChart.columns * PhotoChart.tile) - let height = CGFloat(model.breeds.count * PhotoChart.rowHeight) - let scale = min((proxy.size.width - labelWidth) / width, proxy.size.height / height) - let rowHeight = CGFloat(PhotoChart.rowHeight) * scale + let geometry = model.chartGeometry + let width = CGFloat(geometry.width) + let height = CGFloat(geometry.height) + let available = CGSize(width: proxy.size.width - labelWidth, height: proxy.size.height) + let scale = min(available.width / width, available.height / height) + let rowHeight = CGFloat(geometry.rowHeight) * scale HStack(alignment: .top, spacing: 0) { VStack(alignment: .trailing, spacing: 0) { ForEach(model.breeds, id: \.self) { breed in HStack(spacing: 6) { - Text(breed).lineLimit(1).foregroundStyle(color(for: breed)) + Text(breed).lineLimit(1).minimumScaleFactor(0.7).foregroundStyle(color(for: breed)) Text("\(model.counts[breed] ?? 0)").monospacedDigit().foregroundStyle(.secondary) - .frame(width: 34, alignment: .trailing) + .frame(width: 30, alignment: .trailing) } - .font(.system(size: max(9, min(13, rowHeight * 0.55)), weight: .semibold)) + .font(.system(size: max(8, min(13, rowHeight * 0.6)), weight: .semibold)) .frame(width: labelWidth - 8, height: rowHeight, alignment: .trailing) .padding(.trailing, 8) } @@ -239,6 +275,8 @@ struct ContentView: View { .frame(width: width * scale, height: height * scale, alignment: .topLeading) .reportFrame("chart") } + .onAppear { model.fitChart(to: available) } + .onChange(of: available) { _, size in model.fitChart(to: size) } } } diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index d32fcc4..df30dca 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -128,15 +128,39 @@ final class ImageSortModel: ObservableObject { } } - func reset() { - runner?.cancel() - runner = nil - chart = PhotoChart(rows: breeds.count) { [breeds] row in + /// Pixel size and shape of the chart bitmap, for the view's layout. + struct ChartGeometry: Equatable { + var width = 1 + var height = 1 + var rowHeight = 1 + } + + @Published private(set) var chartGeometry = ChartGeometry() + private var levels = 2 + + private func makeChart() { + let chart = PhotoChart(rows: breeds.count, levels: levels) { [breeds] row in Self.cats.contains(breeds[row]) ? CGColor(red: 1, green: 0.6, blue: 0.2, alpha: 0.10) : CGColor(red: 0.3, green: 0.55, blue: 1, alpha: 0.10) } - chartImage = chart?.snapshot() + self.chart = chart + chartGeometry = ChartGeometry(width: chart.width, height: chart.height, rowHeight: chart.rowHeight) + chartImage = chart.snapshot() + } + + /// Reshapes the empty chart to fill `size` (points available for the tiles). Ignored once sorting has begun. + func fitChart(to size: CGSize) { + let best = PhotoChart.bestLevels(rows: breeds.count, size: size) + guard best != levels, sorted == 0, flights.isEmpty, phase != .running, chart != nil else { return } + levels = best + makeChart() + } + + func reset() { + runner?.cancel() + runner = nil + makeChart() queue = items remaining = items.count landed = [] @@ -196,11 +220,8 @@ final class ImageSortModel: ObservableObject { currentImage = Self.thumbnail(placed.item.file, size: 480) let breed = placed.result.breed let row = breeds.firstIndex(of: breed) ?? 0 - let inAir = flights.filter { - $0.slot.minY >= CGFloat(row * PhotoChart.rowHeight) - && $0.slot.minY < CGFloat((row + 1) * PhotoChart.rowHeight) - }.count - if let slot = PhotoChart.slot(row: row, index: (counts[breed] ?? 0) + inAir) { + let inAir = flights.filter { chart?.row(of: $0.slot) == row }.count + if let slot = chart?.slot(row: row, index: (counts[breed] ?? 0) + inAir) { fly(image: placed.tile ?? currentImage, to: slot, wrong: !placed.matchesGold) { [weak self] in self?.land([placed], highlight: true, updateCurrent: false) } @@ -315,7 +336,7 @@ final class ImageSortModel: ObservableObject { let index = updated[placed.result.breed, default: 0] let row = breeds.firstIndex(of: placed.result.breed) ?? 0 chart.draw(placed.tile, row: row, index: index, wrong: !placed.matchesGold) - if let slot = PhotoChart.slot(row: row, index: index) { slots.append((placed, slot)) } + if let slot = chart.slot(row: row, index: index) { slots.append((placed, slot)) } updated[placed.result.breed] = index + 1 correct += placed.matchesGold ? 1 : 0 modelMilliseconds.append(placed.result.milliseconds) diff --git a/Sources/ImageSortDemo/PhotoChart.swift b/Sources/ImageSortDemo/PhotoChart.swift index fec16bf..0864219 100644 --- a/Sources/ImageSortDemo/PhotoChart.swift +++ b/Sources/ImageSortDemo/PhotoChart.swift @@ -2,46 +2,65 @@ import CoreGraphics import Foundation /// A bar chart made of photos: one row per breed, each sorted photo drawn as a tile at the end of its row. -/// Tiles go into one bitmap, so thousands of them cost one image on screen. +/// Tiles go into one bitmap, so thousands of them cost one image on screen. `levels` tiles stack in each row; +/// the column count follows so every row holds `capacity` photos. @MainActor final class PhotoChart { static let tile = 16 - static let tilesPerColumn = 2 - static let columns = 130 static let rowGap = 6 - static var rowHeight: Int { tile * tilesPerColumn + rowGap } - static var capacity: Int { tilesPerColumn * columns } + static let capacity = 260 let rows: Int + let levels: Int + let columns: Int let width: Int let height: Int + var rowHeight: Int { Self.tile * levels + Self.rowGap } private let context: CGContext - init(rows: Int, tint: (Int) -> CGColor) { + init(rows: Int, levels: Int, tint: (Int) -> CGColor) { self.rows = rows - width = Self.columns * Self.tile - height = rows * Self.rowHeight + self.levels = levels + columns = (Self.capacity + levels - 1) / levels + width = columns * Self.tile + height = rows * (Self.tile * levels + Self.rowGap) context = CGContext( data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpace(name: CGColorSpace.sRGB)!, bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! for row in 0.. Int { + guard size.width > 0, size.height > 0 else { return 2 } + return (1...10).max { scale(rows: rows, levels: $0, size: size) < scale(rows: rows, levels: $1, size: size) } + ?? 2 + } + + private static func scale(rows: Int, levels: Int, size: CGSize) -> CGFloat { + let width = CGFloat((capacity + levels - 1) / levels * tile) + let height = CGFloat(rows * (tile * levels + rowGap)) + return min(size.width / width, size.height / height) + } + /// Tile rectangle for the `index`-th photo of `row`, in top-left pixel coordinates (nil past capacity). - static func slot(row: Int, index: Int) -> CGRect? { - guard index < capacity else { return nil } - let column = index / tilesPerColumn - let level = index % tilesPerColumn - return CGRect(x: column * tile, y: row * rowHeight + level * tile, width: tile, height: tile) + func slot(row: Int, index: Int) -> CGRect? { + guard index < Self.capacity else { return nil } + let column = index / levels + let level = index % levels + return CGRect( + x: column * Self.tile, y: row * rowHeight + level * Self.tile, width: Self.tile, height: Self.tile) } + /// Row a slot belongs to. + func row(of slot: CGRect) -> Int { Int(slot.minY) / rowHeight } + func draw(_ image: CGImage?, row: Int, index: Int, wrong: Bool) { - guard let slot = Self.slot(row: row, index: index) else { return } - let target = rect( - row: row, x: Int(slot.minX), width: Self.tile, height: Self.tile, level: index % Self.tilesPerColumn) + guard let slot = slot(row: row, index: index) else { return } + let target = rect(top: Int(slot.minY), x: Int(slot.minX), width: Self.tile, height: Self.tile) if let image { context.draw(Self.squareCrop(image), in: target.insetBy(dx: 0.5, dy: 0.5)) } @@ -55,9 +74,8 @@ final class PhotoChart { func snapshot() -> CGImage? { context.makeImage() } /// Converts a top-left rectangle to the context's bottom-left coordinates. - private func rect(row: Int, x: Int, width: Int, height: Int, level: Int = 0) -> CGRect { - let top = row * Self.rowHeight + level * Self.tile - return CGRect(x: x, y: self.height - top - height, width: width, height: height) + private func rect(top: Int, x: Int, width: Int, height: Int) -> CGRect { + CGRect(x: x, y: self.height - top - height, width: width, height: height) } private static func squareCrop(_ image: CGImage) -> CGImage { From 3a49d98f7c528b4e6a2f116a3631e44161603ae2 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:18:01 -0400 Subject: [PATCH 07/14] ImageSortDemo: narrow windows put the current photo, stats, and buttons in one band --- Sources/ImageSortDemo/ContentView.swift | 99 ++++++++++++------------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index a4c530c..eed7489 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -22,35 +22,57 @@ struct ContentView: View { @State private var frames: [String: CGRect] = [:] var body: some View { - VStack(spacing: 0) { - header - Divider() - switch model.phase { - case .loading(let message): - status(message, spinning: true) - case .failed(let message): - status("Failed: \(message)", spinning: false) - default: - board + GeometryReader { proxy in + // Narrow windows merge the header and the current photo into one band so the chart gets the rest. + let narrow = proxy.size.width < 980 + VStack(spacing: 0) { + if narrow { compactTop } else { header } + Divider() + switch model.phase { + case .loading(let message): + status(message, spinning: true) + case .failed(let message): + status("Failed: \(message)", spinning: false) + default: + board(narrow: narrow) + } + Divider() + footer } - Divider() - footer + .coordinateSpace(name: "board") + .onPreferenceChange(FramesKey.self) { frames = $0 } + .overlay(alignment: .topLeading) { flying } } .background(Color(nsColor: .windowBackgroundColor)) } + private var compactTop: some View { + HStack(alignment: .top, spacing: 16) { + photo.frame(width: 176, height: 176) + VStack(alignment: .leading, spacing: 10) { + titleBlock + stats(size: 17) + topFive.frame(maxWidth: 380, alignment: .leading) + } + Spacer(minLength: 8) + controls + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + } + private var header: some View { ViewThatFits(in: .horizontal) { HStack(alignment: .center, spacing: 18) { titleBlock Spacer(minLength: 12) - stats + stats() controls } VStack(alignment: .leading, spacing: 10) { titleBlock HStack(alignment: .center, spacing: 14) { - stats + stats() Spacer(minLength: 8) controls } @@ -68,13 +90,13 @@ struct ContentView: View { } } - private var stats: some View { - HStack(spacing: 16) { - stat("Sorted", "\(model.sorted) / \(model.total)") - stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–") - stat("Elapsed", String(format: "%.1f s", model.elapsed)) - stat("ms / photo", millisecondsPerPhoto) - stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–") + private func stats(size: CGFloat = 20) -> some View { + HStack(spacing: size < 20 ? 12 : 16) { + stat("Sorted", "\(model.sorted) / \(model.total)", size: size) + stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–", size: size) + stat("Elapsed", String(format: "%.1f s", model.elapsed), size: size) + stat("ms / photo", millisecondsPerPhoto, size: size) + stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–", size: size) } } @@ -85,11 +107,11 @@ struct ContentView: View { return model.medianMilliseconds.map { String(format: "%.1f", $0) } ?? "–" } - private func stat(_ title: String, _ value: String) -> some View { + private func stat(_ title: String, _ value: String, size: CGFloat) -> some View { VStack(alignment: .trailing, spacing: 2) { Text(title.uppercased()).font(.caption2.weight(.semibold)).foregroundStyle(.secondary) .lineLimit(1).fixedSize() - Text(value).font(.system(size: 20, weight: .semibold, design: .rounded)).monospacedDigit() + Text(value).font(.system(size: size, weight: .semibold, design: .rounded)).monospacedDigit() .contentTransition(.numericText()).fixedSize() } } @@ -119,14 +141,10 @@ struct ContentView: View { } } - private var board: some View { - GeometryReader { proxy in - // Narrow windows stack the current photo above the chart so the chart gets the full width. - if proxy.size.width < 980 { - VStack(alignment: .leading, spacing: 12) { - compactNowSorting.frame(height: min(200, proxy.size.height * 0.28)) - chart(labelWidth: 150) - } + private func board(narrow: Bool) -> some View { + Group { + if narrow { + chart(labelWidth: 150) } else { HStack(alignment: .top, spacing: 18) { nowSorting.frame(width: 250) @@ -135,9 +153,6 @@ struct ContentView: View { } } .padding(14) - .coordinateSpace(name: "board") - .onPreferenceChange(FramesKey.self) { frames = $0 } - .overlay(alignment: .topLeading) { flying } } /// Photos travelling from the Now Sorting panel to their tile; they shrink to tile size on arrival. @@ -211,22 +226,6 @@ struct ContentView: View { } } - private var compactNowSorting: some View { - HStack(alignment: .top, spacing: 14) { - photo - VStack(alignment: .leading, spacing: 8) { - Text("NOW SORTING · \(model.remaining) left").font(.caption.weight(.semibold)) - .foregroundStyle(.secondary) - topFive - Spacer(minLength: 0) - Text("Labels are the 37 breed names, typed once: \"a photo of a {breed}, a type of pet.\"") - .font(.caption).foregroundStyle(.secondary) - } - .frame(maxWidth: 420, alignment: .leading) - Spacer(minLength: 0) - } - } - private func topRow(_ breed: String, share: Float, first: Bool, gold: String) -> some View { HStack(spacing: 6) { Text(breed).font(first ? .callout.weight(.bold) : .caption).lineLimit(1) From 4a2c9a9fd81c94c1c0bd06b406f51b9039a4f7bd Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:21:25 -0400 Subject: [PATCH 08/14] ImageSortDemo: Turbo only; photos/s and ms per photo; lighter terminal log Removes Show mode, the mode picker, and Pace. Stats show photos per second and wall-clock ms per photo; the caption explains the four photos processed in parallel. The terminal log prints one photo per update with a +N more count, so printing does not slow the run under screen recording. --- Sources/ImageSortDemo/ContentView.swift | 30 ++----- Sources/ImageSortDemo/ImageSortModel.swift | 97 +++++----------------- Sources/ImageSortDemo/README.md | 5 +- 3 files changed, 29 insertions(+), 103 deletions(-) diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index eed7489..8d01adb 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -93,18 +93,16 @@ struct ContentView: View { private func stats(size: CGFloat = 20) -> some View { HStack(spacing: size < 20 ? 12 : 16) { stat("Sorted", "\(model.sorted) / \(model.total)", size: size) - stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–", size: size) stat("Elapsed", String(format: "%.1f s", model.elapsed), size: size) - stat("ms / photo", millisecondsPerPhoto, size: size) + stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–", size: size) + stat("ms per photo", millisecondsPerPhoto, size: size) stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–", size: size) } } - /// Show: one model call, pre- and post-processing included. Turbo: wall time per photo with calls overlapping. + /// Wall time per photo with several model calls overlapping. private var millisecondsPerPhoto: String { - guard model.sorted > 0 else { return "–" } - if model.mode == .turbo { return String(format: "%.1f", 1000 / model.photosPerSecond) } - return model.medianMilliseconds.map { String(format: "%.1f", $0) } ?? "–" + model.sorted > 0 ? String(format: "%.1f", 1000 / model.photosPerSecond) : "–" } private func stat(_ title: String, _ value: String, size: CGFloat) -> some View { @@ -125,19 +123,8 @@ struct ContentView: View { .buttonStyle(.borderedProminent) Button("Reset") { model.reset() }.disabled(model.phase == .running) } - Picker("Mode", selection: $model.mode) { - ForEach(ImageSortModel.Mode.allCases) { Text($0.rawValue).tag($0) } - } - .pickerStyle(.segmented).labelsHidden().frame(width: 150) - if model.mode == .show { - HStack(spacing: 6) { - Text("Pace").font(.caption).fixedSize() - Slider(value: $model.pace, in: 2...30).frame(width: 90) - Text(String(format: "%.0f/s", model.pace)).font(.caption).monospacedDigit().fixedSize() - } - } else { - Text("\(ImageSortModel.turboInFlight) calls in flight").font(.caption).foregroundStyle(.secondary) - } + Text("\(ImageSortModel.turboInFlight) photos processed in parallel").font(.caption).foregroundStyle( + .secondary) } } @@ -265,11 +252,6 @@ struct ContentView: View { Image(decorative: image, scale: 1).resizable().interpolation(.medium) .frame(width: width * scale, height: height * scale) } - if let slot = model.lastSlot { - RoundedRectangle(cornerRadius: 3).stroke(Color.yellow, lineWidth: 2) - .frame(width: slot.width * scale + 6, height: slot.height * scale + 6) - .offset(x: slot.minX * scale - 3, y: slot.minY * scale - 3) - } } .frame(width: width * scale, height: height * scale, alignment: .topLeading) .reportFrame("chart") diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index df30dca..3e6cfed 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -16,14 +16,6 @@ final class ImageSortModel: ObservableObject { case failed(String) } - enum Mode: String, CaseIterable, Identifiable { - /// One photo at a time, with its top-5 breeds. - case show = "Show" - /// Several calls in flight, as fast as the model goes. - case turbo = "Turbo" - var id: String { rawValue } - } - struct Placed: Identifiable, Sendable { let item: PetItem let result: ImageSorter.Result @@ -33,9 +25,6 @@ final class ImageSortModel: ObservableObject { } @Published private(set) var phase: Phase = .loading("Starting…") - @Published var mode: Mode = .show - /// Photos per second in Show mode. - @Published var pace: Double = 8 @Published private(set) var remaining = 0 @Published private(set) var sorted = 0 @Published private(set) var correct = 0 @@ -45,8 +34,6 @@ final class ImageSortModel: ObservableObject { /// The photo just sorted, shown large with its top-5 breeds. @Published private(set) var current: Placed? @Published private(set) var currentImage: CGImage? - /// Where the latest tile landed, for the highlight in Show mode. - @Published private(set) var lastSlot: CGRect? /// Photos travelling from the Now Sorting panel to their chart slot. @Published private(set) var flights: [Flight] = [] @@ -59,7 +46,7 @@ final class ImageSortModel: ObservableObject { var arrived = false } - static let travel = 0.55 + static let travel = 0.4 static let turboFlightsPerFlush = 2 private var flightCounter = 0 @@ -84,7 +71,6 @@ final class ImageSortModel: ObservableObject { private var modelMilliseconds: [Double] = [] private var runStart: Date? private var elapsedBeforePause: Double = 0 - private var shownInShow = 0 private var preparing = false init() { @@ -111,16 +97,9 @@ final class ImageSortModel: ObservableObject { _ = try await sorter?.sort(items[0]) reset() print("ready at \(Date().timeIntervalSince1970)") - // IMAGE_SORT_WAIT=1 keeps the chart empty until Start (Space). IMAGE_SORT_AUTOPLAY=N sorts N photos in - // Show mode and then switches to Turbo; IMAGE_SORT_AUTOSTART=show|turbo starts in that mode. - if environment["IMAGE_SORT_WAIT"] == "1" { - return - } else if environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init) != nil { + // IMAGE_SORT_WAIT=1 keeps the chart empty until Start (Space). + if environment["IMAGE_SORT_WAIT"] != "1" { try? await Task.sleep(for: .seconds(1.5)) - mode = .show - toggleRun() - } else if let start = environment["IMAGE_SORT_AUTOSTART"], let mode = Mode(rawValue: start.capitalized) { - self.mode = mode toggleRun() } } catch { @@ -167,14 +146,12 @@ final class ImageSortModel: ObservableObject { counts = [:] current = nil currentImage = nil - lastSlot = nil flights = [] sorted = 0 correct = 0 elapsed = 0 elapsedBeforePause = 0 modelMilliseconds = [] - shownInShow = 0 phase = .ready } @@ -197,11 +174,7 @@ final class ImageSortModel: ObservableObject { private func run() async { guard let sorter else { return } while !Task.isCancelled, !queue.isEmpty { - if mode == .turbo { - await runTurbo(sorter) - } else { - await runShow(sorter) - } + await runTurbo(sorter) } if !Task.isCancelled, queue.isEmpty { tick() @@ -209,37 +182,10 @@ final class ImageSortModel: ObservableObject { print( "finished \(sorted) photos in \(String(format: "%.2f", elapsed)) s " + "(\(String(format: "%.0f", photosPerSecond)) photos/s), " - + "correct \(String(format: "%.1f", (accuracy ?? 0) * 100))% (\(mode.rawValue))") + + "correct \(String(format: "%.1f", (accuracy ?? 0) * 100))%") } } - private func runShow(_ sorter: ImageSorter) async { - for await placed in Self.stream(sorter, items: queue, inFlight: 2) { - if Task.isCancelled || mode != .show { break } - current = placed - currentImage = Self.thumbnail(placed.item.file, size: 480) - let breed = placed.result.breed - let row = breeds.firstIndex(of: breed) ?? 0 - let inAir = flights.filter { chart?.row(of: $0.slot) == row }.count - if let slot = chart?.slot(row: row, index: (counts[breed] ?? 0) + inAir) { - fly(image: placed.tile ?? currentImage, to: slot, wrong: !placed.matchesGold) { [weak self] in - self?.land([placed], highlight: true, updateCurrent: false) - } - } else { - land([placed], highlight: true, updateCurrent: false) - } - shownInShow += 1 - if let count = environment["IMAGE_SORT_AUTOPLAY"].flatMap(Int.init), shownInShow >= count { - mode = .turbo - } - let started = Date() - try? await Task.sleep(for: .seconds(1 / pace)) - if Date().timeIntervalSince(started) < 1 / pace { break } - } - // Let photos already in the air land before Turbo takes over the queue. - while !Task.isCancelled, !flights.isEmpty { try? await Task.sleep(for: .milliseconds(20)) } - } - private func runTurbo(_ sorter: ImageSorter) async { var pending: [Placed] = [] var lastFlush = Date() @@ -250,28 +196,27 @@ final class ImageSortModel: ObservableObject { pending.removeAll(keepingCapacity: true) lastFlush = Date() } - if Task.isCancelled || mode != .turbo { break } + if Task.isCancelled { break } } flush(pending) } - /// Lands a Turbo batch at once; a couple of its photos also fly so the motion stays visible. + /// Lands a batch at once; a couple of its photos also fly so the motion stays visible. private func flush(_ batch: [Placed]) { - let slots = land(batch, highlight: false, updateCurrent: true) + let slots = land(batch) for (placed, slot) in slots.suffix(Self.turboFlightsPerFlush) where flights.count < 24 { - fly(image: placed.tile, to: slot, wrong: !placed.matchesGold, decorative: true) {} + fly(image: placed.tile, to: slot, wrong: !placed.matchesGold) } } - /// Animates `image` from the Now Sorting panel to `slot`, then runs `arrival`. + /// Animates `image` from the Now Sorting panel to `slot`; the tile is already drawn there. private func fly( - image: CGImage?, to slot: CGRect, wrong: Bool, decorative: Bool = false, - arrival: @escaping @MainActor () -> Void + image: CGImage?, to slot: CGRect, wrong: Bool ) { flightCounter += 1 let id = flightCounter flights.append(Flight(id: id, image: image, slot: slot, wrong: wrong)) - let travel = decorative ? Self.travel * 0.7 : Self.travel + let travel = Self.travel Task { @MainActor [weak self] in try? await Task.sleep(for: .milliseconds(20)) withAnimation(.easeIn(duration: travel)) { @@ -279,7 +224,6 @@ final class ImageSortModel: ObservableObject { } try? await Task.sleep(for: .seconds(travel)) self?.flights.removeAll { $0.id == id } - arrival() } } @@ -327,7 +271,7 @@ final class ImageSortModel: ObservableObject { /// Draws the batch into the chart and updates the statistics; returns each photo's slot. @discardableResult - private func land(_ incoming: [Placed], highlight: Bool, updateCurrent: Bool) -> [(Placed, CGRect)] { + private func land(_ incoming: [Placed]) -> [(Placed, CGRect)] { let batch = incoming.filter { self.landed.insert($0.id).inserted } guard let last = batch.last, let chart else { return [] } var updated = counts @@ -343,11 +287,8 @@ final class ImageSortModel: ObservableObject { } counts = updated chartImage = chart.snapshot() - lastSlot = highlight ? slots.last?.1 : nil - if updateCurrent { - current = last - currentImage = Self.thumbnail(last.item.file, size: 480) - } + current = last + currentImage = Self.thumbnail(last.item.file, size: 480) let ids = Set(batch.map(\.id)) queue.removeAll { ids.contains($0.id) } remaining = queue.count @@ -360,8 +301,9 @@ final class ImageSortModel: ObservableObject { private func log(_ batch: [Placed]) { let (cyan, yellow, red, green, dim, reset) = ("\u{1B}[1;36m", "\u{1B}[33m", "\u{1B}[31m", "\u{1B}[32m", "\u{1B}[2m", "\u{1B}[0m") + // One photo per update keeps the terminal readable (and cheap) at hundreds of photos per second. var lines = "" - for (offset, placed) in batch.enumerated() { + for (offset, placed) in batch.enumerated().suffix(1) { let number = sorted - batch.count + offset + 1 let mark = placed.matchesGold ? "\(green)✓\(reset)" : "\(red)✗ label: \(placed.item.breed)\(reset)" lines += "\(cyan)▶ #\(number) photo \(placed.item.id).jpg\(reset)\n" @@ -369,7 +311,10 @@ final class ImageSortModel: ObservableObject { " \(yellow)→ \(placed.result.breed)\(reset) · \(Int(placed.result.share * 100))% of 37 · " + "\(red)model call \(String(format: "%.1f", placed.result.milliseconds)) ms\(reset) \(mark)\n" } - lines += "\(dim) sorted \(sorted)/\(total) · \(String(format: "%.1f", elapsed)) s\(reset)\n" + let wrong = batch.filter { !$0.matchesGold }.count + lines += + "\(dim) +\(batch.count - 1) more (\(wrong) wrong) · sorted \(sorted)/\(total) · " + + "\(String(format: "%.1f", elapsed)) s\(reset)\n" print(lines, terminator: "") } diff --git a/Sources/ImageSortDemo/README.md b/Sources/ImageSortDemo/README.md index 7540b81..91f9f39 100644 --- a/Sources/ImageSortDemo/README.md +++ b/Sources/ImageSortDemo/README.md @@ -4,11 +4,10 @@ Sorts Oxford-IIIT Pets photos into 37 breeds with SigLIP 2 (base, 256 px) on Cor gets is each breed's name, in the prompt `a photo of a {breed}, a type of pet.`; nothing is trained on these photos. ```bash -SIGLIP2_MODEL_DIR=/path/to/siglip2-base-patch16-256 IMAGE_SORT_AUTOPLAY=12 swift run -c release ImageSortDemo +SIGLIP2_MODEL_DIR=/path/to/siglip2-base-patch16-256 swift run -c release ImageSortDemo ``` -- `IMAGE_SORT_AUTOPLAY=N` flies N photos in Show mode, then switches to Turbo; `IMAGE_SORT_AUTOSTART=show|turbo` - starts a run in that mode. Without either, press Start. +- Four model calls stay in flight; a couple of photos per update fly from the current-photo panel to their tile. - `IMAGE_SORT_COUNT` sets the sample size (default 1,000; the test split has 3,669, and a cache that also holds the train split allows up to 7,349). `IMAGE_SORT_WAIT=1` waits for Start (Space); `IMAGE_SORT_LOG=1` prints each decision. From 1d3cf20d6e5c23c0234bf1f1718e9c8717af0081 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:22:28 -0400 Subject: [PATCH 09/14] ImageSortDemo: drop the parallel-calls caption --- Sources/ImageSortDemo/ContentView.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index 8d01adb..a5842ad 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -123,8 +123,6 @@ struct ContentView: View { .buttonStyle(.borderedProminent) Button("Reset") { model.reset() }.disabled(model.phase == .running) } - Text("\(ImageSortModel.turboInFlight) photos processed in parallel").font(.caption).foregroundStyle( - .secondary) } } From 418c78ab75fdb72bac652ca301316ead5a48a281 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:23:13 -0400 Subject: [PATCH 10/14] ImageSortDemo: show the image encoder's own time on the Neural Engine SigLIP2Manager.imageEncoderMilliseconds times the image model alone (median of 30 sequential calls after 5 warmups); the demo measures it at launch (5.2 ms on an M5 Pro) and shows it next to the live photos/s and ms per photo. --- Sources/FluidUse/SigLIP2/SigLIP2Manager.swift | 16 ++++++++++++++++ Sources/ImageSort/ImageSorter.swift | 3 +++ Sources/ImageSortDemo/ContentView.swift | 2 ++ Sources/ImageSortDemo/ImageSortModel.swift | 3 +++ 4 files changed, 24 insertions(+) diff --git a/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift b/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift index 1f405e3..c2ab2eb 100644 --- a/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift +++ b/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift @@ -83,6 +83,22 @@ public final class SigLIP2Manager: Sendable { return try Self.vector(output, name: "image_embeds") } + /// Median wall time of the image encoder alone (no decoding or resizing), one call at a time after `warmup` + /// calls. Measures the compute units the manager was loaded with. + public func imageEncoderMilliseconds(iterations: Int = 30, warmup: Int = 5) async throws -> Double { + let size = NSNumber(value: config.imageSize) + let input = try MLMultiArray(shape: [1, 3, size, size], dataType: .float32) + input.dataPointer.initializeMemory(as: Float.self, repeating: 0, count: input.count) + let features = try MLDictionaryFeatureProvider(dictionary: ["pixel_values": MLFeatureValue(multiArray: input)]) + var times: [Double] = [] + for index in 0..<(warmup + iterations) { + let start = DispatchTime.now().uptimeNanoseconds + _ = try await imageModel.prediction(from: features) + if index >= warmup { times.append(Double(DispatchTime.now().uptimeNanoseconds - start) / 1e6) } + } + return times.sorted()[times.count / 2] + } + /// Scores `image` against label embeddings from `embed(labels:)`. public func classify(image: CGImage, labels: [String], labelEmbeddings: [[Float]]) async throws -> SigLIP2Answer { guard labels.count == labelEmbeddings.count, !labels.isEmpty else { diff --git a/Sources/ImageSort/ImageSorter.swift b/Sources/ImageSort/ImageSorter.swift index ee37890..c07fcc2 100644 --- a/Sources/ImageSort/ImageSorter.swift +++ b/Sources/ImageSort/ImageSorter.swift @@ -38,6 +38,9 @@ public final class ImageSorter: Sendable { public var modelName: String { manager.config.name } + /// Median time of the image encoder alone on the Neural Engine, one call at a time. + public func encoderMilliseconds() async throws -> Double { try await manager.imageEncoderMilliseconds() } + public func sort(_ item: PetItem) async throws -> Result { let image = try Self.decode(item.file) let start = DispatchTime.now().uptimeNanoseconds diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index a5842ad..f381d6a 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -96,6 +96,8 @@ struct ContentView: View { stat("Elapsed", String(format: "%.1f s", model.elapsed), size: size) stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–", size: size) stat("ms per photo", millisecondsPerPhoto, size: size) + stat( + "Neural Engine ms", model.encoderMilliseconds.map { String(format: "%.1f", $0) } ?? "–", size: size) stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–", size: size) } } diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index 3e6cfed..fd301a7 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -29,6 +29,8 @@ final class ImageSortModel: ObservableObject { @Published private(set) var sorted = 0 @Published private(set) var correct = 0 @Published private(set) var elapsed: Double = 0 + /// Image encoder alone, measured once at launch. + @Published private(set) var encoderMilliseconds: Double? @Published private(set) var counts: [String: Int] = [:] @Published private(set) var chartImage: CGImage? /// The photo just sorted, shown large with its top-5 breeds. @@ -95,6 +97,7 @@ final class ImageSortModel: ObservableObject { phase = .loading("Loading SigLIP 2 and embedding 37 breed names…") sorter = try await ImageSorter.load() _ = try await sorter?.sort(items[0]) + encoderMilliseconds = try await sorter?.encoderMilliseconds() reset() print("ready at \(Date().timeIntervalSince1970)") // IMAGE_SORT_WAIT=1 keeps the chart empty until Start (Space). From 60f93d0df6799980d01b9fe2dbc62f9f14136eef Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:23:53 -0400 Subject: [PATCH 11/14] ImageSortDemo: Neural Engine ms reads 0.0 until the run starts --- Sources/ImageSortDemo/ContentView.swift | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index f381d6a..9338b35 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -96,12 +96,17 @@ struct ContentView: View { stat("Elapsed", String(format: "%.1f s", model.elapsed), size: size) stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–", size: size) stat("ms per photo", millisecondsPerPhoto, size: size) - stat( - "Neural Engine ms", model.encoderMilliseconds.map { String(format: "%.1f", $0) } ?? "–", size: size) + stat("Neural Engine ms", neuralEngineMilliseconds, size: size) stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–", size: size) } } + /// The launch measurement, revealed once photos start landing (0 before, and again after Reset). + private var neuralEngineMilliseconds: String { + guard model.sorted > 0, let value = model.encoderMilliseconds else { return "0.0" } + return String(format: "%.1f", value) + } + /// Wall time per photo with several model calls overlapping. private var millisecondsPerPhoto: String { model.sorted > 0 ? String(format: "%.1f", 1000 / model.photosPerSecond) : "–" From 7afd66effb0e8631df72ed7bee4ffb4994ce1f55 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:26:50 -0400 Subject: [PATCH 12/14] ImageSortDemo: live Neural Engine time; equal-width stat tiles Neural Engine ms is measured live from each Core ML call's start and end: end minus the later of its start and the previous call's end, which removes queueing when calls overlap (median of the last 200). Stats are equal-width centered tiles that start at zero; buttons sit beside the title in narrow windows. --- Sources/FluidUse/SigLIP2/SigLIP2Manager.swift | 21 ++++++++-- Sources/ImageSort/ImageSorter.swift | 9 +++- Sources/ImageSortDemo/ContentView.swift | 41 ++++++++++--------- Sources/ImageSortDemo/ImageSortModel.swift | 20 +++++++-- 4 files changed, 63 insertions(+), 28 deletions(-) diff --git a/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift b/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift index c2ab2eb..6d47353 100644 --- a/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift +++ b/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift @@ -70,17 +70,30 @@ public final class SigLIP2Manager: Sendable { return embeddings } + /// Image embedding plus when the Core ML call began and ended (`DispatchTime` uptime nanoseconds). + public struct TimedEmbedding: Sendable { + public let embedding: [Float] + public let predictionStart: UInt64 + public let predictionEnd: UInt64 + } + /// L2-normalized image embedding. - public func embed(image: CGImage) async throws -> [Float] { + public func embed(image: CGImage) async throws -> [Float] { try await embedTimed(image: image).embedding } + + /// L2-normalized image embedding, with the timing of the model call alone (no decoding or resizing). + public func embedTimed(image: CGImage) async throws -> TimedEmbedding { let pixels = try SigLIP2ImagePreprocessor.pixels(from: image, config: config) let size = NSNumber(value: config.imageSize) let input = try MLMultiArray(shape: [1, 3, size, size], dataType: .float32) pixels.withUnsafeBufferPointer { source in input.dataPointer.assumingMemoryBound(to: Float.self).update(from: source.baseAddress!, count: pixels.count) } - let output = try await imageModel.prediction( - from: MLDictionaryFeatureProvider(dictionary: ["pixel_values": MLFeatureValue(multiArray: input)])) - return try Self.vector(output, name: "image_embeds") + let features = try MLDictionaryFeatureProvider(dictionary: ["pixel_values": MLFeatureValue(multiArray: input)]) + let start = DispatchTime.now().uptimeNanoseconds + let output = try await imageModel.prediction(from: features) + let end = DispatchTime.now().uptimeNanoseconds + return TimedEmbedding( + embedding: try Self.vector(output, name: "image_embeds"), predictionStart: start, predictionEnd: end) } /// Median wall time of the image encoder alone (no decoding or resizing), one call at a time after `warmup` diff --git a/Sources/ImageSort/ImageSorter.swift b/Sources/ImageSort/ImageSorter.swift index c07fcc2..785bbca 100644 --- a/Sources/ImageSort/ImageSorter.swift +++ b/Sources/ImageSort/ImageSorter.swift @@ -14,6 +14,9 @@ public final class ImageSorter: Sendable { /// The five most likely breeds with their shares, best first. public let top: [(breed: String, share: Float)] public let milliseconds: Double + /// When the Core ML call itself began and ended (`DispatchTime` uptime nanoseconds). + public let predictionStart: UInt64 + public let predictionEnd: UInt64 } public let breeds: [String] @@ -44,7 +47,8 @@ public final class ImageSorter: Sendable { public func sort(_ item: PetItem) async throws -> Result { let image = try Self.decode(item.file) let start = DispatchTime.now().uptimeNanoseconds - let answer = try await manager.classify(image: image, labels: breeds, labelEmbeddings: embeddings) + let timed = try await manager.embedTimed(image: image) + let answer = manager.score(imageEmbedding: timed.embedding, labels: breeds, labelEmbeddings: embeddings) let milliseconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e6 let scale = manager.config.logitScale let best = answer.similarities[answer.selectedIndex] @@ -55,7 +59,8 @@ public final class ImageSorter: Sendable { } return Result( breed: answer.selectedLabel, probability: answer.probabilities[answer.selectedIndex], share: 1 / total, - top: ranked, milliseconds: milliseconds) + top: ranked, milliseconds: milliseconds, predictionStart: timed.predictionStart, + predictionEnd: timed.predictionEnd) } public static func decode(_ file: URL) throws -> CGImage { diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index 9338b35..39adf66 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -49,13 +49,15 @@ struct ContentView: View { private var compactTop: some View { HStack(alignment: .top, spacing: 16) { photo.frame(width: 176, height: 176) - VStack(alignment: .leading, spacing: 10) { - titleBlock - stats(size: 17) - topFive.frame(maxWidth: 380, alignment: .leading) + VStack(alignment: .leading, spacing: 12) { + HStack(alignment: .top) { + titleBlock + Spacer(minLength: 8) + controls + } + stats(size: 18) + topFive.frame(maxWidth: 420, alignment: .leading) } - Spacer(minLength: 8) - controls } .padding(.horizontal, 16) .padding(.vertical, 12) @@ -91,34 +93,35 @@ struct ContentView: View { } private func stats(size: CGFloat = 20) -> some View { - HStack(spacing: size < 20 ? 12 : 16) { - stat("Sorted", "\(model.sorted) / \(model.total)", size: size) - stat("Elapsed", String(format: "%.1f s", model.elapsed), size: size) - stat("Photos / s", model.sorted > 0 ? String(format: "%.0f", model.photosPerSecond) : "–", size: size) - stat("ms per photo", millisecondsPerPhoto, size: size) - stat("Neural Engine ms", neuralEngineMilliseconds, size: size) - stat("Correct breed", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–", size: size) + HStack(spacing: 0) { + stat("Sorted", "\(model.sorted)", size: size) + stat("Elapsed time", String(format: "%.1f s", model.elapsed), size: size) + stat("Photos/s", String(format: "%.0f", model.sorted > 0 ? model.photosPerSecond : 0), size: size) + stat("ms/photo", millisecondsPerPhoto, size: size) + stat("Neural Engine", neuralEngineMilliseconds + " ms", size: size) + stat("Accuracy", String(format: "%.1f%%", (model.accuracy ?? 0) * 100), size: size) } } - /// The launch measurement, revealed once photos start landing (0 before, and again after Reset). + /// Live Neural Engine time per image during the run; 0.0 before Start and after Reset. private var neuralEngineMilliseconds: String { - guard model.sorted > 0, let value = model.encoderMilliseconds else { return "0.0" } - return String(format: "%.1f", value) + model.liveEncoderMilliseconds.map { String(format: "%.1f", $0) } ?? "0.0" } /// Wall time per photo with several model calls overlapping. private var millisecondsPerPhoto: String { - model.sorted > 0 ? String(format: "%.1f", 1000 / model.photosPerSecond) : "–" + model.sorted > 0 ? String(format: "%.1f", 1000 / model.photosPerSecond) : "0.0" } + /// Equal-width tile, label over value, both centered, so columns stay put as numbers change. private func stat(_ title: String, _ value: String, size: CGFloat) -> some View { - VStack(alignment: .trailing, spacing: 2) { + VStack(spacing: 3) { Text(title.uppercased()).font(.caption2.weight(.semibold)).foregroundStyle(.secondary) .lineLimit(1).fixedSize() Text(value).font(.system(size: size, weight: .semibold, design: .rounded)).monospacedDigit() - .contentTransition(.numericText()).fixedSize() + .contentTransition(.numericText()).lineLimit(1).fixedSize() } + .frame(width: size < 20 ? 104 : 120) } private var controls: some View { diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index fd301a7..b901526 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -29,8 +29,11 @@ final class ImageSortModel: ObservableObject { @Published private(set) var sorted = 0 @Published private(set) var correct = 0 @Published private(set) var elapsed: Double = 0 - /// Image encoder alone, measured once at launch. - @Published private(set) var encoderMilliseconds: Double? + /// Live Neural Engine time per image: each call's end minus the later of its start and the previous call's end + /// (calls run one at a time, so this removes queueing), median of the last 200. + @Published private(set) var liveEncoderMilliseconds: Double? + private var serviceTimes: [Double] = [] + private var lastPredictionEnd: UInt64 = 0 @Published private(set) var counts: [String: Int] = [:] @Published private(set) var chartImage: CGImage? /// The photo just sorted, shown large with its top-5 breeds. @@ -97,7 +100,6 @@ final class ImageSortModel: ObservableObject { phase = .loading("Loading SigLIP 2 and embedding 37 breed names…") sorter = try await ImageSorter.load() _ = try await sorter?.sort(items[0]) - encoderMilliseconds = try await sorter?.encoderMilliseconds() reset() print("ready at \(Date().timeIntervalSince1970)") // IMAGE_SORT_WAIT=1 keeps the chart empty until Start (Space). @@ -155,6 +157,9 @@ final class ImageSortModel: ObservableObject { elapsed = 0 elapsedBeforePause = 0 modelMilliseconds = [] + serviceTimes = [] + lastPredictionEnd = 0 + liveEncoderMilliseconds = nil phase = .ready } @@ -289,6 +294,15 @@ final class ImageSortModel: ObservableObject { modelMilliseconds.append(placed.result.milliseconds) } counts = updated + for placed in batch.sorted(by: { $0.result.predictionEnd < $1.result.predictionEnd }) { + let begin = max(placed.result.predictionStart, lastPredictionEnd) + if placed.result.predictionEnd > begin { + serviceTimes.append(Double(placed.result.predictionEnd - begin) / 1e6) + } + lastPredictionEnd = max(lastPredictionEnd, placed.result.predictionEnd) + } + if serviceTimes.count > 200 { serviceTimes.removeFirst(serviceTimes.count - 200) } + if !serviceTimes.isEmpty { liveEncoderMilliseconds = serviceTimes.sorted()[serviceTimes.count / 2] } chartImage = chart.snapshot() current = last currentImage = Self.thumbnail(last.item.file, size: 480) From 1a51dc454512691bb0ab1d5f093977026f871e79 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:29:15 -0400 Subject: [PATCH 13/14] ImageSortDemo: last photo flies home at the end; drop the Neural Engine tile --- Sources/FluidUse/SigLIP2/SigLIP2Manager.swift | 16 ----------- Sources/ImageSort/ImageSorter.swift | 3 -- Sources/ImageSortDemo/ContentView.swift | 6 ---- Sources/ImageSortDemo/ImageSortModel.swift | 28 ++++++++----------- 4 files changed, 11 insertions(+), 42 deletions(-) diff --git a/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift b/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift index 6d47353..9d8f9a5 100644 --- a/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift +++ b/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift @@ -96,22 +96,6 @@ public final class SigLIP2Manager: Sendable { embedding: try Self.vector(output, name: "image_embeds"), predictionStart: start, predictionEnd: end) } - /// Median wall time of the image encoder alone (no decoding or resizing), one call at a time after `warmup` - /// calls. Measures the compute units the manager was loaded with. - public func imageEncoderMilliseconds(iterations: Int = 30, warmup: Int = 5) async throws -> Double { - let size = NSNumber(value: config.imageSize) - let input = try MLMultiArray(shape: [1, 3, size, size], dataType: .float32) - input.dataPointer.initializeMemory(as: Float.self, repeating: 0, count: input.count) - let features = try MLDictionaryFeatureProvider(dictionary: ["pixel_values": MLFeatureValue(multiArray: input)]) - var times: [Double] = [] - for index in 0..<(warmup + iterations) { - let start = DispatchTime.now().uptimeNanoseconds - _ = try await imageModel.prediction(from: features) - if index >= warmup { times.append(Double(DispatchTime.now().uptimeNanoseconds - start) / 1e6) } - } - return times.sorted()[times.count / 2] - } - /// Scores `image` against label embeddings from `embed(labels:)`. public func classify(image: CGImage, labels: [String], labelEmbeddings: [[Float]]) async throws -> SigLIP2Answer { guard labels.count == labelEmbeddings.count, !labels.isEmpty else { diff --git a/Sources/ImageSort/ImageSorter.swift b/Sources/ImageSort/ImageSorter.swift index 785bbca..1dfa7be 100644 --- a/Sources/ImageSort/ImageSorter.swift +++ b/Sources/ImageSort/ImageSorter.swift @@ -41,9 +41,6 @@ public final class ImageSorter: Sendable { public var modelName: String { manager.config.name } - /// Median time of the image encoder alone on the Neural Engine, one call at a time. - public func encoderMilliseconds() async throws -> Double { try await manager.imageEncoderMilliseconds() } - public func sort(_ item: PetItem) async throws -> Result { let image = try Self.decode(item.file) let start = DispatchTime.now().uptimeNanoseconds diff --git a/Sources/ImageSortDemo/ContentView.swift b/Sources/ImageSortDemo/ContentView.swift index 39adf66..1cd3163 100644 --- a/Sources/ImageSortDemo/ContentView.swift +++ b/Sources/ImageSortDemo/ContentView.swift @@ -98,16 +98,10 @@ struct ContentView: View { stat("Elapsed time", String(format: "%.1f s", model.elapsed), size: size) stat("Photos/s", String(format: "%.0f", model.sorted > 0 ? model.photosPerSecond : 0), size: size) stat("ms/photo", millisecondsPerPhoto, size: size) - stat("Neural Engine", neuralEngineMilliseconds + " ms", size: size) stat("Accuracy", String(format: "%.1f%%", (model.accuracy ?? 0) * 100), size: size) } } - /// Live Neural Engine time per image during the run; 0.0 before Start and after Reset. - private var neuralEngineMilliseconds: String { - model.liveEncoderMilliseconds.map { String(format: "%.1f", $0) } ?? "0.0" - } - /// Wall time per photo with several model calls overlapping. private var millisecondsPerPhoto: String { model.sorted > 0 ? String(format: "%.1f", 1000 / model.photosPerSecond) : "0.0" diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index b901526..bad60d0 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -29,11 +29,8 @@ final class ImageSortModel: ObservableObject { @Published private(set) var sorted = 0 @Published private(set) var correct = 0 @Published private(set) var elapsed: Double = 0 - /// Live Neural Engine time per image: each call's end minus the later of its start and the previous call's end - /// (calls run one at a time, so this removes queueing), median of the last 200. - @Published private(set) var liveEncoderMilliseconds: Double? - private var serviceTimes: [Double] = [] - private var lastPredictionEnd: UInt64 = 0 + /// Chart slot of the photo shown in the Now Sorting panel. + private var currentSlot: CGRect? @Published private(set) var counts: [String: Int] = [:] @Published private(set) var chartImage: CGImage? /// The photo just sorted, shown large with its top-5 breeds. @@ -151,15 +148,13 @@ final class ImageSortModel: ObservableObject { counts = [:] current = nil currentImage = nil + currentSlot = nil flights = [] sorted = 0 correct = 0 elapsed = 0 elapsedBeforePause = 0 modelMilliseconds = [] - serviceTimes = [] - lastPredictionEnd = 0 - liveEncoderMilliseconds = nil phase = .ready } @@ -187,6 +182,13 @@ final class ImageSortModel: ObservableObject { if !Task.isCancelled, queue.isEmpty { tick() phase = .finished + // The last photo leaves the panel for its tile too, leaving only the finished chart. + if let current, let slot = currentSlot { + fly(image: currentImage ?? current.tile, to: slot, wrong: !current.matchesGold) + } + current = nil + currentImage = nil + currentSlot = nil print( "finished \(sorted) photos in \(String(format: "%.2f", elapsed)) s " + "(\(String(format: "%.0f", photosPerSecond)) photos/s), " @@ -294,18 +296,10 @@ final class ImageSortModel: ObservableObject { modelMilliseconds.append(placed.result.milliseconds) } counts = updated - for placed in batch.sorted(by: { $0.result.predictionEnd < $1.result.predictionEnd }) { - let begin = max(placed.result.predictionStart, lastPredictionEnd) - if placed.result.predictionEnd > begin { - serviceTimes.append(Double(placed.result.predictionEnd - begin) / 1e6) - } - lastPredictionEnd = max(lastPredictionEnd, placed.result.predictionEnd) - } - if serviceTimes.count > 200 { serviceTimes.removeFirst(serviceTimes.count - 200) } - if !serviceTimes.isEmpty { liveEncoderMilliseconds = serviceTimes.sorted()[serviceTimes.count / 2] } chartImage = chart.snapshot() current = last currentImage = Self.thumbnail(last.item.file, size: 480) + currentSlot = slots.last { $0.0.id == last.id }?.1 let ids = Set(batch.map(\.id)) queue.removeAll { ids.contains($0.id) } remaining = queue.count From acf7913999c04712b66eb3f14e323ffbc59ed564 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 25 Sep 2026 23:46:24 -0400 Subject: [PATCH 14/14] SigLIP 2: download the published Core ML packages; test split by default SigLIP2ModelStore fetches FluidInference/siglip2-base-patch16-256-coreml at a pinned revision with SHA-256 checks; SIGLIP2_MODEL_DIR still loads a local conversion. ImageSortCheck scores the 3,669 test photos by default (--split=all for 7,349): 94.85% / 94.26% at 202 photos/s from the downloaded packages. VideoSortDemo moves to its own branch (its videos are local only). --- Package.swift | 1 - Sources/FluidUse/SigLIP2/SigLIP2Manager.swift | 17 ++ .../FluidUse/SigLIP2/SigLIP2ModelStore.swift | 84 ++++++ Sources/ImageSort/GridClassifier.swift | 94 ------ Sources/ImageSort/ImageSorter.swift | 12 +- Sources/ImageSort/PetsSample.swift | 9 +- Sources/ImageSortCheck/main.swift | 6 +- Sources/ImageSortDemo/ImageSortModel.swift | 7 +- Sources/ImageSortDemo/README.md | 28 +- Sources/VideoSortDemo/ContentView.swift | 285 ------------------ Sources/VideoSortDemo/README.md | 21 -- Sources/VideoSortDemo/VideoFrames.swift | 59 ---- Sources/VideoSortDemo/VideoSortDemoApp.swift | 28 -- Sources/VideoSortDemo/VideoSortModel.swift | 277 ----------------- 14 files changed, 138 insertions(+), 790 deletions(-) create mode 100644 Sources/FluidUse/SigLIP2/SigLIP2ModelStore.swift delete mode 100644 Sources/ImageSort/GridClassifier.swift delete mode 100644 Sources/VideoSortDemo/ContentView.swift delete mode 100644 Sources/VideoSortDemo/README.md delete mode 100644 Sources/VideoSortDemo/VideoFrames.swift delete mode 100644 Sources/VideoSortDemo/VideoSortDemoApp.swift delete mode 100644 Sources/VideoSortDemo/VideoSortModel.swift diff --git a/Package.swift b/Package.swift index 71e2d2a..9fc5e4f 100644 --- a/Package.swift +++ b/Package.swift @@ -53,7 +53,6 @@ let package = Package( .target(name: "ImageSort", dependencies: ["FluidUse"]), .executableTarget(name: "ImageSortCheck", dependencies: ["ImageSort", "FluidUse"]), .executableTarget(name: "ImageSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]), - .executableTarget(name: "VideoSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]), .testTarget( name: "FluidUseTests", dependencies: ["FluidUse", "LayaTetris"], resources: [.copy("Fixtures")] diff --git a/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift b/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift index 9d8f9a5..6d64488 100644 --- a/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift +++ b/Sources/FluidUse/SigLIP2/SigLIP2Manager.swift @@ -18,6 +18,23 @@ public final class SigLIP2Manager: Sendable { self.textModel = textModel } + /// Downloads (once, checksum-verified) and loads the published fp16 packages. + public static func load( + cacheDirectory: URL? = nil, computeUnits: MLComputeUnits = .cpuAndNeuralEngine, + progress: SigLIP2ModelStore.Progress? = nil + ) async throws -> SigLIP2Manager { + let directory = try await SigLIP2ModelStore.ensure(cacheDirectory: cacheDirectory, progress: progress) + return try await load(from: directory, computeUnits: computeUnits) + } + + /// Loads the manager from `SIGLIP2_MODEL_DIR` when set, otherwise from the published packages. + public static func loadDefault(progress: SigLIP2ModelStore.Progress? = nil) async throws -> SigLIP2Manager { + if let path = ProcessInfo.processInfo.environment["SIGLIP2_MODEL_DIR"], !path.isEmpty { + return try await load(from: URL(fileURLWithPath: path)) + } + return try await load(progress: progress) + } + /// Loads `config.json`, `tokenizer.json`, and the image and text packages (`.mlmodelc` preferred) from /// `directory`, as written by the mobius converter. public static func load( diff --git a/Sources/FluidUse/SigLIP2/SigLIP2ModelStore.swift b/Sources/FluidUse/SigLIP2/SigLIP2ModelStore.swift new file mode 100644 index 0000000..cabfe10 --- /dev/null +++ b/Sources/FluidUse/SigLIP2/SigLIP2ModelStore.swift @@ -0,0 +1,84 @@ +import CryptoKit +import Foundation + +/// Downloads the pinned SigLIP 2 Core ML packages from Hugging Face into the FluidUse cache. +public enum SigLIP2ModelStore { + public typealias Progress = @Sendable (_ file: String, _ bytes: Int64) -> Void + + public static let repository = "FluidInference/siglip2-base-patch16-256-coreml" + static let revision = "524a5a7d666f23002853831915cf1cc13734f73f" + + private struct Asset { + let path: String + let sha256: String + } + + private static let assets = [ + Asset(path: "config.json", sha256: "b5d7aaa84399aaa277f9cc00c6c401edc05d535f47f4feb1880e0e4ed2a1a8d4"), + Asset( + path: "siglip2-base-patch16-256-image-fp16.mlpackage/Data/com.apple.CoreML/model.mlmodel", + sha256: "702fdc0b557984e16e4bbcce4eec3568f08a7459bd3c27c770a487823bda808b"), + Asset( + path: "siglip2-base-patch16-256-image-fp16.mlpackage/Data/com.apple.CoreML/weights/weight.bin", + sha256: "da086438b60ada3566f8c91bd8a632462f28d344186e7f60cb5abd223d2e99a8"), + Asset( + path: "siglip2-base-patch16-256-image-fp16.mlpackage/Manifest.json", + sha256: "964a40aa63c2d201a30f0aeab68aef8abc95c854a34f4cae533051cd99996e72"), + Asset( + path: "siglip2-base-patch16-256-text-fp16.mlpackage/Data/com.apple.CoreML/model.mlmodel", + sha256: "69fce0f538fbe78c45b953e1cc396fb912d701c6f985ac825bdec92833aca452"), + Asset( + path: "siglip2-base-patch16-256-text-fp16.mlpackage/Data/com.apple.CoreML/weights/weight.bin", + sha256: "9a52dd8222973b6b4cdf55983689bb2fa18a27fcd20b83f8d8fb89de544530b5"), + Asset( + path: "siglip2-base-patch16-256-text-fp16.mlpackage/Manifest.json", + sha256: "a5dfacf23259261d32c5af26c5cfe4c41a44f0584739b2ef10e70c7f11c2a620"), + Asset( + path: "tokenizer_config.json", sha256: "9c8a03337138d3b5509e4c032f6863e769b7448750718372c907407d67f6a91b"), + Asset(path: "tokenizer.json", sha256: "caefd63119539a63be2d55ef3e05023fbb793948c4bda5bc0c366b42a382f903"), + ] + + /// Ensures the packages, tokenizer, and config exist and match their checksums; returns their directory. + public static func ensure(cacheDirectory: URL? = nil, progress: Progress? = nil) async throws -> URL { + let root = cacheDirectory ?? LayaModelStore.defaultCacheDirectory() + let directory = root.appendingPathComponent("siglip2-base-patch16-256-coreml") + let manager = FileManager.default + try manager.createDirectory(at: directory, withIntermediateDirectories: true) + for asset in assets { + let destination = directory.appendingPathComponent(asset.path) + if manager.fileExists(atPath: destination.path), try checksum(of: destination) == asset.sha256 { + continue + } + try manager.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + progress?(asset.path, 0) + let escaped = asset.path.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? asset.path + guard let url = URL(string: "https://huggingface.co/\(repository)/resolve/\(revision)/\(escaped)") else { + throw SigLIP2Error.invalidAsset("Invalid Hugging Face asset URL") + } + let (temporary, response) = try await URLSession.shared.download(from: url) + defer { try? manager.removeItem(at: temporary) } + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw SigLIP2Error.invalidAsset("Download failed for \(asset.path)") + } + let actual = try checksum(of: temporary) + guard actual == asset.sha256 else { + throw SigLIP2Error.invalidAsset( + "Checksum mismatch for \(asset.path): expected \(asset.sha256), got \(actual)") + } + let size = (try manager.attributesOfItem(atPath: temporary.path)[.size] as? NSNumber)?.int64Value ?? 0 + try LayaModelStore.installDownloadedFile(temporary, at: destination) + progress?(asset.path, size) + } + return directory + } + + private static func checksum(of file: URL) throws -> String { + let handle = try FileHandle(forReadingFrom: file) + defer { try? handle.close() } + var digest = SHA256() + while let chunk = try handle.read(upToCount: 1_048_576), !chunk.isEmpty { + digest.update(data: chunk) + } + return digest.finalize().map { String(format: "%02x", $0) }.joined() + } +} diff --git a/Sources/ImageSort/GridClassifier.swift b/Sources/ImageSort/GridClassifier.swift deleted file mode 100644 index e8d8abd..0000000 --- a/Sources/ImageSort/GridClassifier.swift +++ /dev/null @@ -1,94 +0,0 @@ -import CoreGraphics -import FluidUse -import Foundation - -/// Labels every cell of a `columns × rows` grid over a frame with SigLIP 2 on Core ML. -/// Cells are scored concurrently; one call handles one cell. -public final class GridClassifier: Sendable { - public struct Cell: Sendable { - public let label: Int - /// Softmax share of the chosen label among all labels. - public let share: Float - /// The five most likely labels with their shares, best first. - public let top: [Ranked] - } - - public struct Ranked: Sendable { - public let label: Int - public let share: Float - } - - public struct Grid: Sendable { - public let cells: [Cell] - public let milliseconds: Double - } - - public let labels: [String] - public let columns: Int - public let rows: Int - private let manager: SigLIP2Manager - private let embeddings: [[Float]] - - private init(manager: SigLIP2Manager, labels: [String], embeddings: [[Float]], columns: Int, rows: Int) { - self.manager = manager - self.labels = labels - self.embeddings = embeddings - self.columns = columns - self.rows = rows - } - - /// Loads the encoders from `SIGLIP2_MODEL_DIR` and embeds `template` (with `{}` replaced) once per label. - public static func load( - labels: [String], template: String = "a photo of {}.", columns: Int, rows: Int - ) async throws -> GridClassifier { - guard let path = ProcessInfo.processInfo.environment["SIGLIP2_MODEL_DIR"], !path.isEmpty else { - throw SigLIP2Error.invalidAsset("Set SIGLIP2_MODEL_DIR to the converted siglip2-base-patch16-256 folder") - } - let manager = try await SigLIP2Manager.load(from: URL(fileURLWithPath: path)) - let embeddings = try await manager.embed( - labels: labels.map { template.replacingOccurrences(of: "{}", with: $0) }) - return GridClassifier(manager: manager, labels: labels, embeddings: embeddings, columns: columns, rows: rows) - } - - /// Classifies every cell of `frame`, keeping `inFlight` calls running. - public func classify(_ frame: CGImage, inFlight: Int = 6) async throws -> Grid { - let start = DispatchTime.now().uptimeNanoseconds - let width = frame.width / columns - let height = frame.height / rows - let crops = (0..<(columns * rows)).map { index in - frame.cropping( - to: CGRect(x: (index % columns) * width, y: (index / columns) * height, width: width, height: height)) - } - var cells = [Cell?](repeating: nil, count: crops.count) - try await withThrowingTaskGroup(of: (Int, Cell).self) { group in - var next = 0 - func launch() { - guard next < crops.count else { return } - let index = next - next += 1 - group.addTask { [self] in (index, try await cell(crops[index])) } - } - for _ in 0.. Cell { - guard let crop else { throw SigLIP2Error.invalidInput("Empty cell") } - let answer = manager.score( - imageEmbedding: try await manager.embed(image: crop), labels: labels, labelEmbeddings: embeddings) - let scale = manager.config.logitScale - let best = answer.similarities[answer.selectedIndex] - let weights = answer.similarities.map { exp(scale * ($0 - best)) } - let total = weights.reduce(0, +) - let top = weights.indices.sorted { weights[$0] > weights[$1] }.prefix(5).map { - Ranked(label: $0, share: weights[$0] / total) - } - return Cell(label: answer.selectedIndex, share: 1 / total, top: top) - } -} diff --git a/Sources/ImageSort/ImageSorter.swift b/Sources/ImageSort/ImageSorter.swift index 1dfa7be..14e986d 100644 --- a/Sources/ImageSort/ImageSorter.swift +++ b/Sources/ImageSort/ImageSorter.swift @@ -29,12 +29,12 @@ public final class ImageSorter: Sendable { self.embeddings = embeddings } - /// Loads the encoders from `SIGLIP2_MODEL_DIR` and embeds the breed prompts once. - public static func load(breeds: [String] = PetsSample.breeds) async throws -> ImageSorter { - guard let path = ProcessInfo.processInfo.environment["SIGLIP2_MODEL_DIR"], !path.isEmpty else { - throw SigLIP2Error.invalidAsset("Set SIGLIP2_MODEL_DIR to the converted siglip2-base-patch16-256 folder") - } - let manager = try await SigLIP2Manager.load(from: URL(fileURLWithPath: path)) + /// Loads the encoders (from `SIGLIP2_MODEL_DIR`, or downloaded once from Hugging Face) and embeds the breed + /// prompts once. + public static func load( + breeds: [String] = PetsSample.breeds, progress: SigLIP2ModelStore.Progress? = nil + ) async throws -> ImageSorter { + let manager = try await SigLIP2Manager.loadDefault(progress: progress) let embeddings = try await manager.embed(labels: breeds.map(PetsSample.prompt(for:))) return ImageSorter(manager: manager, breeds: breeds, embeddings: embeddings) } diff --git a/Sources/ImageSort/PetsSample.swift b/Sources/ImageSort/PetsSample.swift index 993e163..1dadcfb 100644 --- a/Sources/ImageSort/PetsSample.swift +++ b/Sources/ImageSort/PetsSample.swift @@ -53,10 +53,12 @@ public enum PetsSample { .appendingPathComponent("FluidUse/image-sort/oxford-pets-test") } - /// `count` photos in a seeded shuffled order (every cached photo when `count` is nil). The viewer API fetches the + /// `count` photos in a seeded shuffled order (every eligible photo when `count` is nil); `testOnly` restricts + /// the pool to the 3,669 test photos. The viewer API fetches the /// 3,669 test photos; a cache that also holds the train split (ids from 3,669) samples from both. public static func load( - count: Int? = 1000, seed: UInt64 = 0, progress: (@Sendable (Int, Int) -> Void)? = nil + count: Int? = 1000, seed: UInt64 = 0, testOnly: Bool = false, + progress: (@Sendable (Int, Int) -> Void)? = nil ) async throws -> [PetItem] { let directory = cacheDirectory() let manifestURL = directory.appendingPathComponent("manifest.json") @@ -82,7 +84,8 @@ public enum PetsSample { } var generator = SeededGenerator(seed: seed) - let chosen = Array(labels.keys.sorted().shuffled(using: &generator).prefix(count ?? labels.count)) + let pool = labels.keys.filter { !testOnly || $0 < testCount }.sorted() + let chosen = Array(pool.shuffled(using: &generator).prefix(count ?? pool.count)) let missing = chosen.filter { !manager.fileExists(atPath: file(for: $0).path) } if !missing.isEmpty { if sources.isEmpty { diff --git a/Sources/ImageSortCheck/main.swift b/Sources/ImageSortCheck/main.swift index 72dcc8a..8d61ae7 100644 --- a/Sources/ImageSortCheck/main.swift +++ b/Sources/ImageSortCheck/main.swift @@ -4,7 +4,8 @@ import ImageSort // Headless checks for the SigLIP 2 image sorter. // ImageSortCheck tokenizer token ids must equal the Python tokenizer's -// ImageSortCheck [--count=N] [--inflight=N] [--predictions=out.json] zero-shot Pets accuracy and speed +// ImageSortCheck [--split=test|all] [--count=N] [--inflight=N] [--predictions=out.json] +// zero-shot Pets accuracy and speed; test split (3,669) by default, all 7,349 when the train split is cached setvbuf(stdout, nil, _IOLBF, 0) let arguments = CommandLine.arguments.dropFirst() @@ -39,7 +40,8 @@ if arguments.first == "tokenizer", let path = arguments.dropFirst().first { let count = option("count").flatMap(Int.init) let inFlight = option("inflight").flatMap(Int.init) ?? 4 -let items = try await PetsSample.load(count: count ?? PetsSample.testCount) { done, total in +let testOnly = option("split") != "all" +let items = try await PetsSample.load(count: count, testOnly: testOnly) { done, total in if done % 250 == 0 || done == total { print("cached \(done)/\(total) photos") } } let sorter = try await ImageSorter.load() diff --git a/Sources/ImageSortDemo/ImageSortModel.swift b/Sources/ImageSortDemo/ImageSortModel.swift index bad60d0..6989bd9 100644 --- a/Sources/ImageSortDemo/ImageSortModel.swift +++ b/Sources/ImageSortDemo/ImageSortModel.swift @@ -94,8 +94,11 @@ final class ImageSortModel: ObservableObject { items = try await PetsSample.load(count: total) { [weak self] done, all in Task { @MainActor in self?.phase = .loading("Cached \(done) / \(all) photos") } } - phase = .loading("Loading SigLIP 2 and embedding 37 breed names…") - sorter = try await ImageSorter.load() + phase = .loading("Loading SigLIP 2 (first run downloads 750 MB) and embedding 37 breed names…") + sorter = try await ImageSorter.load { [weak self] file, bytes in + guard bytes > 0 else { return } + Task { @MainActor in self?.phase = .loading("Downloaded \(file)") } + } _ = try await sorter?.sort(items[0]) reset() print("ready at \(Date().timeIntervalSince1970)") diff --git a/Sources/ImageSortDemo/README.md b/Sources/ImageSortDemo/README.md index 91f9f39..fcf5e49 100644 --- a/Sources/ImageSortDemo/README.md +++ b/Sources/ImageSortDemo/README.md @@ -4,19 +4,23 @@ Sorts Oxford-IIIT Pets photos into 37 breeds with SigLIP 2 (base, 256 px) on Cor gets is each breed's name, in the prompt `a photo of a {breed}, a type of pet.`; nothing is trained on these photos. ```bash -SIGLIP2_MODEL_DIR=/path/to/siglip2-base-patch16-256 swift run -c release ImageSortDemo +swift run -c release ImageSortDemo ``` -- Four model calls stay in flight; a couple of photos per update fly from the current-photo panel to their tile. -- `IMAGE_SORT_COUNT` sets the sample size (default 1,000; the test split has 3,669, and a cache that also holds - the train split allows up to 7,349). `IMAGE_SORT_WAIT=1` waits for Start (Space); `IMAGE_SORT_LOG=1` prints - each decision. -- Each sorted photo becomes a tile in its breed's row, so the rows grow into a bar chart made of photos; a red - frame marks a photo whose gold breed differs. Orange rows are cat breeds, blue are dog breeds. The left panel - shows the latest photo with its five most likely breeds. +The first run downloads the Core ML packages (about 750 MB, checksum-verified) from +[FluidInference/siglip2-base-patch16-256-coreml](https://huggingface.co/FluidInference/siglip2-base-patch16-256-coreml) +and 1,000 test photos from the Hugging Face dataset viewer; both are cached. `SIGLIP2_MODEL_DIR` loads a local +conversion instead. -Headless: `swift run -c release ImageSortCheck --inflight=4` (M5 Pro, macOS 27: 94.85% on all 3,669 test photos, -193 photos/s). +- Each sorted photo flies from the current-photo panel into its breed's row, so the rows grow into a bar chart made + of photos; a red frame marks a photo whose true breed differs. Orange rows are cat breeds, blue are dog breeds. + The panel shows the latest photo with its five most likely breeds. +- Four photos are in flight at once, so decoding and resizing on the CPU overlap the encoder on the Neural Engine. +- `IMAGE_SORT_COUNT` sets the sample size (default 1,000; up to 3,669 test photos, or 7,349 when the train split is + also cached). `IMAGE_SORT_WAIT=1` waits for Start (Space); `IMAGE_SORT_LOG=1` prints decisions to the terminal. -Data: Oxford-IIIT Pets test split (Parkhi et al., 2012), CC BY-SA 4.0. Model: google/siglip2-base-patch16-256, -Apache-2.0. +Headless: `swift run -c release ImageSortCheck` (M5 Pro, macOS 27: 94.85% on the 3,669 test photos, 202 photos/s; +`--split=all` 94.26% on 7,349). The same 7,349 photos with the PyTorch model (transformers, fp32, MPS, batch 32) +take 102 s at 72 photos/s and 4.2 GB peak memory, against 36 s, 202 photos/s, and 262 MB here. + +Data: Oxford-IIIT Pets (Parkhi et al., 2012), CC BY-SA 4.0. Model: google/siglip2-base-patch16-256, Apache-2.0. diff --git a/Sources/VideoSortDemo/ContentView.swift b/Sources/VideoSortDemo/ContentView.swift deleted file mode 100644 index 4ec6ee2..0000000 --- a/Sources/VideoSortDemo/ContentView.swift +++ /dev/null @@ -1,285 +0,0 @@ -import ImageSort -import SwiftUI - -struct ContentView: View { - @EnvironmentObject private var model: VideoSortModel - - private var scene: VideoSortModel.Scene { model.scene } - - var body: some View { - VStack(spacing: 0) { - header - Divider() - switch model.phase { - case .loading(let message): - status(message, spinning: true) - case .failed(let message): - status("Failed: \(message)", spinning: false) - default: - if scene.isGrid { - HStack(alignment: .top, spacing: 16) { - video - gridLegend.frame(width: 240) - } - .padding(14) - } else { - VStack(spacing: 12) { - HStack(alignment: .top, spacing: 16) { - video - topFive.frame(width: 290) - } - spotted - } - .padding(14) - } - } - Divider() - footer - } - .background(Color(nsColor: .windowBackgroundColor)) - } - - // MARK: Header - - private var header: some View { - ViewThatFits(in: .horizontal) { - HStack(alignment: .center, spacing: 18) { - titleBlock - Spacer(minLength: 12) - stats - button - } - VStack(alignment: .leading, spacing: 10) { - titleBlock - HStack(spacing: 14) { - stats - Spacer(minLength: 8) - button - } - } - } - .padding(.horizontal, 20) - .padding(.vertical, 12) - } - - private var titleBlock: some View { - VStack(alignment: .leading, spacing: 2) { - Text(scene.title).font(.system(size: 26, weight: .bold)) - Text(subtitle).font(.callout).foregroundStyle(.secondary).lineLimit(1).fixedSize() - } - } - - private var subtitle: String { - scene.isGrid - ? "SigLIP 2 · Core ML on the Neural Engine · \(scene.columns * scene.rows) cells per frame, zero-shot" - : "SigLIP 2 · Core ML on the Neural Engine · \(scene.labels.count) animals, zero-shot, every frame" - } - - private var stats: some View { - HStack(spacing: 16) { - stat( - scene.isGrid ? "Frames / s" : "Frames / s", - model.phase == .running ? String(format: "%.0f", model.framesPerSecond) : "–") - stat( - scene.isGrid ? "ms / frame" : "ms / frame", - model.grid.map { String(format: "%.1f", $0.milliseconds) } ?? "–") - stat("Frames labeled", "\(model.framesLabeled)") - if !scene.isGrid { - stat("Correct frames", model.accuracy.map { String(format: "%.1f%%", $0 * 100) } ?? "–") - stat("Species", "\(model.speciesSpotted) / \(scene.labels.count)") - } - } - } - - private func stat(_ title: String, _ value: String) -> some View { - VStack(alignment: .trailing, spacing: 2) { - Text(title.uppercased()).font(.caption2.weight(.semibold)).foregroundStyle(.secondary) - .lineLimit(1).fixedSize() - Text(value).font(.system(size: 20, weight: .semibold, design: .rounded)).monospacedDigit() - .contentTransition(.numericText()).fixedSize() - } - } - - private var button: some View { - Button(model.phase == .running ? "Stop" : "Start") { model.toggle() } - .keyboardShortcut(.space, modifiers: []) - .buttonStyle(.borderedProminent) - .disabled(!(model.phase == .ready || model.phase == .running)) - } - - // MARK: Video - - private var video: some View { - Color.black - .aspectRatio(16 / 9, contentMode: .fit) - .overlay { - if let frame = model.frame { - Image(decorative: frame, scale: 1).resizable().aspectRatio(contentMode: .fit) - } - } - .overlay { if model.phase == .running { overlay } } - .clipShape(RoundedRectangle(cornerRadius: 10)) - .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) - } - - @ViewBuilder - private var overlay: some View { - if let grid = model.grid { - if scene.isGrid { - GeometryReader { proxy in - let width = proxy.size.width / CGFloat(scene.columns) - let height = proxy.size.height / CGFloat(scene.rows) - ForEach(grid.cells.indices, id: \.self) { index in - let cell = grid.cells[index] - let label = scene.labels[cell.label] - ZStack(alignment: .topLeading) { - Rectangle().fill(label.color.opacity(0.12)) - Rectangle().stroke(label.color, lineWidth: 2) - Text("\(label.name) \(Int(cell.share * 100))%") - .font(.system(size: max(10, min(15, width / 13)), weight: .bold)) - .foregroundStyle(label.color).lineLimit(1) - .padding(.horizontal, 5).padding(.vertical, 2) - .background(Color.black.opacity(0.6)) - } - .frame(width: width, height: height) - .offset(x: CGFloat(index % scene.columns) * width, y: CGFloat(index / scene.columns) * height) - } - } - } else if let cell = grid.cells.first { - let label = scene.labels[cell.label] - let correct = model.truth.map { $0 == cell.label } - VStack { - Spacer() - HStack(alignment: .firstTextBaseline, spacing: 10) { - Text(label.name).font(.system(size: 44, weight: .heavy, design: .rounded)) - Text("\(Int(cell.share * 100))%").font(.system(size: 28, weight: .bold, design: .rounded)) - .monospacedDigit() - if correct == false { - Image(systemName: "xmark.circle.fill").foregroundStyle(.red).font(.system(size: 26)) - } - } - .foregroundStyle(label.color) - .padding(.horizontal, 18).padding(.vertical, 8) - .background(Capsule().fill(.black.opacity(0.65))) - .frame(maxWidth: .infinity, alignment: .leading) - .padding(18) - } - } - } - } - - // MARK: Side panels - - private var topFive: some View { - VStack(alignment: .leading, spacing: 10) { - Text("TOP 5 · THIS FRAME").font(.caption.weight(.semibold)).foregroundStyle(.secondary) - if let cell = model.grid?.cells.first, model.phase == .running { - ForEach(Array(cell.top.enumerated()), id: \.offset) { rank, entry in - let label = scene.labels[entry.label] - HStack(spacing: 8) { - Text(label.name).font(rank == 0 ? .title3.weight(.bold) : .callout) - .foregroundStyle(rank == 0 ? label.color : .primary).lineLimit(1) - .frame(width: 120, alignment: .leading) - GeometryReader { proxy in - Capsule().fill(label.color.opacity(rank == 0 ? 1 : 0.6)) - .frame(width: max(2, proxy.size.width * CGFloat(entry.share))) - } - .frame(height: rank == 0 ? 12 : 7) - Text("\(Int(entry.share * 100))%").font(.callout).monospacedDigit() - .frame(width: 42, alignment: .trailing) - } - } - } - Spacer(minLength: 0) - Text( - "Labels are \(scene.labels.count) animal names, typed once: \"a photo of a {animal}.\" Nothing is trained on this video." - ) - .font(.caption).foregroundStyle(.secondary) - } - } - - private var spotted: some View { - VStack(alignment: .leading, spacing: 6) { - Text("SPOTTED · \(model.spots.count)").font(.caption.weight(.semibold)).foregroundStyle(.secondary) - ScrollViewReader { reader in - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: 8) { - ForEach(model.spots) { spot in - let label = scene.labels[spot.label] - VStack(spacing: 3) { - Color.secondary.opacity(0.1) - .frame(width: 92, height: 92) - .overlay { - if let image = spot.thumbnail { - Image(decorative: image, scale: 1).resizable().aspectRatio( - contentMode: .fill) - } - } - .clipShape(RoundedRectangle(cornerRadius: 8)) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke( - spot.correct.map { $0 ? Color.green : Color.red } ?? label.color, - lineWidth: 2.5)) - Text(label.name).font(.caption.weight(.semibold)).foregroundStyle(label.color) - .lineLimit(1) - Text(String(format: "%.1f s · %d%%", spot.seconds, Int(spot.share * 100))) - .font(.caption2).foregroundStyle(.secondary).monospacedDigit() - } - .frame(width: 96) - .id(spot.id) - .transition(.scale.combined(with: .opacity)) - } - } - .animation(.spring(duration: 0.35), value: model.spots.count) - } - .onChange(of: model.spots.count) { _, _ in - if let last = model.spots.last { withAnimation { reader.scrollTo(last.id, anchor: .trailing) } } - } - } - .frame(height: 132) - } - } - - private var gridLegend: some View { - VStack(alignment: .leading, spacing: 8) { - Text("CELLS IN THIS FRAME").font(.caption.weight(.semibold)).foregroundStyle(.secondary) - let counts = model.counts - let total = max(1, counts.reduce(0, +)) - ForEach(Array(scene.labels.enumerated()), id: \.offset) { index, label in - HStack(spacing: 6) { - Text(label.name).font(.callout.weight(counts[index] > 0 ? .semibold : .regular)) - .foregroundStyle(counts[index] > 0 ? label.color : .secondary).lineLimit(1) - .frame(width: 130, alignment: .leading) - GeometryReader { proxy in - Capsule().fill(label.color) - .frame(width: max(2, proxy.size.width * CGFloat(counts[index]) / CGFloat(total))) - } - .frame(height: 8) - Text("\(counts[index])").font(.callout).monospacedDigit().frame(width: 24, alignment: .trailing) - } - } - Spacer(minLength: 0) - Text("Labels are plain phrases, typed once: \"a photo of {label}.\" Nothing is trained on this video.") - .font(.caption).foregroundStyle(.secondary) - } - } - - private func status(_ message: String, spinning: Bool) -> some View { - VStack(spacing: 12) { - if spinning { ProgressView() } - Text(message).foregroundStyle(.secondary) - } - .frame(maxWidth: .infinity, maxHeight: .infinity) - } - - private var footer: some View { - HStack { - Text(scene.credit) - Spacer() - Text("Model: google/siglip2-base-patch16-256 (Apache-2.0), converted to Core ML by FluidInference") - } - .font(.caption).foregroundStyle(.secondary).lineLimit(1) - .padding(.horizontal, 20).padding(.vertical, 8) - } -} diff --git a/Sources/VideoSortDemo/README.md b/Sources/VideoSortDemo/README.md deleted file mode 100644 index d3f864c..0000000 --- a/Sources/VideoSortDemo/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# SigLIP 2 on video - -Plays a video in real time and labels the newest frame with SigLIP 2 (base, 256 px) on Core ML, as fast as the -model allows. - -```bash -SIGLIP2_MODEL_DIR=/path/to/siglip2-base-patch16-256 VIDEO_SORT_WAIT=1 swift run -c release VideoSortDemo -``` - -- Default scene, **Name the animal**: `animals.mp4`, 26 real clips of different animals from Wikimedia Commons, - about 4 s each. Every frame is scored against the 26 animal names (`a photo of a {animal}.`); the caption shows - the top label, the side panel the top five, and the strip below adds a card each time a new animal holds for six - labeled frames. "Correct frames" compares each frame with the animal its clip shows (`animals-credits.json`). - M5 Pro: every frame at 31 fps, 9.5 ms per frame, 94.2% of frames correct, 26 of 26 species spotted. -- `VIDEO_SORT_SCENE=potatoes`: a USDA potato-sorting video (public domain) with a 6 × 4 grid; each cell is labeled - from ten phrases (potatoes, a gloved hand, a truck, the sky, …), about 8 grids per second. -- `VIDEO_SORT_WAIT=1` waits on the first frame for Start (Space); `VIDEO_SORT_LOG=1` prints each labeled frame; - `VIDEO_SORT_FILE` / `VIDEO_SORT_START` play another file. - -Videos are read from `~/Library/Caches/FluidUse/video-sort/`; they are not bundled. Clip sources, licenses, and -authors are listed in `animals-credits.json`. diff --git a/Sources/VideoSortDemo/VideoFrames.swift b/Sources/VideoSortDemo/VideoFrames.swift deleted file mode 100644 index 1bc736a..0000000 --- a/Sources/VideoSortDemo/VideoFrames.swift +++ /dev/null @@ -1,59 +0,0 @@ -import AVFoundation -import CoreGraphics -import VideoToolbox - -/// Decodes a video file at playback speed, looping from `start` seconds, scaled to `width × height`. -enum VideoFrames { - struct Frame: Sendable { - let image: CGImage - let seconds: Double - let number: Int - } - - static func stream(url: URL, start: Double, width: Int, height: Int) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - let task = Task.detached { - do { - var number = 0 - while !Task.isCancelled { - let asset = AVURLAsset(url: url) - guard let track = try await asset.loadTracks(withMediaType: .video).first else { - throw CocoaError(.fileReadCorruptFile) - } - let duration = try await asset.load(.duration) - let reader = try AVAssetReader(asset: asset) - reader.timeRange = CMTimeRange( - start: CMTime(seconds: start, preferredTimescale: 600), end: duration) - let output = AVAssetReaderTrackOutput( - track: track, - outputSettings: [ - kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, - kCVPixelBufferWidthKey as String: width, - kCVPixelBufferHeightKey as String: height, - ]) - reader.add(output) - reader.startReading() - let clock = ContinuousClock() - let began = clock.now - while !Task.isCancelled, let sample = output.copyNextSampleBuffer() { - let seconds = sample.presentationTimeStamp.seconds - try await clock.sleep(until: began + .seconds(seconds - start)) - guard let buffer = sample.imageBuffer else { continue } - var image: CGImage? - VTCreateCGImageFromCVPixelBuffer(buffer, options: nil, imageOut: &image) - if let image { - number += 1 - continuation.yield(Frame(image: image, seconds: seconds, number: number)) - } - } - reader.cancelReading() - } - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - continuation.onTermination = { _ in task.cancel() } - } - } -} diff --git a/Sources/VideoSortDemo/VideoSortDemoApp.swift b/Sources/VideoSortDemo/VideoSortDemoApp.swift deleted file mode 100644 index 60247ed..0000000 --- a/Sources/VideoSortDemo/VideoSortDemoApp.swift +++ /dev/null @@ -1,28 +0,0 @@ -import AppKit -import SwiftUI - -@main -struct VideoSortDemoApp: App { - @StateObject private var model = VideoSortModel() - - init() { - setvbuf(stdout, nil, _IOLBF, 0) - // Bare SwiftPM executables start as background processes; make this one a regular windowed app. - for key in UserDefaults.standard.dictionaryRepresentation().keys where key.hasPrefix("NSWindow Frame") { - UserDefaults.standard.removeObject(forKey: key) - } - NSApplication.shared.setActivationPolicy(.regular) - NSApplication.shared.activate(ignoringOtherApps: true) - } - - var body: some Scene { - WindowGroup("SigLIP 2 on-device — video") { - ContentView() - .environmentObject(model) - .frame(minWidth: 720, minHeight: 520) - .task { await model.prepare() } - } - .defaultSize(width: 1560, height: 980) - .windowResizability(.contentMinSize) - } -} diff --git a/Sources/VideoSortDemo/VideoSortModel.swift b/Sources/VideoSortDemo/VideoSortModel.swift deleted file mode 100644 index f0620ec..0000000 --- a/Sources/VideoSortDemo/VideoSortModel.swift +++ /dev/null @@ -1,277 +0,0 @@ -import CoreGraphics -import Foundation -import ImageIO -import ImageSort -import SwiftUI - -/// Plays a video in real time and keeps labeling the newest frame as fast as the model allows. -@MainActor -final class VideoSortModel: ObservableObject { - enum Phase: Equatable { - case loading(String) - case ready - case running - case failed(String) - } - - struct Label: Identifiable { - let name: String - let color: Color - var id: String { name } - } - - /// What to play and how to label it. - struct Scene { - let title: String - let file: String - let start: Double - let labels: [Label] - let template: String - let columns: Int - let rows: Int - let credit: String - /// JSON next to the video listing each single-label clip's `animal`, `clip_start` and `clip_end` seconds. - let clipsFile: String? - var isGrid: Bool { columns * rows > 1 } - } - - /// An animal the stream settled on, with the frame it was first seen in. - struct Spot: Identifiable { - let id: Int - let label: Int - let share: Float - let seconds: Double - let thumbnail: CGImage? - let correct: Bool? - } - - static let directory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] - .appendingPathComponent("FluidUse/video-sort") - - static let animals: Scene = { - let names = [ - "elephant", "giraffe", "zebra", "tiger", "penguin", "flamingo", "brown bear", "wolf", "deer", "horse", - "goat", - "duck", "owl", "eagle", "monkey", "hippopotamus", "rhinoceros", "seal", "bison", "moose", "fox", "pelican", - "swan", "camel", "cat", "dog", - ] - let labels = names.enumerated().map { index, name in - Label( - name: name, color: Color(hue: Double(index) / Double(names.count), saturation: 0.65, brightness: 0.95)) - } - return Scene( - title: "Name the animal", file: "animals.mp4", start: 0, labels: labels, template: "a photo of a {}.", - columns: 1, rows: 1, - credit: - "Video: 26 clips from Wikimedia Commons (CC BY / CC BY-SA / CC0 / public domain; animals-credits.json)", - clipsFile: "animals-credits.json") - }() - - static let potatoes = Scene( - title: "Label every frame", file: "potatoes.mp4", start: 9.5, - labels: [ - Label(name: "potatoes", color: Color(red: 0.95, green: 0.78, blue: 0.35)), - Label(name: "a gloved hand", color: .red), Label(name: "a person", color: .purple), - Label(name: "a conveyor belt", color: .blue), Label(name: "a truck", color: .orange), - Label(name: "the sky", color: .cyan), Label(name: "a metal machine", color: .gray), - Label(name: "gravel ground", color: .brown), Label(name: "a wire basket", color: .green), - Label(name: "a car", color: .pink), - ], - template: "a photo of {}.", columns: 6, rows: 4, - credit: "Video: \"Potatoes Sorting Montana2026\", USDA (Brien Aho), public domain, via Wikimedia Commons", - clipsFile: nil) - - let scene: Scene - @Published private(set) var phase: Phase = .loading("Starting…") - @Published private(set) var frame: CGImage? - @Published private(set) var grid: GridClassifier.Grid? - @Published private(set) var framesPerSecond: Double = 0 - @Published private(set) var framesLabeled = 0 - @Published private(set) var framesCorrect = 0 - @Published private(set) var spots: [Spot] = [] - /// Label the current clip really shows, when the scene knows it. - @Published private(set) var truth: Int? - - private let environment = ProcessInfo.processInfo.environment - private let logFrames = ProcessInfo.processInfo.environment["VIDEO_SORT_LOG"] == "1" - private var classifier: GridClassifier? - private var player: Task? - private var labeler: Task? - private var latest: VideoFrames.Frame? - private var labelTimes: [Date] = [] - private var streak: (label: Int, count: Int) = (-1, 0) - /// (start, end, label) per clip, when the scene has single-label clips. - private var clips: [(start: Double, end: Double, label: Int)] = [] - private var preparing = false - - init() { - scene = ProcessInfo.processInfo.environment["VIDEO_SORT_SCENE"] == "potatoes" ? Self.potatoes : Self.animals - } - - var counts: [Int] { - var counts = [Int](repeating: 0, count: scene.labels.count) - for cell in grid?.cells ?? [] { counts[cell.label] += 1 } - return counts - } - - var accuracy: Double? { - framesLabeled > 0 && !clips.isEmpty ? Double(framesCorrect) / Double(framesLabeled) : nil - } - - private func expectedLabel(at seconds: Double) -> Int? { - clips.first { seconds >= $0.start && seconds < $0.end }?.label - } - - private func loadClips() { - struct Clip: Decodable { - let animal: String - let clipStart: Double - let clipEnd: Double - enum CodingKeys: String, CodingKey { - case animal - case clipStart = "clip_start" - case clipEnd = "clip_end" - } - } - guard let file = scene.clipsFile, - let data = try? Data(contentsOf: Self.directory.appendingPathComponent(file)), - let decoded = try? JSONDecoder().decode([Clip].self, from: data) - else { return } - let names = scene.labels.map(\.name) - clips = decoded.compactMap { clip in - names.firstIndex(of: clip.animal).map { (clip.clipStart, clip.clipEnd, $0) } - } - } - - var speciesSpotted: Int { Set(spots.filter { $0.correct != false }.map(\.label)).count } - - func prepare() async { - guard !preparing else { return } - preparing = true - do { - loadClips() - phase = .loading("Loading SigLIP 2 and embedding \(scene.labels.count) labels…") - classifier = try await GridClassifier.load( - labels: scene.labels.map(\.name), template: scene.template, columns: scene.columns, rows: scene.rows) - phase = .ready - startPlayback() - print("ready at \(Date().timeIntervalSince1970)") - // VIDEO_SORT_WAIT=1 shows the first frame and waits for Start (Space). - if environment["VIDEO_SORT_WAIT"] != "1" { toggle() } - } catch { - phase = .failed(error.localizedDescription) - } - } - - private func startPlayback() { - let path = environment["VIDEO_SORT_FILE"] ?? Self.directory.appendingPathComponent(scene.file).path - let start = environment["VIDEO_SORT_START"].flatMap(Double.init) ?? scene.start - player?.cancel() - latest = nil - player = Task { [weak self] in - do { - for try await frame in VideoFrames.stream( - url: URL(fileURLWithPath: path), start: start, width: 1280, height: 720) - { - guard let self else { return } - self.frame = frame.image - self.latest = frame - truth = expectedLabel(at: frame.seconds) - // Hold the first frame until Start. - if phase == .ready, environment["VIDEO_SORT_WAIT"] == "1" { break } - } - } catch { - self?.phase = .failed("Video: \(error.localizedDescription)") - } - } - } - - func toggle() { - if phase == .running { - labeler?.cancel() - player?.cancel() - phase = .ready - return - } - guard phase == .ready, let classifier else { return } - phase = .running - framesLabeled = 0 - framesCorrect = 0 - spots = [] - streak = (-1, 0) - labelTimes = [] - startPlayback() - let inFlight = scene.isGrid ? 6 : 1 - labeler = Task { [weak self] in - var lastNumber = -1 - while !Task.isCancelled { - guard let frame = self?.latest, frame.number != lastNumber else { - try? await Task.sleep(for: .milliseconds(3)) - continue - } - lastNumber = frame.number - guard let grid = try? await Self.label(frame.image, with: classifier, inFlight: inFlight) else { - continue - } - self?.apply(grid, frame: frame) - } - } - } - - private nonisolated static func label( - _ image: CGImage, with classifier: GridClassifier, inFlight: Int - ) - async throws -> GridClassifier.Grid - { - try await Task.detached { try await classifier.classify(image, inFlight: inFlight) }.value - } - - private func apply(_ grid: GridClassifier.Grid, frame: VideoFrames.Frame) { - self.grid = grid - framesLabeled += 1 - let clipTruth = expectedLabel(at: frame.seconds) - if let clipTruth, grid.cells.first?.label == clipTruth { framesCorrect += 1 } - let now = Date() - labelTimes.append(now) - labelTimes.removeAll { now.timeIntervalSince($0) > 1 } - framesPerSecond = Double(labelTimes.count) - if !scene.isGrid, let cell = grid.cells.first { track(cell, frame: frame, truth: clipTruth) } - if logFrames { log(grid, frame: frame, truth: clipTruth) } - } - - /// Adds a spot once the same confident label holds for several labeled frames in a row. - private func track(_ cell: GridClassifier.Cell, frame: VideoFrames.Frame, truth: Int?) { - streak = cell.label == streak.label ? (cell.label, streak.count + 1) : (cell.label, 1) - guard streak.count == 6, cell.share >= 0.5, spots.last?.label != cell.label else { return } - let thumbnail = frame.image.cropping( - to: CGRect( - x: (frame.image.width - frame.image.height) / 2, y: 0, width: frame.image.height, - height: frame.image.height)) - spots.append( - Spot( - id: spots.count, label: cell.label, share: cell.share, seconds: frame.seconds, thumbnail: thumbnail, - correct: truth.map { $0 == cell.label })) - } - - private func log(_ grid: GridClassifier.Grid, frame: VideoFrames.Frame, truth: Int?) { - let (cyan, yellow, red, green, dim, reset) = - ("\u{1B}[1;36m", "\u{1B}[33m", "\u{1B}[31m", "\u{1B}[32m", "\u{1B}[2m", "\u{1B}[0m") - let time = String(format: "%.2f", frame.seconds) - if scene.isGrid { - let summary = counts.enumerated().filter { $0.element > 0 }.sorted { $0.element > $1.element } - .map { "\(scene.labels[$0.offset].name) \($0.element)" }.joined(separator: " · ") - print( - "\(cyan)▶ frame \(frame.number) @ \(time) s\(reset) \(yellow)\(summary)\(reset) " - + "\(red)\(grid.cells.count) cells in \(String(format: "%.0f", grid.milliseconds)) ms\(reset)") - return - } - guard let cell = grid.cells.first else { return } - let mark: String = - truth.map { $0 == cell.label ? "\(green)✓\(reset)" : "\(red)✗ clip: \(scene.labels[$0].name)\(reset)" } - ?? "" - print( - "\(cyan)▶ frame \(frame.number) @ \(time) s\(reset) \(yellow)→ \(scene.labels[cell.label].name)\(reset) · " - + "\(Int(cell.share * 100))% · \(red)model call \(String(format: "%.1f", grid.milliseconds)) ms\(reset) " - + "\(mark)\(dim)\(reset)") - } -}