Add SigLIP 2 zero-shot image classification and the Sort Photos demo

Package.swift

@@ -50,6 +50,9 @@ let package = Package(
         .executableTarget(name: "SortDecisionsCheck", dependencies: ["SortAnything"]),
         .executableTarget(name: "SortDecisionsDemo", dependencies: ["SortAnything"], exclude: ["README.md"]),
         .executableTarget(name: "SortAnythingDemo", dependencies: ["SortAnything"], exclude: ["README.md"]),
+        .target(name: "ImageSort", dependencies: ["FluidUse"]),
+        .executableTarget(name: "ImageSortCheck", dependencies: ["ImageSort", "FluidUse"]),
+        .executableTarget(name: "ImageSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]),
         .testTarget(
             name: "FluidUseTests", dependencies: ["FluidUse", "LayaTetris"],
             resources: [.copy("Fixtures")]

Sources/FluidUse/SigLIP2/SigLIP2ImagePreprocessor.swift

@@ -0,0 +1,95 @@
+import CoreGraphics
+import Foundation
+
+/// Matches the Hugging Face SigLIP image processor: PIL bilinear resize (antialiased when shrinking) to a square,
+/// rescale to [0, 1], normalize per channel. Output is planar float32 `[3, size, size]`.
+public enum SigLIP2ImagePreprocessor {
+    public static func pixels(from image: CGImage, config: SigLIP2Config) throws -> [Float] {
+        let width = image.width
+        let height = image.height
+        let size = config.imageSize
+        var rgba = [UInt8](repeating: 0, count: width * height * 4)
+        guard
+            let space = CGColorSpace(name: CGColorSpace.sRGB),
+            let context = CGContext(
+                data: &rgba, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4,
+                space: space, bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)
+        else {
+            throw SigLIP2Error.invalidInput("Could not decode a \(width)×\(height) image")
+        }
+        context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
+        // Horizontal pass, then vertical, each rounded to 8 bits like PIL. Layout stays 4 bytes per pixel.
+        let horizontal = resample(
+            rgba, lines: height, inLength: width, outLength: size, sampleStride: 4, lineStride: width * 4,
+            outSampleStride: 4, outLineStride: size * 4, outCount: height * size * 4)
+        let resized = resample(
+            horizontal, lines: size, inLength: height, outLength: size, sampleStride: size * 4, lineStride: 4,
+            outSampleStride: size * 4, outLineStride: 4, outCount: size * size * 4)
+        var planar = [Float](repeating: 0, count: 3 * size * size)
+        for channel in 0..<3 {
+            let scale = 1 / (255 * config.imageStd[channel])
+            let offset = config.imageMean[channel] / config.imageStd[channel]
+            let base = channel * size * size
+            for pixel in 0..<(size * size) {
+                planar[base + pixel] = Float(resized[pixel * 4 + channel]) * scale - offset
+            }
+        }
+        return planar
+    }
+
+    /// One separable pass of PIL's `ImagingResample` with the triangle filter, rounding to 8 bits like PIL.
+    static func resample(
+        _ input: [UInt8], lines: Int, inLength: Int, outLength: Int, sampleStride: Int, lineStride: Int,
+        outSampleStride: Int, outLineStride: Int, outCount: Int
+    ) -> [UInt8] {
+        let scale = Double(inLength) / Double(outLength)
+        let filterScale = max(scale, 1)
+        var starts = [Int](repeating: 0, count: outLength)
+        var counts = [Int](repeating: 0, count: outLength)
+        let taps = Int((filterScale * 2).rounded(.up)) + 2
+        var weights = [Float](repeating: 0, count: outLength * taps)
+        for out in 0..<outLength {
+            let center = (Double(out) + 0.5) * scale
+            let low = max(Int((center - filterScale + 0.5).rounded(.down)), 0)
+            let high = min(Int((center + filterScale + 0.5).rounded(.down)), inLength)
+            var row = [Double](repeating: 0, count: high - low)
+            for index in low..<high { row[index - low] = max(0, 1 - abs((Double(index) - center + 0.5) / filterScale)) }
+            let total = row.reduce(0, +)
+            for (offset, weight) in row.enumerated() {
+                weights[out * taps + offset] = Float(total > 0 ? weight / total : 0)
+            }
+            starts[out] = low
+            counts[out] = min(high - low, taps)
+        }
+        var output = [UInt8](repeating: 0, count: outCount)
+        input.withUnsafeBufferPointer { source in
+            output.withUnsafeMutableBufferPointer { destination in
+                weights.withUnsafeBufferPointer { weight in
+                    for line in 0..<lines {
+                        let lineBase = line * lineStride
+                        let outBase = line * outLineStride
+                        for out in 0..<outLength {
+                            var r: Float = 0
+                            var g: Float = 0
+                            var b: Float = 0
+                            var index = lineBase + starts[out] * sampleStride
+                            let weightBase = out * taps
+                            for tap in 0..<counts[out] {
+                                let w = weight[weightBase + tap]
+                                r += w * Float(source[index])
+                                g += w * Float(source[index + 1])
+                                b += w * Float(source[index + 2])
+                                index += sampleStride
+                            }
+                            let target = outBase + out * outSampleStride
+                            destination[target] = UInt8(max(0, min(255, r.rounded())))
+                            destination[target + 1] = UInt8(max(0, min(255, g.rounded())))
+                            destination[target + 2] = UInt8(max(0, min(255, b.rounded())))
+                        }
+                    }
+                }
+            }
+        }
+        return output
+    }
+}

Sources/FluidUse/SigLIP2/SigLIP2Manager.swift

@@ -0,0 +1,136 @@
+@preconcurrency import CoreML
+import CoreGraphics
+import Foundation
+
+/// Zero-shot image classification with SigLIP 2's Core ML image and text encoders: embed labels once, then score
+/// each image against them. Prediction uses Core ML's async API, so callers may keep several images in flight.
+public final class SigLIP2Manager: Sendable {
+    public let config: SigLIP2Config
+    public let tokenizer: SigLIP2Tokenizer
+
+    private let imageModel: MLModel
+    private let textModel: MLModel
+
+    public init(config: SigLIP2Config, tokenizer: SigLIP2Tokenizer, imageModel: MLModel, textModel: MLModel) {
+        self.config = config
+        self.tokenizer = tokenizer
+        self.imageModel = imageModel
+        self.textModel = textModel
+    }
+
+    /// Downloads (once, checksum-verified) and loads the published fp16 packages.
+    public static func load(
+        cacheDirectory: URL? = nil, computeUnits: MLComputeUnits = .cpuAndNeuralEngine,
+        progress: SigLIP2ModelStore.Progress? = nil
+    ) async throws -> SigLIP2Manager {
+        let directory = try await SigLIP2ModelStore.ensure(cacheDirectory: cacheDirectory, progress: progress)
+        return try await load(from: directory, computeUnits: computeUnits)
+    }
+
+    /// Loads the manager from `SIGLIP2_MODEL_DIR` when set, otherwise from the published packages.
+    public static func loadDefault(progress: SigLIP2ModelStore.Progress? = nil) async throws -> SigLIP2Manager {
+        if let path = ProcessInfo.processInfo.environment["SIGLIP2_MODEL_DIR"], !path.isEmpty {
+            return try await load(from: URL(fileURLWithPath: path))
+        }
+        return try await load(progress: progress)
+    }
+
+    /// Loads `config.json`, `tokenizer.json`, and the image and text packages (`.mlmodelc` preferred) from
+    /// `directory`, as written by the mobius converter.
+    public static func load(
+        from directory: URL, computeUnits: MLComputeUnits = .cpuAndNeuralEngine
+    ) async throws
+        -> SigLIP2Manager
+    {
+        let configURL = directory.appendingPathComponent("config.json")
+        guard let configData = try? Data(contentsOf: configURL) else {
+            throw SigLIP2Error.invalidAsset("Missing config.json in \(directory.path)")
+        }
+        let config = try JSONDecoder().decode(SigLIP2Config.self, from: configData)
+        let tokenizer = try SigLIP2Tokenizer(
+            tokenizerJsonURL: directory.appendingPathComponent("tokenizer.json"), length: config.textLength)
+        let configuration = MLModelConfiguration()
+        configuration.computeUnits = computeUnits
+        async let image = loadModel(named: "\(config.name)-image-\(config.precision)", in: directory, configuration)
+        async let text = loadModel(named: "\(config.name)-text-\(config.precision)", in: directory, configuration)
+        return try await SigLIP2Manager(config: config, tokenizer: tokenizer, imageModel: image, textModel: text)
+    }
+
+    private static func loadModel(
+        named name: String, in directory: URL, _ configuration: MLModelConfiguration
+    ) async throws -> MLModel {
+        let compiled = directory.appendingPathComponent("\(name).mlmodelc")
+        let package = directory.appendingPathComponent("\(name).mlpackage")
+        let url: URL
+        if FileManager.default.fileExists(atPath: compiled.path) {
+            url = compiled
+        } else if FileManager.default.fileExists(atPath: package.path) {
+            url = try await MLModel.compileModel(at: package)
+        } else {
+            throw SigLIP2Error.invalidAsset("Missing \(name).mlmodelc or .mlpackage in \(directory.path)")
+        }
+        return try await MLModel.load(contentsOf: url, configuration: configuration)
+    }
+
+    /// L2-normalized text embedding per label. Compute once per label set and reuse.
+    public func embed(labels: [String]) async throws -> [[Float]] {
+        var embeddings: [[Float]] = []
+        for label in labels {
+            let ids = try tokenizer.encode(label)
+            let input = try MLMultiArray(shape: [1, NSNumber(value: ids.count)], dataType: .int32)
+            let pointer = input.dataPointer.assumingMemoryBound(to: Int32.self)
+            for (index, id) in ids.enumerated() { pointer[index] = id }
+            let output = try await textModel.prediction(
+                from: MLDictionaryFeatureProvider(dictionary: ["input_ids": MLFeatureValue(multiArray: input)]))
+            embeddings.append(try Self.vector(output, name: "text_embeds"))
+        }
+        return embeddings
+    }
+
+    /// Image embedding plus when the Core ML call began and ended (`DispatchTime` uptime nanoseconds).
+    public struct TimedEmbedding: Sendable {
+        public let embedding: [Float]
+        public let predictionStart: UInt64
+        public let predictionEnd: UInt64
+    }
+
+    /// L2-normalized image embedding.
+    public func embed(image: CGImage) async throws -> [Float] { try await embedTimed(image: image).embedding }
+
+    /// L2-normalized image embedding, with the timing of the model call alone (no decoding or resizing).
+    public func embedTimed(image: CGImage) async throws -> TimedEmbedding {
+        let pixels = try SigLIP2ImagePreprocessor.pixels(from: image, config: config)
+        let size = NSNumber(value: config.imageSize)
+        let input = try MLMultiArray(shape: [1, 3, size, size], dataType: .float32)
+        pixels.withUnsafeBufferPointer { source in
+            input.dataPointer.assumingMemoryBound(to: Float.self).update(from: source.baseAddress!, count: pixels.count)
+        }
+        let features = try MLDictionaryFeatureProvider(dictionary: ["pixel_values": MLFeatureValue(multiArray: input)])
+        let start = DispatchTime.now().uptimeNanoseconds
+        let output = try await imageModel.prediction(from: features)
+        let end = DispatchTime.now().uptimeNanoseconds
+        return TimedEmbedding(
+            embedding: try Self.vector(output, name: "image_embeds"), predictionStart: start, predictionEnd: end)
+    }
+
+    /// Scores `image` against label embeddings from `embed(labels:)`.
+    public func classify(image: CGImage, labels: [String], labelEmbeddings: [[Float]]) async throws -> SigLIP2Answer {
+        guard labels.count == labelEmbeddings.count, !labels.isEmpty else {
+            throw SigLIP2Error.invalidInput("Expected one embedding per label")
+        }
+        return score(imageEmbedding: try await embed(image: image), labels: labels, labelEmbeddings: labelEmbeddings)
+    }
+
+    public func score(imageEmbedding: [Float], labels: [String], labelEmbeddings: [[Float]]) -> SigLIP2Answer {
+        let similarities = labelEmbeddings.map { label in zip(label, imageEmbedding).reduce(0) { $0 + $1.0 * $1.1 } }
+        let probabilities = similarities.map { 1 / (1 + exp(-(config.logitScale * $0 + config.logitBias))) }
+        return SigLIP2Answer(labels: labels, similarities: similarities, probabilities: probabilities)
+    }
+
+    private static func vector(_ output: MLFeatureProvider, name: String) throws -> [Float] {
+        guard let array = output.featureValue(for: name)?.multiArrayValue else {
+            throw SigLIP2Error.predictionFailed("Missing \(name)")
+        }
+        return (0..<array.count).map { array[$0].floatValue }
+    }
+}

Sources/FluidUse/SigLIP2/SigLIP2ModelStore.swift

@@ -0,0 +1,84 @@
+import CryptoKit
+import Foundation
+
+/// Downloads the pinned SigLIP 2 Core ML packages from Hugging Face into the FluidUse cache.
+public enum SigLIP2ModelStore {
+    public typealias Progress = @Sendable (_ file: String, _ bytes: Int64) -> Void
+
+    public static let repository = "FluidInference/siglip2-base-patch16-256-coreml"
+    static let revision = "524a5a7d666f23002853831915cf1cc13734f73f"
+
+    private struct Asset {
+        let path: String
+        let sha256: String
+    }
+
+    private static let assets = [
+        Asset(path: "config.json", sha256: "b5d7aaa84399aaa277f9cc00c6c401edc05d535f47f4feb1880e0e4ed2a1a8d4"),
+        Asset(
+            path: "siglip2-base-patch16-256-image-fp16.mlpackage/Data/com.apple.CoreML/model.mlmodel",
+            sha256: "702fdc0b557984e16e4bbcce4eec3568f08a7459bd3c27c770a487823bda808b"),
+        Asset(
+            path: "siglip2-base-patch16-256-image-fp16.mlpackage/Data/com.apple.CoreML/weights/weight.bin",
+            sha256: "da086438b60ada3566f8c91bd8a632462f28d344186e7f60cb5abd223d2e99a8"),
+        Asset(
+            path: "siglip2-base-patch16-256-image-fp16.mlpackage/Manifest.json",
+            sha256: "964a40aa63c2d201a30f0aeab68aef8abc95c854a34f4cae533051cd99996e72"),
+        Asset(
+            path: "siglip2-base-patch16-256-text-fp16.mlpackage/Data/com.apple.CoreML/model.mlmodel",
+            sha256: "69fce0f538fbe78c45b953e1cc396fb912d701c6f985ac825bdec92833aca452"),
+        Asset(
+            path: "siglip2-base-patch16-256-text-fp16.mlpackage/Data/com.apple.CoreML/weights/weight.bin",
+            sha256: "9a52dd8222973b6b4cdf55983689bb2fa18a27fcd20b83f8d8fb89de544530b5"),
+        Asset(
+            path: "siglip2-base-patch16-256-text-fp16.mlpackage/Manifest.json",
+            sha256: "a5dfacf23259261d32c5af26c5cfe4c41a44f0584739b2ef10e70c7f11c2a620"),
+        Asset(
+            path: "tokenizer_config.json", sha256: "9c8a03337138d3b5509e4c032f6863e769b7448750718372c907407d67f6a91b"),
+        Asset(path: "tokenizer.json", sha256: "caefd63119539a63be2d55ef3e05023fbb793948c4bda5bc0c366b42a382f903"),
+    ]
+
+    /// Ensures the packages, tokenizer, and config exist and match their checksums; returns their directory.
+    public static func ensure(cacheDirectory: URL? = nil, progress: Progress? = nil) async throws -> URL {
+        let root = cacheDirectory ?? LayaModelStore.defaultCacheDirectory()
+        let directory = root.appendingPathComponent("siglip2-base-patch16-256-coreml")
+        let manager = FileManager.default
+        try manager.createDirectory(at: directory, withIntermediateDirectories: true)
+        for asset in assets {
+            let destination = directory.appendingPathComponent(asset.path)
+            if manager.fileExists(atPath: destination.path), try checksum(of: destination) == asset.sha256 {
+                continue
+            }
+            try manager.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true)
+            progress?(asset.path, 0)
+            let escaped = asset.path.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? asset.path
+            guard let url = URL(string: "https://huggingface.co/\(repository)/resolve/\(revision)/\(escaped)") else {
+                throw SigLIP2Error.invalidAsset("Invalid Hugging Face asset URL")
+            }
+            let (temporary, response) = try await URLSession.shared.download(from: url)
+            defer { try? manager.removeItem(at: temporary) }
+            guard let http = response as? HTTPURLResponse, http.statusCode == 200 else {
+                throw SigLIP2Error.invalidAsset("Download failed for \(asset.path)")
+            }
+            let actual = try checksum(of: temporary)
+            guard actual == asset.sha256 else {
+                throw SigLIP2Error.invalidAsset(
+                    "Checksum mismatch for \(asset.path): expected \(asset.sha256), got \(actual)")
+            }
+            let size = (try manager.attributesOfItem(atPath: temporary.path)[.size] as? NSNumber)?.int64Value ?? 0
+            try LayaModelStore.installDownloadedFile(temporary, at: destination)
+            progress?(asset.path, size)
+        }
+        return directory
+    }
+
+    private static func checksum(of file: URL) throws -> String {
+        let handle = try FileHandle(forReadingFrom: file)
+        defer { try? handle.close() }
+        var digest = SHA256()
+        while let chunk = try handle.read(upToCount: 1_048_576), !chunk.isEmpty {
+            digest.update(data: chunk)
+        }
+        return digest.finalize().map { String(format: "%02x", $0) }.joined()
+    }
+}