Add Cua-S1-4B-0.2 Core ML runtime

Documentation/DecisionModelSupport.md

@@ -7,6 +7,7 @@ FluidUse serves the weighted sub-1B models on the [Jev Decision Index](https://h
 | Laya | `LayaManager` | [laya-coreml](https://huggingface.co/FluidInference/laya-coreml) | Native Swift |
 | GLiNER 2.5 small / base / multilingual | `GLiNER2Manager` (`.small`, `.base`, `.multilingual`) | [small](https://huggingface.co/FluidInference/gliner2-5-small-coreml), [base](https://huggingface.co/FluidInference/gliner2-5-base-coreml), [multi](https://huggingface.co/FluidInference/gliner2-5-multi-coreml) | Native Swift |
 | Verdict | `VerdictManager` | [verdict-coreml](https://huggingface.co/FluidInference/verdict-coreml) | Native Swift, calibrated, with trained abstention |
+| Cua-S1-4B-0.2 (text / multimodal) | `CuaS1FourBManager` | [cua-s1-4b-coreml](https://huggingface.co/FluidInference/cua-s1-4b-coreml) | Native Swift, GPU, fp16 / w8 / gptq |
 | GLiClass Edge Apps v2 | `GLiClassManager` | [gliclass-edge-apps-coreml](https://huggingface.co/FluidInference/gliclass-edge-apps-coreml) | Native Swift |
 | Kev 0.5B / 0.6B | `PublishedCoreMLManager` + `evaluate(SystemOneRequest)` | [0.5B](https://huggingface.co/FluidInference/kev-0-5b-coreml), [0.6B](https://huggingface.co/FluidInference/kev-0.6b-coreml) | Bridge |
 | Decision 1.0 Kai / Lex | `PublishedCoreMLManager` + `evaluate(SystemOneRequest)` | [Kai](https://huggingface.co/FluidInference/decision-1.0-kai-coreml), [Lex](https://huggingface.co/FluidInference/decision-1.0-lex-coreml) | Bridge |

Package.swift

@@ -30,6 +30,7 @@ let package = Package(
             dependencies: ["FluidUse", "Game2048", "LayaTetris", .product(name: "FluidAudio", package: "FluidAudio")]
         ),
         .executableTarget(name: "FluidUseOfficialBench", dependencies: ["FluidUse"]),
+        .executableTarget(name: "FluidUseCuaS1", dependencies: ["FluidUse"]),
         .executableTarget(
             name: "LayaTetrisDemo",
             dependencies: ["FluidUse", "LayaTetris"],

README.md

@@ -107,6 +107,30 @@ The current ten-seed capped run averages 3,667 pieces for GLiClass LUT8 and 2,87
 heuristic control. See [Benchmarks.md](Benchmarks.md) for the exact policy, per-seed results, and the
 limits of comparison with the older laya measurements.
 
+## Cua-S1-4B GUI decisions
+
+`CuaS1FourBManager` runs [Cua-S1-4B-0.2](https://huggingface.co/cua-ai/cua-s1-4b-0.2) (Qwen3.5-4B + Cua's
+LoRA adapters) on the GPU: one prefill pass scores a closed list of `(element, action)` options for an
+accessibility tree (`.text`) or a screenshot (`.multimodal`). Models download pinned and SHA-256 checked from
+[FluidInference/cua-s1-4b-coreml](https://huggingface.co/FluidInference/cua-s1-4b-coreml) on first use.
+
+```swift
+let cua = try await CuaS1FourBManager.load(configuration: .init(modality: .text, variant: "gptq"))
+let decision = try await cua.decide(CuaS1FourBState(
+    app: "portal", taskFamily: "login_auth", goal: "Log in",
+    accessibilityTree: "- [el_0] Button \"Log in\"",
+    options: [.init(elementId: "el_0", role: "Button", label: "Log in", action: "click"),
+              .init(elementId: "el_0", role: "Button", label: "Log in", action: "skip")]))
+print(decision.bestPerElement())
+```
+
+On a 613-task GUI-360 text split the fp16 and `gptq` (2.6 GB) builds both score 85.5% at about 1.1 s per
+decision on an M5 Pro; the Swift runtime matches the Python Core ML path exactly (38/38 fixture parity for
+both modalities). `swift run -c release FluidUseCuaS1 parity --models <dir> --fixtures <swift-text.json>`
+reruns the parity check against a local mobius build. Call `prewarm()` after `load` in an app: the first prediction of a fresh
+4B Core ML graph spends about 100 s specializing GPU kernels (cached by the OS afterwards). The text decoder is 2.6-6.8 GB and the multimodal
+one 4.0-7.4 GB, so this is a Mac-class model.
+
 ## GLiNER 2.5 classification
 
 `GLiNER2Manager` runs the published base or multilingual classification head on device. Both

Sources/FluidUse/CuaS1FourB/CuaS1FourBModelStore.swift

@@ -0,0 +1,92 @@
+import Foundation
+
+/// Pinned, checksum-verified download of `FluidInference/cua-s1-4b-coreml`.
+///
+/// `Resources/cua-s1-4b-manifest.json` records every file's size and SHA-256 at one Hub revision
+/// (regenerate with `Tools/pin_cua_s1_4b.py`). Only the files one configuration needs are fetched:
+/// the shared tokenizer and embedding table, the requested decoder bucket(s), and for multimodal the
+/// vision tower. Files land in `~/Library/Application Support/FluidUse/Models/cua-s1-4b-coreml`.
+public enum CuaS1FourBModelStore {
+    public static let repository = "FluidInference/cua-s1-4b-coreml"
+    public typealias Progress = @Sendable (_ file: String, _ bytes: Int64) -> Void
+
+    struct Manifest: Decodable {
+        let repository: String
+        let revision: String
+        let files: [PublishedCoreMLModelStore.Manifest.File]
+    }
+
+    static func manifest() throws -> Manifest {
+        guard
+            let url = Bundle.module.url(
+                forResource: "cua-s1-4b-manifest", withExtension: "json", subdirectory: "Resources")
+        else { throw CuaS1FourBError.invalidAsset("cua-s1-4b-manifest.json is not bundled") }
+        let manifest = try JSONDecoder().decode(Manifest.self, from: Data(contentsOf: url))
+        guard manifest.repository == repository else {
+            throw CuaS1FourBError.invalidAsset("manifest pins \(manifest.repository), expected \(repository)")
+        }
+        return manifest
+    }
+
+    /// Path prefixes one configuration needs.
+    static func requiredPrefixes(_ configuration: CuaS1FourBManager.Configuration) -> [String] {
+        var prefixes = ["tokenizer.json", "embeddings.f16", "LICENSE", "NOTICE"]
+        let suffix = configuration.variant.isEmpty ? "" : "-\(configuration.variant)"
+        for length in configuration.lengths {
+            prefixes.append("\(configuration.modality.rawValue)/L\(length)\(suffix)/")
+        }
+        if configuration.modality == .multimodal { prefixes.append("multimodal/vision/") }
+        return prefixes
+    }
+
+    /// Ensure the files for `configuration` exist under `cacheDirectory/cua-s1-4b-coreml`, downloading
+    /// missing or mismatched ones. Returns the repository directory.
+    public static func ensure(
+        configuration: CuaS1FourBManager.Configuration, cacheDirectory: URL? = nil, progress: Progress? = nil
+    ) async throws -> URL {
+        let manifest = try manifest()
+        let prefixes = requiredPrefixes(configuration)
+        let files = manifest.files.filter { file in prefixes.contains { file.path.hasPrefix($0) } }
+        for prefix in prefixes where prefix.hasSuffix("/") && !files.contains(where: { $0.path.hasPrefix(prefix) }) {
+            throw CuaS1FourBError.invalidAsset("\(prefix) is not in the pinned \(repository) revision")
+        }
+        let root = cacheDirectory ?? LayaModelStore.defaultCacheDirectory()
+        let directory = root.appendingPathComponent("cua-s1-4b-coreml", isDirectory: true)
+        let manager = FileManager.default
+        for file in files {
+            try Task.checkCancellation()
+            let destination = directory.appendingPathComponent(file.path)
+            if try PublishedCoreMLModelStore.matches(destination, file) { continue }
+            try manager.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true)
+            let escaped = file.path.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? file.path
+            guard
+                let url = URL(string: "https://huggingface.co/\(repository)/resolve/\(manifest.revision)/\(escaped)")
+            else { throw CuaS1FourBError.invalidAsset("invalid download URL for \(file.path)") }
+            progress?(file.path, 0)
+            let (temporary, response) = try await URLSession.shared.download(from: url)
+            defer { try? manager.removeItem(at: temporary) }
+            guard let http = response as? HTTPURLResponse, http.statusCode == 200 else {
+                throw CuaS1FourBError.invalidAsset(
+                    "download of \(file.path) failed (\((response as? HTTPURLResponse)?.statusCode ?? -1))")
+            }
+            guard try PublishedCoreMLModelStore.matches(temporary, file) else {
+                throw CuaS1FourBError.invalidAsset("size or checksum mismatch for \(file.path)")
+            }
+            try LayaModelStore.installDownloadedFile(temporary, at: destination)
+            progress?(file.path, file.size)
+        }
+        return directory
+    }
+}
+
+extension CuaS1FourBManager {
+    /// Download (pinned, verified) and load one configuration from the FluidUse model cache.
+    public static func load(
+        configuration: Configuration = Configuration(), cacheDirectory: URL? = nil,
+        progress: CuaS1FourBModelStore.Progress? = nil
+    ) async throws -> CuaS1FourBManager {
+        let directory = try await CuaS1FourBModelStore.ensure(
+            configuration: configuration, cacheDirectory: cacheDirectory, progress: progress)
+        return try await load(from: directory, configuration: configuration)
+    }
+}

Sources/FluidUse/CuaS1FourB/CuaS1FourBPrompt.swift

@@ -0,0 +1,56 @@
+import Foundation
+
+/// Prompt contract of `cua_s1.four_b` (letters, `build_prompt`) rendered with the Qwen3.5 chat template.
+///
+/// The template's generation prompt opens a thinking block (`<think>\n`); Cua trains and evaluates the
+/// adapters with exactly that suffix, so the letter logits are read after it.
+public enum CuaS1FourBPrompt {
+    public static let letters = (UInt8(ascii: "A")...UInt8(ascii: "Z")).map { String(UnicodeScalar($0)) }
+
+    public static let systemPrompt =
+        "You are a one-pass computer-use decision model. You are shown the current state of a screen and a "
+        + "fixed, closed list of candidate (element, action) options, each given a single letter. Choose exactly "
+        + "one option: the single best next action to take. Answer with ONLY that option's letter -- no words, "
+        + "no punctuation, no explanation."
+
+    /// Placeholder the host expands to one `<|image_pad|>` per merged image token.
+    public static let imagePlaceholder = "<|vision_start|><|image_pad|><|vision_end|>"
+
+    public static func optionLine(letter: String, option: CuaS1FourBOption) -> String {
+        var action = option.action
+        if option.action == "fill", let entity = option.entityId, !entity.isEmpty {
+            action += " (with entity '\(entity)')"
+        }
+        return "\(letter). \(option.role) \"\(option.label)\" -> \(action)"
+    }
+
+    /// `build_prompt`'s user text.
+    public static func userText(state: CuaS1FourBState, modality: CuaS1FourBModality) throws -> String {
+        guard !state.options.isEmpty else { throw CuaS1FourBError.invalidInput("no options") }
+        guard state.options.count <= letters.count else {
+            throw CuaS1FourBError.invalidInput("\(state.options.count) options exceeds the 26-letter budget")
+        }
+        let lines = zip(letters, state.options).map { optionLine(letter: $0, option: $1) }.joined(separator: "\n")
+        var text = ""
+        if let goal = state.goal, !goal.isEmpty { text += "Goal: \(goal)\n\n" }
+        text += "App: \(state.app)\nTask family: \(state.taskFamily)\n\n"
+        switch modality {
+        case .text:
+            guard let tree = state.accessibilityTree, !tree.isEmpty else {
+                throw CuaS1FourBError.invalidInput("text modality requires an accessibility tree")
+            }
+            text += "Accessibility tree:\n\(tree)\n\n"
+        case .multimodal:
+            text += "The current screenshot is attached.\n\n"
+        }
+        return text + "Options:\n\(lines)\n\nAnswer with a single letter."
+    }
+
+    /// The full chat string passed to the tokenizer (`apply_chat_template(..., add_generation_prompt=True)`).
+    public static func chat(state: CuaS1FourBState, modality: CuaS1FourBModality) throws -> String {
+        let user = try userText(state: state, modality: modality)
+        let content = modality == .multimodal ? imagePlaceholder + user : user
+        return "<|im_start|>system\n\(systemPrompt)<|im_end|>\n<|im_start|>user\n\(content)<|im_end|>\n"
+            + "<|im_start|>assistant\n<think>\n"
+    }
+}

Sources/FluidUse/CuaS1FourB/CuaS1FourBTypes.swift

@@ -0,0 +1,102 @@
+import CoreGraphics
+import Foundation
+
+/// Errors from the Cua-S1-4B runtime.
+public enum CuaS1FourBError: Error, LocalizedError, Sendable {
+    case invalidAsset(String)
+    case invalidModel(String)
+    case invalidInput(String)
+    case promptTooLong(tokens: Int, maximum: Int)
+
+    public var errorDescription: String? {
+        switch self {
+        case .invalidAsset(let detail): return "Cua-S1-4B asset: \(detail)"
+        case .invalidModel(let detail): return "Cua-S1-4B model: \(detail)"
+        case .invalidInput(let detail): return "Cua-S1-4B input: \(detail)"
+        case .promptTooLong(let tokens, let maximum):
+            return "Cua-S1-4B prompt is \(tokens) tokens; the largest loaded bucket holds \(maximum)"
+        }
+    }
+}
+
+/// Which LoRA adapter (and so which converted decoder) a manager runs.
+public enum CuaS1FourBModality: String, Sendable, CaseIterable {
+    /// Accessibility-tree text state.
+    case text
+    /// Screenshot state (vision tower + decoder trained with the multimodal adapter).
+    case multimodal
+}
+
+/// One candidate `(element, action)` decision for a screen state, as in `cua_s1.four_b.Option`.
+public struct CuaS1FourBOption: Sendable, Hashable {
+    public var elementId: String
+    public var role: String
+    public var label: String
+    public var action: String
+    /// Only meaningful for `fill`: which extracted value would be entered.
+    public var entityId: String?
+
+    public init(elementId: String, role: String, label: String, action: String, entityId: String? = nil) {
+        self.elementId = elementId
+        self.role = role
+        self.label = label
+        self.action = action
+        self.entityId = entityId
+    }
+}
+
+/// The screen state and closed option list for one decision.
+public struct CuaS1FourBState: Sendable {
+    public var app: String
+    public var taskFamily: String
+    /// The episode goal when the state itself does not show it.
+    public var goal: String?
+    /// Accessibility tree text (text modality).
+    public var accessibilityTree: String?
+    /// Screenshot (multimodal modality).
+    public var screenshot: CGImage?
+    public var options: [CuaS1FourBOption]
+
+    public init(
+        app: String, taskFamily: String, goal: String? = nil, accessibilityTree: String? = nil,
+        screenshot: CGImage? = nil, options: [CuaS1FourBOption]
+    ) {
+        self.app = app
+        self.taskFamily = taskFamily
+        self.goal = goal
+        self.accessibilityTree = accessibilityTree
+        self.screenshot = screenshot
+        self.options = options
+    }
+}
+
+/// Scored options for one state, in the caller's option order.
+public struct CuaS1FourBDecision: Sendable {
+    public struct Scored: Sendable {
+        public let option: CuaS1FourBOption
+        public let letter: String
+        /// Raw answer-letter logit at the last prompt position.
+        public let logit: Float
+        /// Softmax over all option letters (the `FourBModel.forward` readout).
+        public let probability: Float
+    }
+
+    public let options: [Scored]
+    /// Prompt length in tokens and the bucket it ran in.
+    public let tokens: Int
+    public let bucketLength: Int
+
+    /// The single best option overall (nil only for an empty decision, which `decide` never returns).
+    public var best: Scored? { options.max { $0.logit < $1.logit } }
+
+    /// Per element, the best of that element's own options (Cua's benchmark readout).
+    public func bestPerElement() -> [String: Scored] {
+        var result: [String: Scored] = [:]
+        for scored in options {
+            let id = scored.option.elementId
+            if let current = result[id], current.logit >= scored.logit { continue }
+            result[id] = scored
+        }
+        return result
+    }
+}