diff --git a/Package.swift b/Package.swift index 9fc5e4f..1a75b92 100644 --- a/Package.swift +++ b/Package.swift @@ -53,6 +53,9 @@ let package = Package( .target(name: "ImageSort", dependencies: ["FluidUse"]), .executableTarget(name: "ImageSortCheck", dependencies: ["ImageSort", "FluidUse"]), .executableTarget(name: "ImageSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]), + .executableTarget(name: "KevCheck", dependencies: ["FluidUse"]), + .executableTarget( + name: "KevGuessWhoDemo", dependencies: ["FluidUse", "SortAnything"], exclude: ["README.md", "demo.sh"]), .testTarget( name: "FluidUseTests", dependencies: ["FluidUse", "LayaTetris"], resources: [.copy("Fixtures")] diff --git a/README.md b/README.md index 1998f7c..c4d613a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,29 @@ contain the classification path; the native entity, relation, and record extract are not exposed by this Swift manager. A ten-seed 2048 comparison with GLiClass is in [Benchmarks.md](Benchmarks.md). +## Kev decisions + +`KevFastManager` runs [Kev-0.8B](https://huggingface.co/jaredpalmer/kev-0.8b) (Qwen3.5 backbone, Apache-2.0) on the +GPU through Core ML. One call reads the text and answers all of a request's questions (multiple choice, yes/no, or a +score, with calibrated probabilities); questions that do not fit fall back to one call per question. The pinned, +checksummed snapshot downloads from [FluidInference/kev-0.8b-coreml](https://huggingface.co/FluidInference/kev-0.8b-coreml) +on first use (~3.5 GB, macOS 15 / iOS 18). + +```swift +let kev = try await KevFastManager.load(from: try await KevModelStore.ensure()) +let answers = try await kev.answer( + state: "Shoes arrived two weeks late and in the wrong size.", + questions: [ + .choice("Which team should handle this?", options: [("returns", nil), ("shipping", nil), ("billing", nil)]), + .noul("Does this need urgent human attention?"), + ]) +print(answers.map(\.best)) +``` + +On an M5 Pro a short ticket with two questions takes 18 ms and a Wikipedia bio with twelve yes/no questions about 38 ms. +`swift run -c release KevGuessWhoDemo` plays Guess Who over 80 Wikipedia people with it +([Sources/KevGuessWhoDemo](Sources/KevGuessWhoDemo/README.md)). + ## Demo ```bash diff --git a/Sources/FluidUse/Kev/KevFastManager.swift b/Sources/FluidUse/Kev/KevFastManager.swift new file mode 100644 index 0000000..073981a --- /dev/null +++ b/Sources/FluidUse/Kev/KevFastManager.swift @@ -0,0 +1,323 @@ +@preconcurrency import CoreML +import Foundation + +/// Kev in one call per request: the state and every question run through a single `fused_S*_P*` function, the +/// questions packed end to end (each restarting from the state) instead of one row per question. Questions longer +/// than a lane (128 tokens) or with more than 16 options fall back to `KevManager`'s rows. +@available(macOS 15.0, iOS 18.0, *) +public final class KevFastManager: Sendable { + struct Shape: Sendable { + let hidden: Int + let rotary: Int + let theta: Double + let convTail: Int + } + + struct Branch { + let ids: [Int] + let decide: Int + let options: [Int] + } + + static let lane = 128 + static let readouts = 16 + static let maxOptions = 16 + + private let rows: KevManager + private let compiled: URL + private let computeUnits: MLComputeUnits + private let shape: Shape + private let embeddings: Data + private let padID: Int + /// State bucket -> packed buckets, from the package's function names. + private let buckets: [Int: [Int]] + private let stateBuckets: [Int] + private let largestPacked: Int + private let functions = FunctionCache() + + actor FunctionCache { + private var models: [String: MLModel] = [:] + + func model(_ name: String, at url: URL, units: MLComputeUnits) async throws -> MLModel { + if let model = models[name] { return model } + let configuration = MLModelConfiguration() + configuration.computeUnits = units + configuration.functionName = name + let model = try await MLModel.load(contentsOf: url, configuration: configuration) + models[name] = model + return model + } + } + + /// `directory` holds the row buckets `KevManager` loads plus `fused/` with `KevFused.mlmodelc` (or `.mlpackage`) + /// and its `config.json`. + public static func load( + from directory: URL, computeUnits: MLComputeUnits = .cpuAndGPU + ) async throws -> KevFastManager { + // row models load only if a question falls back to them + let rows = try await KevManager.load(from: directory, computeUnits: computeUnits, eager: false) + let fused = directory.appendingPathComponent("fused") + var compiled = fused.appendingPathComponent("KevFused.mlmodelc") + if !FileManager.default.fileExists(atPath: compiled.path) { + compiled = try await KevManager.compiled(fused.appendingPathComponent("KevFused.mlpackage")) + } + guard + let config = try JSONSerialization.jsonObject( + with: Data(contentsOf: fused.appendingPathComponent("config.json"))) as? [String: Any], + let kernel = config["conv_kernel"] as? Int, let hidden = config["hidden_size"] as? Int, + let rotary = config["rotary_dim"] as? Int, let theta = (config["rope_theta"] as? NSNumber)?.doubleValue, + let pad = config["pad_id"] as? Int, let vocab = config["vocab_size"] as? Int + else { throw KevError.invalidAsset("fused/config.json is missing fields") } + // the embedding table ships once, in one of the row bucket folders + guard + let table = try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + .map({ $0.resolvingSymlinksInPath().appendingPathComponent("embeddings.f16") }) + .first(where: { FileManager.default.fileExists(atPath: $0.path) }) + else { throw KevError.invalidAsset("no row bucket folder has embeddings.f16") } + let embeddings = try Data(contentsOf: table, options: .alwaysMapped) + guard embeddings.count == vocab * hidden * 2 else { throw KevError.invalidAsset("embeddings.f16 size") } + var buckets: [Int: [Int]] = [:] + let suffix = "_B\(readouts)_K\(maxOptions)" + for name in try await MLModelAsset(url: compiled).functionNames + where name.hasPrefix("fused_S") && name.hasSuffix(suffix) { + let numbers = name.dropLast(suffix.count).split(whereSeparator: { !$0.isNumber }).compactMap { Int($0) } + guard numbers.count == 2 else { continue } + buckets[numbers[0], default: []].append(numbers[1]) + } + guard !buckets.isEmpty else { + throw KevError.invalidAsset("no fused_S*_P* functions in \(compiled.lastPathComponent)") + } + return KevFastManager( + rows: rows, compiled: compiled, computeUnits: computeUnits, + shape: Shape(hidden: hidden, rotary: rotary, theta: theta, convTail: kernel - 1), embeddings: embeddings, + padID: pad, buckets: buckets.mapValues { $0.sorted() }) + } + + init( + rows: KevManager, compiled: URL, computeUnits: MLComputeUnits, shape: Shape, embeddings: Data, padID: Int, + buckets: [Int: [Int]] + ) { + self.rows = rows + self.compiled = compiled + self.computeUnits = computeUnits + self.shape = shape + self.embeddings = embeddings + self.padID = padID + self.buckets = buckets + self.stateBuckets = buckets.keys.sorted() + self.largestPacked = buckets.values.compactMap(\.last).min() ?? 0 + } + + /// Loads every function (or those of `stateBuckets`) and runs it once, so no request pays for loading or for the + /// first GPU dispatch. + public func warm(stateBuckets: [Int]? = nil) async throws { + for s in stateBuckets ?? self.stateBuckets { + for p in buckets[s] ?? [] { + _ = try await run( + stateIDs: [padID], stateLen: s, branches: [Branch(ids: [padID], decide: 0, options: [0])], + starts: [0], packedLen: p) + } + } + } + + public func answer( + state: String, questions: [KevQuestion], maxStateTokens: Int = KevManager.evaluationMaxStateTokens + ) async throws -> [KevAnswer] { + let special = rows.special + let stateIDs = [special.state] + Array(try rows.userTokens(state).prefix(maxStateTokens - 1)) + var branches: [Branch] = [] + for question in questions { + var ids = [special.question] + (try rows.userTokens(question.instruction)) + var ends: [Int] = [] + for text in question.optionTexts { + ids += [special.option] + (try rows.userTokens(text)) + [special.closeOption] + ends.append(ids.count - 1) + } + ids.append(special.decide) + branches.append(Branch(ids: ids, decide: ids.count - 1, options: ends)) + } + var answers = [KevAnswer?](repeating: nil, count: questions.count) + let fits = branches.indices.filter { + branches[$0].ids.count <= Self.lane && branches[$0].options.count <= Self.maxOptions + } + guard let stateLen = stateBuckets.first(where: { stateIDs.count <= $0 }) else { + for index in answers.indices { + answers[index] = try await rows.answer(stateIDs: stateIDs, question: questions[index]) + } + return answers.compactMap { $0 } + } + for index in answers.indices where !fits.contains(index) { + answers[index] = try await rows.answer(stateIDs: stateIDs, question: questions[index]) + } + for group in Self.packGroups(fits.map { branches[$0].ids.count }, packedLen: largestPacked) { + let indices = group.map { fits[$0] } + let starts = Self.laneStarts(indices.map { branches[$0].ids.count }) + let extent = zip(starts, indices).map { $0 + branches[$1].ids.count }.max() ?? 1 + guard let packedLen = buckets[stateLen]?.first(where: { extent <= $0 }) else { + throw KevError.invalidAsset("no packed bucket of \(extent) tokens for state bucket \(stateLen)") + } + let probabilities = try await run( + stateIDs: stateIDs, stateLen: stateLen, branches: indices.map { branches[$0] }, starts: starts, + packedLen: packedLen) + for (slot, index) in indices.enumerated() { + answers[index] = KevAnswer( + keys: questions[index].keys, probabilities: probabilities[slot], + tokens: stateIDs.count + branches[index].ids.count) + } + } + return answers.compactMap { $0 } + } + + /// Start offsets of questions packed in order, a question that would cross a lane moved to the next lane. + static func laneStarts(_ lengths: [Int]) -> [Int] { + var starts: [Int] = [] + var start = 0 + for n in lengths { + if start / lane != (start + n - 1) / lane { start = (start / lane + 1) * lane } + starts.append(start) + start += n + } + return starts + } + + /// Greedy in-order groups of question indices that fit one call (`packedLen` tokens, readout count). + static func packGroups(_ lengths: [Int], packedLen: Int) -> [[Int]] { + var groups: [[Int]] = [] + var current: [Int] = [] + for (index, _) in lengths.enumerated() { + let candidate = current + [index] + let starts = laneStarts(candidate.map { lengths[$0] }) + let extent = starts.last! + lengths[index] + if !current.isEmpty && (extent > packedLen || candidate.count > readouts) { + groups.append(current) + current = [index] + } else { + current = candidate + } + } + if !current.isEmpty { groups.append(current) } + return groups + } + + private func model(state: Int, packed: Int) async throws -> MLModel { + let name = "fused_S\(state)_P\(packed)_B\(Self.readouts)_K\(Self.maxOptions)" + do { + return try await functions.model(name, at: compiled, units: computeUnits) + } catch { + throw KevError.invalidAsset("function \(name): \(error.localizedDescription)") + } + } + + private func run( + stateIDs: [Int], stateLen: Int, branches: [Branch], starts: [Int], packedLen: Int + ) + async throws -> [[Float]] + { + let model = try await model(state: stateLen, packed: packedLen) + let total = stateLen + packedLen + let n = stateIDs.count + var tokens = [Int](repeating: padID, count: total) + var positions = Array(0..= 0 { + tailPointer[j * stateLen + n - shape.convTail + j] = 1 + } + for i in 0..= s { + keepPointer[(s - 1) * packedLen + start + p] = 1 + } else { + lagTailPointer[((s - 1) * packedLen + start + p) * lags + lags + p - s] = 1 + } + } + } + decidePointer[b * packedLen + start + branch.decide] = 1 + for (slot, index) in branch.options.enumerated() { + optionPointer[(b * Self.maxOptions + slot) * packedLen + start + index] = 1 + maskPointer[b * Self.maxOptions + slot] = 1 + } + } + let (cos, sin) = try rope(positions) + let output = try await model.prediction( + from: MLDictionaryFeatureProvider(dictionary: [ + "hidden": try embed(tokens), "cos": cos, "sin": sin, "valid": valid, "tail_onehot": tail, + "segment": segment, "lag_keep": lagKeep, "lag_tail": lagTail, "decide_onehot": decide, + "option_onehot": options, "option_mask": mask, + ])) + guard let logits = output.featureValue(for: "logits")?.multiArrayValue else { + throw KevError.invalidOutput("fused logits") + } + let strides = logits.strides.map(\.intValue) + let pointer = logits.dataPointer.assumingMemoryBound(to: Float.self) + return branches.enumerated().map { b, branch in + var values = (0.. MLMultiArray { + let array = try half([1, tokens.count, shape.hidden]) + let destination = array.dataPointer.assumingMemoryBound(to: Float16.self) + embeddings.withUnsafeBytes { raw in + let table = raw.bindMemory(to: Float16.self).baseAddress! + for (position, token) in tokens.enumerated() { + (destination + position * shape.hidden).update(from: table + token * shape.hidden, count: shape.hidden) + } + } + return array + } + + private func rope(_ positions: [Int]) throws -> (MLMultiArray, MLMultiArray) { + let cos = try half([positions.count, shape.rotary]) + let sin = try half([positions.count, shape.rotary]) + let c = cos.dataPointer.assumingMemoryBound(to: Float16.self) + let s = sin.dataPointer.assumingMemoryBound(to: Float16.self) + let halfDim = shape.rotary / 2 + for (row, position) in positions.enumerated() { + for i in 0.. MLMultiArray { + let array = try MLMultiArray(shape: dims.map { NSNumber(value: $0) }, dataType: .float16) + array.dataPointer.initializeMemory(as: Float16.self, repeating: 0, count: array.count) + return array + } +} diff --git a/Sources/FluidUse/Kev/KevManager.swift b/Sources/FluidUse/Kev/KevManager.swift new file mode 100644 index 0000000..51e70ec --- /dev/null +++ b/Sources/FluidUse/Kev/KevManager.swift @@ -0,0 +1,318 @@ +@preconcurrency import CoreML +import Foundation + +/// One typed question, as in Kev's System One API. +public enum KevQuestion: Sendable { + /// Pick one of `options` (key, optional description). + case choice(String, options: [(key: String, description: String?)]) + /// Yes or no, with optional descriptions of each side. + case noul(String, no: String? = nil, yes: String? = nil) + /// One of an ordered list of levels. + case score(String, levels: [String]) + + var instruction: String { + switch self { + case .choice(let text, _), .noul(let text, _, _), .score(let text, _): text + } + } + + /// Option texts exactly as Kev's `api.to_record` renders them. + public var optionTexts: [String] { + switch self { + case .choice(_, let options): options.map { Self.optionText($0.key, $0.description) } + case .noul(_, let no, let yes): [Self.optionText("no", no), Self.optionText("yes", yes)] + case .score(_, let levels): levels + } + } + + /// Keys the probabilities are reported under: choice keys, ["false", "true"], or level indices. + public var keys: [String] { + switch self { + case .choice(_, let options): options.map(\.key) + case .noul: ["false", "true"] + case .score(_, let levels): levels.indices.map(String.init) + } + } + + private static func optionText(_ name: String, _ description: String?) -> String { + guard let description, !description.isEmpty else { return name } + return "\(name): \(description)" + } +} + +public struct KevAnswer: Sendable { + public let keys: [String] + public let probabilities: [Float] + public let tokens: Int + + public var bestIndex: Int { probabilities.indices.max { probabilities[$0] < probabilities[$1] } ?? 0 } + public var best: String { keys[bestIndex] } + public var confidence: Float { probabilities[bestIndex] } +} + +public enum KevError: Error, LocalizedError, Sendable { + case invalidAsset(String) + case tooLong(String) + case invalidOutput(String) + + public var errorDescription: String? { + switch self { + case .invalidAsset(let reason): "Invalid Kev asset: \(reason)" + case .tooLong(let reason): "Kev input too long: \(reason)" + case .invalidOutput(let reason): "Invalid Kev output: \(reason)" + } + } +} + +/// Kev (Qwen3.5 backbone + pointer head) on Core ML. Each question runs as its own causal row, state first, exactly +/// as Kev scores its Qwen3.5 checkpoints; the smallest bucket that fits the row is used. Calls are not serialized: +/// Core ML's async prediction is thread-safe, so several questions can be in flight at once. +public final class KevManager: Sendable { + struct Bucket: Sendable { + let length: Int + let maxOptions: Int + /// `.mlmodelc` or `.mlpackage` (compiled on first load). + let url: URL + } + + /// Row models, loaded on first use of their bucket (or all at `load` when eager). + actor RowModels { + private let computeUnits: MLComputeUnits + private var models: [Int: MLModel] = [:] + + init(computeUnits: MLComputeUnits) { self.computeUnits = computeUnits } + + func model(for bucket: Bucket) async throws -> MLModel { + if let model = models[bucket.length] { return model } + let url = + bucket.url.pathExtension == "mlpackage" ? try await KevManager.compiled(bucket.url) : bucket.url + let configuration = MLModelConfiguration() + configuration.computeUnits = computeUnits + let model = try await MLModel.load(contentsOf: url, configuration: configuration) + models[bucket.length] = model + return model + } + } + + struct Special: Sendable { + let state: Int + let question: Int + let option: Int + let closeOption: Int + let decide: Int + } + + let tokenizer: QwenBPETokenizer + private let buckets: [Bucket] + private let rowModels: RowModels + private let embeddings: Data + private let hiddenSize: Int + private let rotaryDim: Int + private let ropeTheta: Double + private let padID: Int + let special: Special + + /// `directory` holds `tokenizer.json` and one `L_K/` folder per bucket with `config.json`, + /// `embeddings.f16` and a `KevRow_*.mlpackage` (or compiled `.mlmodelc`). + /// Compiles `package` once and keeps `.mlmodelc` beside it, so later launches skip compiling; when the folder + /// is read-only the temporary compiled copy is used. + static func compiled(_ package: URL) async throws -> URL { + let destination = package.deletingPathExtension().appendingPathExtension("mlmodelc") + let manager = FileManager.default + if manager.fileExists(atPath: destination.path) { return destination } + let temporary = try await MLModel.compileModel(at: package) + do { + try manager.moveItem(at: temporary, to: destination) + return destination + } catch { + return temporary + } + } + + /// `eager: false` defers loading each row bucket's model to its first question. + public static func load( + from directory: URL, computeUnits: MLComputeUnits = .all, eager: Bool = true + ) async throws -> KevManager { + let tokenizer = try QwenBPETokenizer(tokenizerJsonURL: directory.appendingPathComponent("tokenizer.json")) + let folders = try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + .map { $0.resolvingSymlinksInPath() } + .filter { $0.lastPathComponent.range(of: #"^L\d+_K\d+$"#, options: .regularExpression) != nil } + guard !folders.isEmpty else { throw KevError.invalidAsset("No L_K bucket folders") } + var buckets: [Bucket] = [] + var config: [String: Any] = [:] + for folder in folders { + guard + let parsed = try JSONSerialization.jsonObject( + with: Data(contentsOf: folder.appendingPathComponent("config.json"))) as? [String: Any] + else { throw KevError.invalidAsset("Unreadable \(folder.lastPathComponent)/config.json") } + config = parsed + let files = try FileManager.default.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil) + guard + let modelURL = files.first(where: { $0.pathExtension == "mlmodelc" }) + ?? files.first(where: { $0.pathExtension == "mlpackage" }) + else { throw KevError.invalidAsset("No Core ML model in \(folder.lastPathComponent)") } + buckets.append( + Bucket( + length: parsed["length"] as? Int ?? 0, maxOptions: parsed["max_options"] as? Int ?? 0, + url: modelURL)) + } + guard let hidden = config["hidden_size"] as? Int, let rotary = config["rotary_dim"] as? Int, + let theta = (config["rope_theta"] as? NSNumber)?.doubleValue, let pad = config["pad_id"] as? Int, + let vocab = config["vocab_size"] as? Int, let tokens = config["special_tokens"] as? [String: Int], + let state = tokens["state"], let question = tokens["q"], let option = tokens["opt"], + let close = tokens["close_opt"], let decide = tokens["decide"] + else { throw KevError.invalidAsset("config.json is missing Kev fields") } + guard + let table = folders.map({ $0.appendingPathComponent("embeddings.f16") }) + .first(where: { FileManager.default.fileExists(atPath: $0.path) }) + else { throw KevError.invalidAsset("no bucket folder has embeddings.f16") } + let embeddings = try Data(contentsOf: table, options: .alwaysMapped) + guard embeddings.count == vocab * hidden * 2 else { + throw KevError.invalidAsset("embeddings.f16 has \(embeddings.count) bytes, expected \(vocab * hidden * 2)") + } + let rowModels = RowModels(computeUnits: computeUnits) + if eager { + for bucket in buckets { _ = try await rowModels.model(for: bucket) } + } + return KevManager( + tokenizer: tokenizer, buckets: buckets.sorted { $0.length < $1.length }, rowModels: rowModels, + embeddings: embeddings, + hiddenSize: hidden, rotaryDim: rotary, ropeTheta: theta, padID: pad, + special: Special(state: state, question: question, option: option, closeOption: close, decide: decide)) + } + + init( + tokenizer: QwenBPETokenizer, buckets: [Bucket], rowModels: RowModels, embeddings: Data, hiddenSize: Int, + rotaryDim: Int, + ropeTheta: Double, padID: Int, special: Special + ) { + self.tokenizer = tokenizer + self.buckets = buckets + self.rowModels = rowModels + self.embeddings = embeddings + self.hiddenSize = hiddenSize + self.rotaryDim = rotaryDim + self.ropeTheta = ropeTheta + self.padID = padID + self.special = special + } + + /// Kev's training and evaluation state limit (`kev.model.MAX_STATE`): the state marker plus 383 text tokens. + public static let evaluationMaxStateTokens = 384 + + /// Answers every question about `state`; each question is one Core ML call. The state is cut to + /// `maxStateTokens` (marker included) as Kev's `encode` does; the default matches its published evaluations. + public func answer( + state: String, questions: [KevQuestion], maxStateTokens: Int = KevManager.evaluationMaxStateTokens + ) async throws -> [KevAnswer] { + let stateIDs = [special.state] + Array(try userTokens(state).prefix(maxStateTokens - 1)) + var answers: [KevAnswer] = [] + for question in questions { + answers.append(try await answer(stateIDs: stateIDs, question: question)) + } + return answers + } + + /// Like `answer(state:questions:)`, but every question is in flight at once (one Core ML call each). + public func answerConcurrently( + state: String, questions: [KevQuestion], maxStateTokens: Int = KevManager.evaluationMaxStateTokens + ) async throws -> [KevAnswer] { + let stateIDs = [special.state] + Array(try userTokens(state).prefix(maxStateTokens - 1)) + return try await withThrowingTaskGroup(of: (Int, KevAnswer).self) { group in + for (index, question) in questions.enumerated() { + group.addTask { (index, try await self.answer(stateIDs: stateIDs, question: question)) } + } + var answers = [KevAnswer?](repeating: nil, count: questions.count) + for try await (index, answer) in group { answers[index] = answer } + return answers.compactMap { $0 } + } + } + + /// Kev's `user_tokens`: caller text can never produce `<|name|>` control tokens. + func userTokens(_ text: String) throws -> [Int] { + let escaped = text.replacingOccurrences( + of: #"<\|([A-Za-z0-9_]+)\|>"#, with: "<¦$1¦>", options: .regularExpression) + return try tokenizer.encode(escaped) + } + + func answer(stateIDs: [Int], question: KevQuestion) async throws -> KevAnswer { + var ids = stateIDs + [special.question] + (try userTokens(question.instruction)) + var optionEnds: [Int] = [] + for text in question.optionTexts { + ids += [special.option] + (try userTokens(text)) + [special.closeOption] + optionEnds.append(ids.count - 1) + } + ids.append(special.decide) + let decideIndex = ids.count - 1 + guard let bucket = buckets.first(where: { ids.count <= $0.length && optionEnds.count <= $0.maxOptions }) else { + throw KevError.tooLong("\(ids.count) tokens / \(optionEnds.count) options exceed every bucket") + } + let features = try inputs(ids: ids, decide: decideIndex, options: optionEnds, bucket: bucket) + let output = try await rowModels.model(for: bucket).prediction(from: features) + guard let logits = output.featureValue(for: "logits")?.multiArrayValue else { + throw KevError.invalidOutput("missing logits") + } + let count = optionEnds.count + var values = (0.. MLDictionaryFeatureProvider { + let length = bucket.length + let hidden = try MLMultiArray( + shape: [1, NSNumber(value: length), NSNumber(value: hiddenSize)], dataType: .float32) + let hiddenPointer = hidden.dataPointer.assumingMemoryBound(to: Float.self) + embeddings.withUnsafeBytes { raw in + let table = raw.bindMemory(to: Float16.self) + for position in 0.. MLMultiArray { + let array = try MLMultiArray(shape: shape.map { NSNumber(value: $0) }, dataType: .float32) + array.dataPointer.initializeMemory(as: Float.self, repeating: 0, count: array.count) + return array + } +} diff --git a/Sources/FluidUse/Kev/KevModelStore.swift b/Sources/FluidUse/Kev/KevModelStore.swift new file mode 100644 index 0000000..515b526 --- /dev/null +++ b/Sources/FluidUse/Kev/KevModelStore.swift @@ -0,0 +1,106 @@ +import CryptoKit +import Foundation + +/// Downloads the pinned Kev-0.8B Core ML snapshot (FluidInference/kev-0.8b-coreml): the fused multifunction package, +/// the row buckets, the embedding table and the tokenizer, laid out as `KevFastManager.load(from:)` expects. +public enum KevModelStore { + public typealias Progress = @Sendable (_ file: String, _ bytes: Int64) -> Void + + struct Asset { + let path: String + let sha256: String + } + + static let repository = "FluidInference/kev-0.8b-coreml" + static let revision = "8fa7089169d82a1534a723500ffd7106735b6bc1" + static let assets: [Asset] = [ + Asset(path: "config.json", sha256: "abe42540d7ebef1f3fd1891f37fb2bc79a3c8fb43b76a1a64af2fd274e9cf11c"), + Asset(path: "fused/config.json", sha256: "fdea999e887837f62d4fa4b07d1a941e0474ef89b9c3442fac8f3441bc6bb96e"), + Asset( + path: "fused/KevFused.mlpackage/Data/com.apple.CoreML/model.mlmodel", + sha256: "b628b2c64e3c8052c6b1d78b61a56c219dd033751791c9dc65b68dd58331e5b8"), + Asset( + path: "fused/KevFused.mlpackage/Data/com.apple.CoreML/weights/weight.bin", + sha256: "7e39fc13d99f1ff0d79a5b1965978e6640873893c79dafb874609aa9808ee49b"), + Asset( + path: "fused/KevFused.mlpackage/Manifest.json", + sha256: "94ba26f822aa37427a348dfb74bcdfd56ae086906c23104bdd71aeb51c5eb558"), + Asset( + path: "L1024_K80/config.json", sha256: "83ca9a1e92c1f5d43cb7731ef2ebcc95520755d2b5a7eb41692eddd3da88e084"), + Asset( + path: "L1024_K80/KevRow_fp16.mlpackage/Data/com.apple.CoreML/model.mlmodel", + sha256: "f72f7558cbf0e59d61621dd73bea866d721bc6a97991ef41b64e71071773e1ea"), + Asset( + path: "L1024_K80/KevRow_fp16.mlpackage/Data/com.apple.CoreML/weights/weight.bin", + sha256: "47b57aee6be1bde59e3ad4779897888710e63c425b6aabb8b09c0e1f149273c9"), + Asset( + path: "L1024_K80/KevRow_fp16.mlpackage/Manifest.json", + sha256: "e5d266706c569f7ad1201c06334f17a31158cf83c400d89b1ba2ea661b348b51"), + Asset(path: "L512_K16/config.json", sha256: "c734c4c44adfb669d226b57d75ec5532f6778c58cce0cacd51ebffdf0569c2b7"), + Asset( + path: "L512_K16/embeddings.f16", sha256: "2146dc176cff21240562283cf0b709a91499917d5b5f2c4fb0b7b41b21d407b0"), + Asset( + path: "L512_K16/KevRow_fp16.mlpackage/Data/com.apple.CoreML/model.mlmodel", + sha256: "6a7f904d6fbecc84210ac1399f7aada1573af744c795f6dbc9c5206cb3895757"), + Asset( + path: "L512_K16/KevRow_fp16.mlpackage/Data/com.apple.CoreML/weights/weight.bin", + sha256: "d4607666040c82d5b1b3cd2409e1a7aa279acfe56a28ba6090dc8a0270e3e2a2"), + Asset( + path: "L512_K16/KevRow_fp16.mlpackage/Manifest.json", + sha256: "f772d07c1da6795d99333c982e7697815c9420e108a289b11fed06ba2498b1cd"), + Asset(path: "tokenizer.json", sha256: "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523"), + ] + + /// Ensure the snapshot exists in the FluidUse cache and return its directory. Files are checksummed once per + /// pinned revision; later launches only check that they are present. + public static func ensure(cacheDirectory: URL? = nil, progress: Progress? = nil) async throws -> URL { + let root = cacheDirectory ?? LayaModelStore.defaultCacheDirectory() + let directory = root.appendingPathComponent("kev-0.8b-coreml", isDirectory: true) + let manager = FileManager.default + let verified = directory.appendingPathComponent(".verified-\(revision)") + if manager.fileExists(atPath: verified.path), + assets.allSatisfy({ manager.fileExists(atPath: directory.appendingPathComponent($0.path).path) }) + { + return directory + } + try manager.createDirectory(at: directory, withIntermediateDirectories: true) + for asset in assets { + try Task.checkCancellation() + let destination = directory.appendingPathComponent(asset.path) + if manager.fileExists(atPath: destination.path), try checksum(of: destination) == asset.sha256 { + continue + } + try manager.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + progress?(asset.path, 0) + let escaped = asset.path.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? asset.path + guard let url = URL(string: "https://huggingface.co/\(repository)/resolve/\(revision)/\(escaped)") else { + throw KevError.invalidAsset("Invalid Hugging Face asset URL for \(asset.path)") + } + let (temporary, response) = try await URLSession.shared.download(from: url) + defer { try? manager.removeItem(at: temporary) } + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw KevError.invalidAsset("Download failed for \(asset.path)") + } + let actual = try checksum(of: temporary) + guard actual == asset.sha256 else { + throw KevError.invalidAsset( + "Checksum mismatch for \(asset.path): expected \(asset.sha256), got \(actual)") + } + let size = (try manager.attributesOfItem(atPath: temporary.path)[.size] as? NSNumber)?.int64Value ?? 0 + try LayaModelStore.installDownloadedFile(temporary, at: destination) + progress?(asset.path, size) + } + try Data().write(to: verified) + return directory + } + + private static func checksum(of file: URL) throws -> String { + let handle = try FileHandle(forReadingFrom: file) + defer { try? handle.close() } + var digest = SHA256() + while let chunk = try handle.read(upToCount: 4_194_304), !chunk.isEmpty { + digest.update(data: chunk) + } + return digest.finalize().map { String(format: "%02x", $0) }.joined() + } +} diff --git a/Sources/FluidUse/Qwen/QwenBPETokenizer.swift b/Sources/FluidUse/Qwen/QwenBPETokenizer.swift new file mode 100644 index 0000000..7026c6b --- /dev/null +++ b/Sources/FluidUse/Qwen/QwenBPETokenizer.swift @@ -0,0 +1,166 @@ +import Foundation +import os + +/// Byte-level BPE tokenizer for Qwen `tokenizer.json` files (Qwen2 through Qwen3.5): NFC normalization, added tokens +/// matched literally, Qwen's split regex, GPT-2 byte-to-unicode mapping, then merges by rank. Encoding only; no special +/// tokens are added. +public final class QwenBPETokenizer: Sendable { + private let vocabulary: [String: Int] + private let mergeRanks: [String: Int] + /// Added tokens, longest first, so a longer token wins over one it contains. + private let addedTokens: [(content: String, id: Int)] + private let byteToCharacter: [Character] + /// Encoded pieces; ordinary text repeats the same words, so this saves most of the merge loops. + private let cache = OSAllocatedUnfairLock<[String: [Int]]>(initialState: [:]) + + public init(tokenizerJsonURL: URL) throws { + let data = try Data(contentsOf: tokenizerJsonURL) + guard let root = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let model = root["model"] as? [String: Any], model["type"] as? String == "BPE", + let vocab = model["vocab"] as? [String: Int], let merges = model["merges"] as? [Any], + let added = root["added_tokens"] as? [[String: Any]] + else { throw QwenTokenizerError.invalidAsset("Expected a Hugging Face byte-level BPE tokenizer.json") } + guard (model["byte_fallback"] as? Bool ?? false) == false else { + throw QwenTokenizerError.invalidAsset("byte_fallback BPE is not supported") + } + vocabulary = vocab + var ranks: [String: Int] = [:] + ranks.reserveCapacity(merges.count) + for (rank, merge) in merges.enumerated() { + let pair: [String] + if let list = merge as? [String] { + pair = list + } else if let text = merge as? String { + pair = text.split(separator: " ", maxSplits: 1).map(String.init) + } else { + pair = [] + } + guard pair.count == 2 else { throw QwenTokenizerError.invalidAsset("Malformed merge \(rank)") } + ranks[pair[0] + "\u{0}" + pair[1]] = rank + } + mergeRanks = ranks + addedTokens = added.compactMap { entry in + guard let content = entry["content"] as? String, let id = entry["id"] as? Int else { return nil } + return (content, id) + }.sorted { $0.content.count > $1.content.count } + byteToCharacter = Self.bytesToUnicode() + _ = try Self.splitter() + } + + /// Qwen's pre-tokenizer split. Built per call: NSRegularExpression is not Sendable. + private static func splitter() throws -> NSRegularExpression { + try NSRegularExpression( + pattern: + #"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"# + ) + } + + /// Token ids for `text`, without any special tokens around it. + public func encode(_ text: String) throws -> [Int] { + let splitter = try Self.splitter() + var ids: [Int] = [] + for (segment, addedID) in splitAddedTokens(text.precomposedStringWithCanonicalMapping) { + if let addedID { + ids.append(addedID) + continue + } + let string = segment as NSString + for match in splitter.matches(in: segment, range: NSRange(location: 0, length: string.length)) { + ids += encodePiece(string.substring(with: match.range)) + } + } + return ids + } + + public func id(for token: String) -> Int? { + addedTokens.first { $0.content == token }?.id ?? vocabulary[token] + } + + private func splitAddedTokens(_ text: String) -> [(String, Int?)] { + var parts: [(String, Int?)] = [] + var rest = Substring(text) + while !rest.isEmpty { + var earliest: (range: Range, id: Int)? + for (content, id) in addedTokens { + guard let range = rest.range(of: content, options: .literal) else { continue } + if earliest == nil || range.lowerBound < earliest!.range.lowerBound { + earliest = (range, id) + } + } + guard let found = earliest else { + parts.append((String(rest), nil)) + break + } + if found.range.lowerBound > rest.startIndex { + parts.append((String(rest[.. [Int] { + if let cached = cache.withLock({ $0[piece] }) { return cached } + var symbols = piece.utf8.map { String(byteToCharacter[Int($0)]) } + while symbols.count > 1 { + var best: (rank: Int, index: Int)? + for index in 0..<(symbols.count - 1) { + if let rank = mergeRanks[symbols[index] + "\u{0}" + symbols[index + 1]], + best == nil || rank < best!.rank + { + best = (rank, index) + } + } + guard let (_, index) = best else { break } + let merged = symbols[index] + symbols[index + 1] + // Merge every occurrence of this pair left to right, as the reference BPE does. + var next: [String] = [] + next.reserveCapacity(symbols.count) + var i = 0 + while i < symbols.count { + if i < symbols.count - 1, symbols[i] == symbols[index], symbols[i + 1] == symbols[index + 1] { + next.append(merged) + i += 2 + } else { + next.append(symbols[i]) + i += 1 + } + } + symbols = next + } + let ids = symbols.compactMap { vocabulary[$0] } + cache.withLock { storage in + if storage.count > 100_000 { storage.removeAll(keepingCapacity: true) } + storage[piece] = ids + } + return ids + } + + /// GPT-2's reversible byte -> printable character table. + private static func bytesToUnicode() -> [Character] { + var bytes = Array(33...126) + Array(161...172) + Array(174...255) + var codes = bytes + var extra = 0 + for byte in 0..<256 where !bytes.contains(byte) { + bytes.append(byte) + codes.append(256 + extra) + extra += 1 + } + var table = [Character](repeating: " ", count: 256) + for (byte, code) in zip(bytes, codes) { + table[byte] = Character(UnicodeScalar(code)!) + } + return table + } +} + +public enum QwenTokenizerError: Error, LocalizedError, Sendable { + case invalidAsset(String) + + public var errorDescription: String? { + switch self { + case .invalidAsset(let reason): "Invalid Qwen tokenizer asset: \(reason)" + } + } +} diff --git a/Sources/KevCheck/main.swift b/Sources/KevCheck/main.swift new file mode 100644 index 0000000..f3b047b --- /dev/null +++ b/Sources/KevCheck/main.swift @@ -0,0 +1,283 @@ +import FluidUse +import Foundation + +/// Kev-0.8B checks. +/// +/// swift run -c release KevCheck tokenizer +/// swift run -c release KevCheck parity +/// swift run -c release KevCheck serving (Kev's scripts/serving_bench.py cases) +/// swift run -c release KevCheck fast-parity (state-cache runtime) +/// swift run -c release KevCheck fast-serving +/// swift run -c release KevCheck bios (yes/no questions per state) +@main +struct KevCheck { + struct Fixture: Decodable { + let text: String + let ids: [Int] + } + + static func main() async throws { + let arguments = Array(CommandLine.arguments.dropFirst()) + if #available(macOS 15.0, *), arguments.first == "bios", arguments.count == 4 { + try await bios( + directory: URL(fileURLWithPath: arguments[1]), workload: URL(fileURLWithPath: arguments[2]), + output: URL(fileURLWithPath: arguments[3])) + return + } + if #available(macOS 15.0, *), arguments.first == "fast-serving", arguments.count == 2 { + let manager = try await KevFastManager.load(from: URL(fileURLWithPath: arguments[1])) + try await serving { try await manager.answer(state: $0, questions: $1, maxStateTokens: 8192) } + return + } + if #available(macOS 15.0, *), arguments.first == "fast-parity", arguments.count == 3 { + let manager = try await KevFastManager.load(from: URL(fileURLWithPath: arguments[1])) + try await parity(records: URL(fileURLWithPath: arguments[2])) { + try await manager.answer(state: $0, questions: $1) + } + return + } + if arguments.first == "serving", arguments.count == 2 { + try await serving(directory: URL(fileURLWithPath: arguments[1])) + return + } + if arguments.first == "parity", arguments.count == 3 { + try await parity(directory: URL(fileURLWithPath: arguments[1]), records: URL(fileURLWithPath: arguments[2])) + return + } + guard arguments.first == "tokenizer", arguments.count == 3 else { + fputs("usage: KevCheck tokenizer | parity \n", stderr) + exit(2) + } + let start = Date() + let tokenizer = try QwenBPETokenizer(tokenizerJsonURL: URL(fileURLWithPath: arguments[1])) + print(String(format: "loaded tokenizer in %.2f s", Date().timeIntervalSince(start))) + let fixtures = try JSONDecoder().decode( + [Fixture].self, from: Data(contentsOf: URL(fileURLWithPath: arguments[2]))) + var mismatches = 0 + var tokens = 0 + let encodeStart = Date() + for fixture in fixtures { + let ids = try tokenizer.encode(fixture.text) + tokens += ids.count + if ids != fixture.ids { + mismatches += 1 + if mismatches <= 5 { + let prefix = zip(ids, fixture.ids).prefix { $0 == $1 }.count + print("MISMATCH at token \(prefix): \(fixture.text.prefix(80).debugDescription)") + print( + " swift \(Array(ids.dropFirst(prefix).prefix(8))) hf \(Array(fixture.ids.dropFirst(prefix).prefix(8)))" + ) + } + } + } + print( + String( + format: "fixtures %d, tokens %d, mismatches %d, encode %.2f s", fixtures.count, tokens, mismatches, + Date().timeIntervalSince(encodeStart))) + exit(mismatches == 0 ? 0 : 1) + } + + /// Kev's System One question JSON -> KevQuestion, as `kev.api.to_record` reads it. + static func question(_ json: [String: Any]) throws -> KevQuestion { + let instructions = json["instructions"] as? String ?? "" + func text(_ value: Any?) throws -> String? { + switch value { + case nil, is NSNull: return nil + case let string as String: return string + case let number as NSNumber: return number.stringValue + default: throw KevError.invalidAsset("non-text criteria are not supported by this check") + } + } + switch json["type"] as? String { + case "choice": + guard let criteria = json["criteria"] as? [[Any]] else { throw KevError.invalidAsset("choice criteria") } + return .choice( + instructions, options: try criteria.map { (key: $0[0] as? String ?? "", description: try text($0[1])) }) + case "noul": + let criteria = json["criteria"] as? [String: Any] ?? [:] + return .noul(instructions, no: try text(criteria["false"]), yes: try text(criteria["true"])) + case "score": + guard let levels = json["criteria"] as? [Any] else { throw KevError.invalidAsset("score levels") } + return .score(instructions, levels: try levels.map { try text($0) ?? "" }) + default: + throw KevError.invalidAsset("unknown question type") + } + } + + static func parity(directory: URL, records: URL) async throws { + let start = Date() + let manager = try await KevManager.load(from: directory) + print(String(format: "loaded in %.1f s", Date().timeIntervalSince(start))) + try await parity(records: records) { try await manager.answer(state: $0, questions: $1) } + } + + static func parity( + records: URL, answer: @Sendable (String, [KevQuestion]) async throws -> [KevAnswer] + ) async throws { + guard let rows = try JSONSerialization.jsonObject(with: Data(contentsOf: records)) as? [[String: Any]] else { + throw KevError.invalidAsset("records.json") + } + var questions = 0 + var flips = 0 + var worst: Float = 0 + var skipped = 0 + let runStart = Date() + for row in rows { + guard let state = row["state"] as? String, let specs = row["questions"] as? [[String: Any]], + let expected = row["probabilities"] as? [[Double]] + else { continue } + let typed: [KevQuestion] + do { + typed = try specs.map(question) + } catch { + skipped += 1 + continue + } + let answers = try await answer(state, typed) + for (answer, reference) in zip(answers, expected) { + questions += 1 + let difference = zip(answer.probabilities, reference).map { abs($0 - Float($1)) }.max() ?? 0 + worst = max(worst, difference) + if difference > 0.01 { + print( + " |dp| \(difference) keys \(answer.keys) swift \(answer.probabilities.map { ($0 * 1000).rounded() / 1000 }) " + + "python \(reference.map { ($0 * 1000).rounded() / 1000 }) tokens \(answer.tokens)") + } + let referenceBest = reference.indices.max { reference[$0] < reference[$1] } ?? 0 + if answer.bestIndex != referenceBest { flips += 1 } + } + } + print( + String( + format: "questions %d, top-answer flips %d, max |dp| %.5f, skipped records %d, %.1f ms/question", + questions, + flips, worst, skipped, 1000 * Date().timeIntervalSince(runStart) / Double(max(questions, 1)))) + } + + /// Timed run of yes/no records (`[{"state", "questions": {"q0": {"instructions"}, ...}}]`) through the fused path. + @available(macOS 15.0, *) + static func bios(directory: URL, workload: URL, output: URL) async throws { + guard let records = try JSONSerialization.jsonObject(with: Data(contentsOf: workload)) as? [[String: Any]] + else { throw KevError.invalidAsset("workload") } + func seconds(since start: UInt64) -> Double { Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 } + var start = DispatchTime.now().uptimeNanoseconds + let manager = try await KevFastManager.load(from: directory) + let load = seconds(since: start) + start = DispatchTime.now().uptimeNanoseconds + try await manager.warm() + let warm = seconds(since: start) + var times: [Double] = [] + var probabilities: [[Float]] = [] + let runStart = DispatchTime.now().uptimeNanoseconds + for record in records { + guard let state = record["state"] as? String, + let questions = record["questions"] as? [String: [String: Any]] + else { continue } + let typed = (0.. [(name: String, state: String, questions: [KevQuestion])] { + let department = KevQuestion.choice( + "Which team should handle this?", + options: [ + ("returns", "Exchanges, refunds, wrong or damaged items"), + ("shipping", "Delivery status, delays, lost packages"), + ("billing", "Charges, invoices, payment problems"), + ]) + let returnReason = KevQuestion.choice( + "If the customer wants to return something, why?", + options: [ + ("wrong_size", "The item doesn't fit"), ("wrong_item", "A different product was delivered"), + ("damaged", "The item arrived broken or faulty"), + ("changed_mind", "The item is fine, the customer no longer wants it"), + ("other", "A return reason that fits none of the above"), + ]) + let resolution = KevQuestion.choice( + "What does the customer want to happen?", + options: [ + ("exchange", "Swap the item for a different one"), ("refund", "Money back"), + ("replacement", "The same item sent again"), ("information", "Just an answer, no action needed"), + ]) + let tone = KevQuestion.choice( + "What is the customer's tone?", options: [("calm", nil), ("frustrated", nil), ("angry", nil)]) + let escalate = KevQuestion.noul("Does this message require urgent human attention?") + let frustration = KevQuestion.score( + "How frustrated is the customer?", levels: ["Calm", "Frustrated", "Very angry"]) + let ticket = + "Shoes arrived two weeks late and in the wrong size. Also I see two charges on my card. What are you going to do about this?" + let paragraph = + "I ordered a pair of running shoes on the first of the month and paid with my credit card. The confirmation email said " + + "delivery in three to five business days, but the tracking page did not update for over a week, and when the package " + + "finally arrived the box was crushed on one side. The shoes inside were a size ten instead of the size nine I ordered. " + let five = [department, returnReason, resolution, escalate, frustration] + return [ + ("2 questions, short state", ticket, [department, escalate]), + ("6 questions, short state", ticket, [department, returnReason, resolution, tone, escalate, frustration]), + ("5 questions, 370-token state", String(repeating: paragraph, count: 5), five), + ("5 questions, 2,200-token state", String(repeating: paragraph, count: 30), five), + ] + } + + static func serving(answer: @Sendable (String, [KevQuestion]) async throws -> [KevAnswer]) async throws { + for (name, state, questions) in servingCases() { + do { + _ = try await answer(state, questions) + } catch { + print("\(name): \(error)") + continue + } + var times: [Double] = [] + for i in 1...22 { + let start = DispatchTime.now().uptimeNanoseconds + _ = try await answer("Ticket \(i). \(state)", questions) + if i > 2 { times.append(Double(DispatchTime.now().uptimeNanoseconds - start) / 1e6) } + } + times.sort() + print( + "\(name.padding(toLength: 32, withPad: " ", startingAt: 0)) median \(String(format: "%.1f", times[times.count / 2])) ms" + ) + } + } + + static func serving(directory: URL) async throws { + let manager = try await KevManager.load(from: directory) + for (name, state, questions) in servingCases() { + _ = try await manager.answer(state: state, questions: questions, maxStateTokens: 8192) + for concurrent in [false, true] { + var times: [Double] = [] + for i in 1...22 { + let text = "Ticket \(i). \(state)" + let start = DispatchTime.now().uptimeNanoseconds + _ = + concurrent + ? try await manager.answerConcurrently(state: text, questions: questions, maxStateTokens: 8192) + : try await manager.answer(state: text, questions: questions, maxStateTokens: 8192) + if i > 2 { times.append(Double(DispatchTime.now().uptimeNanoseconds - start) / 1e6) } + } + times.sort() + print( + "\(name.padding(toLength: 32, withPad: " ", startingAt: 0)) \(concurrent ? "concurrent" : "sequential") median \(String(format: "%.1f", times[times.count / 2])) ms" + ) + } + } + } +} diff --git a/Sources/KevGuessWhoDemo/ContentView.swift b/Sources/KevGuessWhoDemo/ContentView.swift new file mode 100644 index 0000000..7574ad6 --- /dev/null +++ b/Sources/KevGuessWhoDemo/ContentView.swift @@ -0,0 +1,192 @@ +import SortAnything +import SwiftUI + +struct ContentView: View { + @EnvironmentObject var model: GuessWhoModel + + private let columns = [GridItem(.adaptive(minimum: 116), spacing: 8)] + + var body: some View { + VStack(spacing: 0) { + header + HStack(alignment: .top, spacing: 16) { + ScrollView { + LazyVGrid(columns: columns, spacing: 8) { + ForEach(model.cards) { card in + CardView(card: card, isSecret: model.secret == card.id, solved: isSolved) + } + } + .padding(.bottom, 8) + } + .frame(maxWidth: .infinity) + sidebar.frame(width: 280) + } + .padding(16) + Text(DBpediaSample.attribution + " · questions answered by Kev-0.8B (jaredpalmer/kev, Apache-2.0)") + .font(.caption2).foregroundStyle(.secondary).padding(.bottom, 8) + } + .background(Color(red: 0.07, green: 0.08, blue: 0.10)) + .foregroundStyle(.white) + } + + private var isSolved: Bool { + if case .solved = model.phase { return true } + return false + } + + private var header: some View { + VStack(alignment: .leading, spacing: 10) { + HStack(alignment: .center, spacing: 20) { + Text("GUESS WHO").font(.system(size: 13, weight: .heavy)).tracking(3).foregroundStyle(.orange) + .fixedSize() + controls + Spacer(minLength: 12) + stat(String(format: "%.1f ms", model.lastCallMs), "last call") + stat(String(format: "%.1f ms", model.medianCallMs), "median call") + stat(String(format: "%.0f/s", model.scanRate), "decisions / s") + stat("\(model.totalDecisions)", "decisions") + } + // second row, full width: a changing status never moves the controls or metrics + Text(status).font(.system(size: 30, weight: .bold)).lineLimit(1).truncationMode(.tail) + .frame(maxWidth: .infinity, alignment: .leading) + } + .padding(.horizontal, 20).padding(.vertical, 14) + .background(Color.white.opacity(0.04)) + } + + private var status: String { + switch model.phase { + case .loading(let text): text + case .dealing: "Dealing \(model.cards.count) new cards…" + case .scanning: "Reading \(model.cards.count) bios · \(GuessWhoModel.questions.count) questions each" + case .asking(let question): question + case .solved(let name): "It's \(name)!" + case .failed(let error): "Error: \(error)" + } + } + + private func stat(_ value: String, _ label: String) -> some View { + VStack(alignment: .trailing, spacing: 2) { + Text(value).font(.system(size: 22, weight: .semibold, design: .monospaced)).lineLimit(1) + Text(label).font(.caption).foregroundStyle(.secondary).lineLimit(1) + } + .frame(width: 112, alignment: .trailing) + } + + private var controls: some View { + HStack(spacing: 8) { + Button { + model.togglePause() + } label: { + Label(model.paused ? "Play" : "Pause", systemImage: model.paused ? "play.fill" : "pause.fill") + .frame(width: 78) + } + .keyboardShortcut(.space, modifiers: []) + .focusable(false) + .help("Play / pause (Space)") + Button { + model.reset() + } label: { + Label("Reset", systemImage: "arrow.counterclockwise").frame(width: 78) + } + .keyboardShortcut("r", modifiers: [.command]) + .focusable(false) + .help("New wall (⌘R)") + } + .buttonStyle(.borderedProminent) + .controlSize(.large) + .tint(Color.orange.opacity(0.85)) + .disabled(!model.ready) + } + + private var sidebar: some View { + VStack(alignment: .leading, spacing: 14) { + Text( + "Game \(max(model.setGame, 1)) of \(GuessWhoModel.gamesPerPlay) · \(model.remaining) of \(model.cards.count) left" + ) + .font(.headline) + if model.scanSeconds > 0 { + Text( + String( + format: "Scan: %d bios × %d questions = %d decisions in %.2f s", model.scanned, + GuessWhoModel.questions.count, model.scanned * GuessWhoModel.questions.count, model.scanSeconds) + ) + .font(.callout).foregroundStyle(.secondary) + } + Divider().overlay(Color.white.opacity(0.2)) + ForEach(Array(model.asked.enumerated()), id: \.offset) { index, turn in + HStack(alignment: .top, spacing: 8) { + Text("\(index + 1).").monospacedDigit().foregroundStyle(.secondary) + VStack(alignment: .leading, spacing: 2) { + Text(turn.question).font(.callout) + Text("\(turn.answer ? "YES" : "NO") · \(turn.removed) flipped down") + .font(.caption.weight(.semibold)) + .foregroundStyle(turn.answer ? .green : .red) + } + } + .transition(.move(edge: .leading).combined(with: .opacity)) + } + Spacer() + Text( + "One fused Core ML call per card answers every question on that bio; each turn asks the question that splits the cards still up closest to half." + ) + .font(.caption).foregroundStyle(.secondary) + } + .animation(.easeOut(duration: 0.3), value: model.asked.count) + } +} + +struct CardView: View { + let card: GuessWhoModel.Card + let isSecret: Bool + let solved: Bool + + var body: some View { + VStack(alignment: .leading, spacing: 4) { + HStack(spacing: 4) { + Text(icon).font(.system(size: 13)) + Text(card.item.title).font(.system(size: 12, weight: .semibold)).lineLimit(2) + } + Spacer(minLength: 0) + HStack(spacing: 2) { + ForEach(0.. Color { + guard let answers = card.answers else { return Color.white.opacity(0.1) } + return answers[q] ? Color.green : Color.white.opacity(0.25) + } +} diff --git a/Sources/KevGuessWhoDemo/GuessWhoModel.swift b/Sources/KevGuessWhoDemo/GuessWhoModel.swift new file mode 100644 index 0000000..3a9bb90 --- /dev/null +++ b/Sources/KevGuessWhoDemo/GuessWhoModel.swift @@ -0,0 +1,318 @@ +import Foundation +import FluidUse +import SortAnything +import SwiftUI + +/// Blocks the inference producer and the turn pacing while the demo is paused. +actor PauseGate { + private var paused = false + + func set(_ value: Bool) { paused = value } + + func wait() async throws { + while paused { + try await Task.sleep(for: .milliseconds(50)) + } + } +} + +/// A self-playing Guess Who: Kev reads every card's Wikipedia abstract once, answering every question in the pool in +/// one fused Core ML call per card, then plays turns until one card (the hidden person) is left. +@MainActor +final class GuessWhoModel: ObservableObject { + struct Card: Identifiable { + let id: Int + let item: SortItem + /// Kev's yes/no per pool question, filled by the scan. + var answers: [Bool]? + var down = false + var flipping = false + } + + enum Phase: Equatable { + case loading(String) + case dealing + case scanning + case asking(String) + case solved(String) + case failed(String) + } + + static let questions = [ + "Is this person an athlete?", + "Is this person a musician or singer?", + "Is this person a politician?", + "Is this person a woman?", + "Was this person born before 1950?", + "Is this person from the United States?", + "Is this person from Europe?", + "Does this person play football (soccer)?", + "Is this person a painter or visual artist?", + "Is this person an actor?", + "Has this person competed at the Olympic Games?", + "Is this person a writer or poet?", + ] + static let cardsPerGame = 80 + static let gamesPerPlay = 4 + + @Published var cards: [Card] = [] + @Published var phase: Phase = .loading("Loading Kev-0.8B…") + @Published var secret: Int? + @Published var asked: [(question: String, answer: Bool, removed: Int)] = [] + @Published var game = 0 + /// Position of the current game in the set started by Play. + @Published var setGame = 0 + @Published var scanned = 0 + @Published var scanSeconds: Double = 0 + @Published var lastCallMs: Double = 0 + @Published var medianCallMs: Double = 0 + @Published var totalDecisions = 0 + @Published var paused = false + @Published var ready = false + + /// KevFastManager.answer (macOS 15+; the package targets macOS 14). + private var answer: (@Sendable (String, [KevQuestion]) async throws -> [KevAnswer])? + /// KevFastManager.warm: a function left idle pays a ~0.3–0.8 s re-setup on its next call, so each scan re-warms + /// first instead of stalling mid-wall. + private var warm: (@Sendable () async throws -> Void)? + private var people: [SortItem] = [] + private var callTimes: [Double] = [] + private var started = false + private var runner: Task? + private let gate = PauseGate() + + /// Decisions per wall-clock second in this game's scan. + var scanRate: Double { scanSeconds > 0 ? Double(scanned * Self.questions.count) / scanSeconds : 0 } + var remaining: Int { cards.filter { !$0.down }.count } + + func start() async { + guard !started else { return } + started = true + do { + guard #available(macOS 15.0, *) else { + phase = .failed("Kev's fused Core ML path needs macOS 15") + return + } + let directory: URL + if let local = Self.localModelDirectory() { + directory = local + } else { + phase = .loading("Downloading Kev-0.8B Core ML (FluidInference/kev-0.8b-coreml)…") + directory = try await KevModelStore.ensure() + } + phase = .loading("Loading Kev-0.8B Core ML…") + let manager = try await KevFastManager.load(from: directory) + phase = .loading("Compiling the GPU functions…") + try await manager.warm() + answer = { try await manager.answer(state: $0, questions: $1) } + warm = { try await manager.warm() } + phase = .loading("Fetching Wikipedia people (DBpedia)…") + people = try await DBpediaSample.load(count: 960, seed: 7, classes: ["artist", "athlete", "politician"]) + log( + "ready · Kev-0.8B fused Core ML (GPU) · \(people.count) Wikipedia people · \(Self.questions.count) questions" + ) + // waits for Play: nothing runs until the presenter starts it + paused = true + phase = .loading("Ready — press Play") + ready = true + } catch { + phase = .failed(error.localizedDescription) + } + } + + func togglePause() { + guard ready else { return } + guard runner != nil else { + paused = false + log("playing") + run() + return + } + paused.toggle() + log(paused ? "paused" : "playing") + let value = paused + Task { await gate.set(value) } + } + + /// Abandons the current game and deals a fresh wall. + func reset() { + guard ready else { return } + runner?.cancel() + paused = false + Task { await gate.set(false) } + callTimes = [] + lastCallMs = 0 + medianCallMs = 0 + totalDecisions = 0 + scanSeconds = 0 + log("reset") + run() + } + + private func run() { + runner = Task { + do { + // a set of games per Play; it stops on the last result and Play (or Reset) deals a new set + for round in 1...Self.gamesPerPlay { + setGame = round + try await playGame() + if round < Self.gamesPerPlay { try await pause(for: .seconds(3)) } + } + runner = nil + paused = true + } catch is CancellationError { + } catch { + phase = .failed(error.localizedDescription) + } + } + } + + /// Sleeps `duration` of unpaused time. + private func pause(for duration: Duration) async throws { + try await gate.wait() + try await Task.sleep(for: duration) + try await gate.wait() + } + + private func playGame() async throws { + guard let answer else { return } + let offset = (game * Self.cardsPerGame) % max(people.count - Self.cardsPerGame, 1) + game += 1 + cards = people[offset..