From a620811351b944ea7c0ec1c79e03e6bd2e7bbebe Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 19:24:32 +0800 Subject: [PATCH 01/16] fix(encoder): snapshot options under lock in encode ReerJSONEncoder.encode read the stored `options` without taking optionsLock, while every setter mutates it under the lock. Concurrent configuration changes could therefore race with encode and crash with a use-after-free of the userInfo dictionary. Take one locked snapshot at the start of encode (matching ReerJSONDecoder) and use it for both the encoder implementation and the output formatting flags. Co-authored-by: Cursor --- Sources/ReerJSON/ReerJSONEncoder.swift | 12 +++++- Tests/ReerJSONTests/RegressionTests.swift | 51 +++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 Tests/ReerJSONTests/RegressionTests.swift diff --git a/Sources/ReerJSON/ReerJSONEncoder.swift b/Sources/ReerJSON/ReerJSONEncoder.swift index 50744b7..6974f28 100644 --- a/Sources/ReerJSON/ReerJSONEncoder.swift +++ b/Sources/ReerJSON/ReerJSONEncoder.swift @@ -177,11 +177,18 @@ open class ReerJSONEncoder { public init() {} + func optionsSnapshot() -> Options { + optionsLock.lock() + defer { optionsLock.unlock() } + return options + } + open func encode(_ value: T) throws -> Data { let doc = yyjson_mut_doc_new(nil)! defer { yyjson_mut_doc_free(doc) } - let encoder = JSONEncoderImpl(doc: doc, codingPath: [], options: options) + let encoder = JSONEncoderImpl(doc: doc, codingPath: [], options: optionsSnapshot()) + let outputFormatting = encoder.options.outputFormatting if let date = value as? Date { encoder.singleValue = try encoder.wrapDateValue(date, for: nil) @@ -242,7 +249,8 @@ open class ReerJSONEncoder { let doc = yyjson_mut_doc_new(nil)! defer { yyjson_mut_doc_free(doc) } - let encoder = JSONEncoderImpl(doc: doc, codingPath: [], options: options) + let encoder = JSONEncoderImpl(doc: doc, codingPath: [], options: optionsSnapshot()) + let outputFormatting = encoder.options.outputFormatting try value.encode(to: encoder, configuration: configuration) guard let root = encoder.takeValue() else { diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift new file mode 100644 index 0000000..54b8bf8 --- /dev/null +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -0,0 +1,51 @@ +// +// Copyright © 2026 reers. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +import Foundation +import Testing +@testable import ReerJSON + +@Suite("Regression") +struct RegressionTests { + + @Test func encoderOptionsAreReadUnderLock() throws { + let encoder = ReerJSONEncoder() + let key = CodingUserInfoKey(rawValue: "k")! + nonisolated(unsafe) let unsafeEncoder = encoder + let group = DispatchGroup() + let writer = Thread { + for i in 0..<20_000 { + unsafeEncoder.userInfo = [key: i] + unsafeEncoder.keyEncodingStrategy = i % 2 == 0 ? .convertToSnakeCase : .useDefaultKeys + } + group.leave() + } + let reader = Thread { + for i in 0..<20_000 { + _ = try? unsafeEncoder.encode(["fooBar": i]) + } + group.leave() + } + group.enter(); group.enter() + writer.start(); reader.start() + group.wait() + } +} From aaf9695fff68358c3265573d8e8ad93c388405a7 Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 19:27:09 +0800 Subject: [PATCH 02/16] perf(stream): bound per-item parse window to fix O(K^2) streaming JSONStreamParser handed yyjson every byte from the read offset to the end of the buffer for each item. In non-INSITU mode yyjson copies its whole input, so K buffered items cost O(K * N) bytes of memcpy and the parse time grew quadratically (16000 NDJSON items: 120 ms). Parse a window sized from the previous item (at least 256 bytes) and double it only when the window cannot decide the value by itself: incomplete input, a value ending exactly at the window end, or an error. Once the window covers all available bytes the parser behaves exactly as before, so boundary deferral and error reporting are unchanged. Co-authored-by: Cursor --- Sources/ReerJSON/StreamParser.swift | 77 ++++++++++++++------ Tests/ReerJSONTests/RegressionTests.swift | 88 +++++++++++++++++++++++ 2 files changed, 143 insertions(+), 22 deletions(-) diff --git a/Sources/ReerJSON/StreamParser.swift b/Sources/ReerJSON/StreamParser.swift index 1d6d52c..b6cca23 100644 --- a/Sources/ReerJSON/StreamParser.swift +++ b/Sources/ReerJSON/StreamParser.swift @@ -94,6 +94,7 @@ public struct JSONStreamParser: Sendable { private var buffer: Data private var readOffset: Int private var arrayState: ArrayParseState + private var lastItemSize: Int = 0 /// The number of bytes buffered but not yet consumed. public var pendingByteCount: Int { @@ -157,6 +158,7 @@ public struct JSONStreamParser: Sendable { buffer.removeAll(keepingCapacity: true) readOffset = 0 arrayState = .expectOpenBracket + lastItemSize = 0 } // MARK: - Private Types @@ -303,32 +305,63 @@ public struct JSONStreamParser: Sendable { let available = buffer.count - readOffset guard available > 0 else { return nil } - // yyjson_read_opts() in non-INSITU mode allocates its own padded buffer - // and copies the input — so we do not need to add YYJSON_PADDING_SIZE - // bytes ourselves here. + // yyjson_read_opts() in non-INSITU mode copies the whole input it is + // given, so handing it everything up to the buffer end would cost + // O(remaining bytes) per item and O(K²) for K buffered items. Parse a + // window sized from the previous item instead, and only widen it when + // the window cannot decide the value on its own. Once the window + // covers all available bytes the behavior is exactly that of a + // whole-buffer parse. let startOffset = readOffset - let result: DocumentRef.StreamParseResult = try buffer.withUnsafeBytes { rawBuffer in - guard let base = rawBuffer.baseAddress else { return .needMoreData } - let bytes = base.advanced(by: startOffset).assumingMemoryBound(to: UInt8.self) - return try DocumentRef.streamParse(bytes: bytes, count: available, options: options) - } + var window = min(available, max(Self.minimumParseWindow, lastItemSize &* 2)) + while true { + let isWholeBuffer = window == available + let result: DocumentRef.StreamParseResult + do { + result = try parseWindow(at: startOffset, count: window) + } catch where !isWholeBuffer { + window = min(available, window &* 2) + continue + } - switch result { - case .needMoreData: - return nil - case .success(let doc, let consumed): - let endOffset = startOffset + consumed - - // Boundary safety: - // A successful parse that ends exactly at the buffer end may be a - // truncated numeric token (e.g. we have "12" so far but the source - // is actually "123"). yyjson cannot tell the difference. Defer the - // value until either more data arrives or the caller finalizes. - if !finalizing && endOffset >= buffer.count { + switch result { + case .needMoreData: + guard isWholeBuffer else { + window = min(available, window &* 2) + continue + } return nil + case .success(let doc, let consumed): + // A value ending exactly at the window end may continue past it + // (e.g. "12" of "123"); only a whole-buffer parse can decide. + if consumed >= window && !isWholeBuffer { + window = min(available, window &* 2) + continue + } + let endOffset = startOffset + consumed + + // Boundary safety: + // A successful parse that ends exactly at the buffer end may be a + // truncated numeric token (e.g. we have "12" so far but the source + // is actually "123"). yyjson cannot tell the difference. Defer the + // value until either more data arrives or the caller finalizes. + if !finalizing && endOffset >= buffer.count { + return nil + } + readOffset = endOffset + lastItemSize = consumed + return ParsedItem(document: doc) } - readOffset = endOffset - return ParsedItem(document: doc) + } + } + + private static let minimumParseWindow = 256 + + private func parseWindow(at offset: Int, count: Int) throws -> DocumentRef.StreamParseResult { + try buffer.withUnsafeBytes { rawBuffer in + guard let base = rawBuffer.baseAddress else { return .needMoreData } + let bytes = base.advanced(by: offset).assumingMemoryBound(to: UInt8.self) + return try DocumentRef.streamParse(bytes: bytes, count: count, options: options) } } diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index 54b8bf8..6d37fda 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -48,4 +48,92 @@ struct RegressionTests { writer.start(); reader.start() group.wait() } + + // MARK: - JSONStreamParser + + private static func ndjson(count: Int) -> Data { + var s = "" + for i in 0.. Void) rethrows -> Double { + var best = Double.greatestFiniteMagnitude + for _ in 0..<3 { + let start = DispatchTime.now().uptimeNanoseconds + try body() + best = min(best, Double(DispatchTime.now().uptimeNanoseconds - start)) + } + return best + } + + @Test(arguments: [JSONStreamMode.jsonLines, .jsonArray]) + func streamParserScalesLinearlyWithItemCount(mode: JSONStreamMode) throws { + func payload(_ count: Int) -> Data { + guard mode == .jsonArray else { return Self.ndjson(count: count) } + var s = String(decoding: Self.ndjson(count: count), as: UTF8.self) + s.removeLast() + return Data(("[" + s.replacingOccurrences(of: "\n", with: ",") + "]").utf8) + } + let small = payload(2_000), large = payload(16_000) + func parseAll(_ data: Data, expected: Int) throws { + var parser = JSONStreamParser(mode: mode) + let count = try parser.parse(data).count + parser.finalize().count + #expect(count == expected) + } + let smallTime = try Self.bestTime { try parseAll(small, expected: 2_000) } + let largeTime = try Self.bestTime { try parseAll(large, expected: 16_000) } + // 8x the items: linear ≈ 8x, quadratic ≈ 64x. + #expect(largeTime / smallTime < 24, "ratio \(largeTime / smallTime)") + } + + @Test(arguments: [JSONStreamMode.jsonLines, .jsonArray]) + func streamParserHandlesMixedItemSizes(mode: JSONStreamMode) throws { + let sizes = [1, 5_000, 3, 70_000, 10, 10, 300_000, 2, 1_000] + var items: [String] = [] + for (i, size) in sizes.enumerated() { + items.append("{\"i\":\(i),\"s\":\"\(String(repeating: "x", count: size))\"}") + items.append("\(i * 1_000_003)") + } + let text = mode == .jsonLines + ? items.joined(separator: " \n") + : "[" + items.joined(separator: ", ") + "]" + let data = Data(text.utf8) + for chunkSize in [data.count, 4_096, 7] { + var parser = JSONStreamParser(mode: mode) + var values: [JSONValue] = [] + var offset = 0 + while offset < data.count { + let end = min(offset + chunkSize, data.count) + values += try parser.parse(data.subdata(in: offset.. Date: Thu, 24 Sep 2026 19:35:27 +0800 Subject: [PATCH 03/16] =?UTF-8?q?perf(serialization):=20single-pass,=20cas?= =?UTF-8?q?t-free=20Foundation=20=E2=86=92=20yyjson=20writer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReerJSONSerialization.data(withJSONObject:) was 11-60x slower than JSONSerialization. Profiling showed three causes: - yyjson_mut_obj_put scans the whole object on every insert, making wide dictionaries O(n^2). NSDictionary keys are unique, so obj_add is safe. - isValidJSONObject walked the full object graph before conversion walked it again. Validation now happens during conversion and still throws the same "Invalid JSON object" error. - Most of the remaining time was Swift dynamic casts from Any and NSString → String bridging copies. On Darwin, values are now dispatched with class checks on AnyObject, containers are read with CFDictionaryGetKeysAndValues / CFArrayGetValues, and strings are copied straight from CFString storage. Linux keeps the Any-based dispatch. Co-authored-by: Cursor --- Sources/ReerJSON/ReerJSONSerialization.swift | 266 ++++++++++++++----- Tests/ReerJSONTests/RegressionTests.swift | 69 +++++ 2 files changed, 267 insertions(+), 68 deletions(-) diff --git a/Sources/ReerJSON/ReerJSONSerialization.swift b/Sources/ReerJSON/ReerJSONSerialization.swift index 1a2bbb2..9c5177c 100644 --- a/Sources/ReerJSON/ReerJSONSerialization.swift +++ b/Sources/ReerJSON/ReerJSONSerialization.swift @@ -152,16 +152,16 @@ public enum ReerJSONSerialization { options.contains(.infAndNaNAsNull) || options.contains(.allowInfAndNaN) - if isTopLevelContainer { - guard isValidJSONObject(obj, allowNonFiniteNumbers: allowNonFiniteNumbers) else { - throw JSONError.invalidData("Invalid JSON object") - } - } else if isFragment { + // Containers are validated while they are converted. + if isFragment { guard options.contains(.fragmentsAllowed) else { throw JSONError.invalidData("Top-level JSON value must be an array or dictionary") } - } else { - throw JSONError.invalidData("Invalid JSON object") + if !allowNonFiniteNumbers, let number = obj as? NSNumber, !number.doubleValue.isFinite { + throw JSONError.invalidData("NaN or Infinity not allowed in JSON") + } + } else if !isTopLevelContainer { + throw invalidObjectError } guard let doc = yyjson_mut_doc_new(nil) else { @@ -233,13 +233,6 @@ public enum ReerJSONSerialization { // MARK: - Private Helpers - private static func isValidJSONObject(_ obj: Any, allowNonFiniteNumbers: Bool) -> Bool { - guard obj is NSArray || obj is NSDictionary else { - return false - } - return isValidJSONObjectRecursive(obj, allowNonFiniteNumbers: allowNonFiniteNumbers) - } - private static func isValidJSONObjectRecursive( _ obj: Any, allowNonFiniteNumbers: Bool @@ -327,80 +320,217 @@ public enum ReerJSONSerialization { return try value.data(options: writeOptions) } + private static let invalidObjectError = JSONError.invalidData("Invalid JSON object") + + /// Converts and validates a Foundation object graph in a single pass. + /// + /// Any value that `isValidJSONObject` would reject throws `invalidObjectError`. private static func foundationObjectToYYJSON( _ obj: Any, doc: UnsafeMutablePointer, options: WritingOptions ) throws -> UnsafeMutablePointer { - switch obj { - case let str as NSString: - return try yyFromString(str as String, in: doc) - - case let num as NSNumber: - let doubleValue = num.doubleValue - if doubleValue.isNaN || doubleValue.isInfinite { - if options.contains(.infAndNaNAsNull) { - return yyjson_mut_null(doc) + #if canImport(Darwin) + // Class checks on AnyObject are plain `isKindOfClass` calls, whereas + // casting `Any` goes through the much slower generic dynamic cast. + return try foundationObjectToYYJSON(object: obj as AnyObject, doc: doc, options: options) + #else + switch obj { + case let str as NSString: + return try yyFromString(str as String, in: doc) + case let num as NSNumber: + return try numberToYYJSON(num, doc: doc, options: options) + case is NSNull: + return yyjson_mut_null(doc) + case let arr as NSArray: + let jsonArr = try makeArray(doc) + for element in arr { + _ = yyjson_mut_arr_append(jsonArr, try foundationObjectToYYJSON(element, doc: doc, options: options)) } - if options.contains(.allowInfAndNaN) { - return yyjson_mut_real(doc, doubleValue) + return jsonArr + case let dict as NSDictionary: + let jsonObj = try makeObject(doc) + if options.contains(.sortedKeys) { + for (key, value) in try sortedEntries(of: dict) { + try addEntry(key: key, value: value, to: jsonObj, doc: doc, options: options) + } + } else { + for (key, value) in dict { + guard let keyString = key as? String else { throw invalidObjectError } + try addEntry(key: keyString, value: value, to: jsonObj, doc: doc, options: options) + } } - throw JSONError.invalidData("NaN or Infinity not allowed in JSON") + return jsonObj + default: + throw invalidObjectError } + #endif + } - if isBoolNumber(num) { - return yyjson_mut_bool(doc, num.boolValue) + #if canImport(Darwin) + private static func foundationObjectToYYJSON( + object: AnyObject, + doc: UnsafeMutablePointer, + options: WritingOptions + ) throws -> UnsafeMutablePointer { + if let str = object as? NSString { + return try stringToYYJSON(str, doc: doc) } - - let objCType = num.objCType.pointee - switch objCType { - case 0x63, 0x73, 0x69, 0x6C, 0x71: // 'c', 's', 'i', 'l', 'q' (signed integers) - return yyjson_mut_sint(doc, num.int64Value) - case 0x43, 0x53, 0x49, 0x4C, 0x51: // 'C', 'S', 'I', 'L', 'Q' (unsigned integers) - return yyjson_mut_uint(doc, num.uint64Value) - default: - return yyjson_mut_real(doc, doubleValue) + if let num = object as? NSNumber { + return try numberToYYJSON(num, doc: doc, options: options) } - - case is NSNull: - return yyjson_mut_null(doc) - - case let arr as NSArray: - guard let jsonArr = yyjson_mut_arr(doc) else { - throw JSONError.invalidData("Failed to create array") + if let dict = object as? NSDictionary { + let jsonObj = try makeObject(doc) + let count = CFDictionaryGetCount(dict) + try withUnsafeTemporaryAllocation(of: UnsafeRawPointer?.self, capacity: max(count * 2, 1)) { buffer in + let keys = buffer.baseAddress!, values = keys + count + CFDictionaryGetKeysAndValues(dict, keys, values) + if options.contains(.sortedKeys) { + var entries: [(key: String, nsKey: NSString, value: AnyObject)] = [] + entries.reserveCapacity(count) + for i in 0...fromOpaque(keys[i]!).takeUnretainedValue() + guard let keyString = key as? NSString else { throw invalidObjectError } + let value = Unmanaged.fromOpaque(values[i]!).takeUnretainedValue() + entries.append((keyString as String, keyString, value)) + } + entries.sort { $0.key < $1.key } + for entry in entries { + let keyVal = try stringToYYJSON(entry.nsKey, doc: doc) + let valueVal = try foundationObjectToYYJSON(object: entry.value, doc: doc, options: options) + _ = yyjson_mut_obj_add(jsonObj, keyVal, valueVal) + } + } else { + for i in 0...fromOpaque(keys[i]!).takeUnretainedValue() + guard let keyString = key as? NSString else { throw invalidObjectError } + let value = Unmanaged.fromOpaque(values[i]!).takeUnretainedValue() + let keyVal = try stringToYYJSON(keyString, doc: doc) + let valueVal = try foundationObjectToYYJSON(object: value, doc: doc, options: options) + _ = yyjson_mut_obj_add(jsonObj, keyVal, valueVal) + } + } + } + return jsonObj } - for element in arr { - let elementVal = try foundationObjectToYYJSON(element, doc: doc, options: options) - _ = yyjson_mut_arr_append(jsonArr, elementVal) + if let arr = object as? NSArray { + let jsonArr = try makeArray(doc) + let count = CFArrayGetCount(arr) + try withUnsafeTemporaryAllocation(of: UnsafeRawPointer?.self, capacity: max(count, 1)) { buffer in + CFArrayGetValues(arr, CFRange(location: 0, length: count), buffer.baseAddress!) + for element in buffer.prefix(count) { + let object = Unmanaged.fromOpaque(element!).takeUnretainedValue() + let elementVal = try foundationObjectToYYJSON(object: object, doc: doc, options: options) + _ = yyjson_mut_arr_append(jsonArr, elementVal) + } + } + return jsonArr } - return jsonArr - - case let dict as NSDictionary: - guard let jsonObj = yyjson_mut_obj(doc) else { - throw JSONError.invalidData("Failed to create object") + if object is NSNull { + return yyjson_mut_null(doc) } - - let keys: [Any] - if options.contains(.sortedKeys) { - keys = (dict.allKeys as? [String])?.sorted() ?? dict.allKeys - } else { - keys = dict.allKeys + throw invalidObjectError + } + + @inline(__always) + private static func stringToYYJSON( + _ string: NSString, + doc: UnsafeMutablePointer + ) throws -> UnsafeMutablePointer { + let cfString = string as CFString + // CFStringGetCStringPtr only succeeds for ASCII storage, where the + // UTF-16 length equals the UTF-8 byte count. + let length = CFStringGetLength(cfString) + let utf8 = CFStringBuiltInEncodings.UTF8.rawValue + if let ptr = CFStringGetCStringPtr(cfString, utf8), + let val = yyjson_mut_strncpy(doc, ptr, length) { + return val } - - for key in keys { - guard let keyString = key as? String else { - throw JSONError.invalidData("Dictionary keys must be strings") + let capacity = CFStringGetMaximumSizeForEncoding(length, utf8) + let val = withUnsafeTemporaryAllocation(of: CChar.self, capacity: max(capacity, 1)) { buffer -> UnsafeMutablePointer? in + let base = buffer.baseAddress! + var used: CFIndex = 0 + let converted = base.withMemoryRebound(to: UInt8.self, capacity: buffer.count) { + CFStringGetBytes(cfString, CFRange(location: 0, length: length), utf8, 0, false, $0, capacity, &used) } - guard let value = dict[key] else { continue } - let keyVal = try yyFromString(keyString, in: doc) - let valueVal = try foundationObjectToYYJSON(value, doc: doc, options: options) - _ = yyjson_mut_obj_put(jsonObj, keyVal, valueVal) + // A short conversion means unpaired surrogates; leave those to String bridging. + guard converted == length else { return nil } + return yyjson_mut_strncpy(doc, base, used) + } + if let val { + return val } - return jsonObj + return try yyFromString(string as String, in: doc) + } + #else + private static func addEntry( + key: String, + value: Any, + to jsonObj: UnsafeMutablePointer, + doc: UnsafeMutablePointer, + options: WritingOptions + ) throws { + let keyVal = try yyFromString(key, in: doc) + let valueVal = try foundationObjectToYYJSON(value, doc: doc, options: options) + _ = yyjson_mut_obj_add(jsonObj, keyVal, valueVal) + } + + /// Returns the entries sorted by key; throws if any key is not a string. + private static func sortedEntries(of dict: NSDictionary) throws -> [(key: String, value: Any)] { + var entries: [(key: String, value: Any)] = [] + entries.reserveCapacity(dict.count) + for (key, value) in dict { + guard let keyString = key as? String else { throw invalidObjectError } + entries.append((keyString, value)) + } + entries.sort { $0.key < $1.key } + return entries + } + #endif + @inline(__always) + private static func numberToYYJSON( + _ num: NSNumber, + doc: UnsafeMutablePointer, + options: WritingOptions + ) throws -> UnsafeMutablePointer { + if isBoolNumber(num) { + return yyjson_mut_bool(doc, num.boolValue) + } + switch num.objCType.pointee { + case 0x63, 0x73, 0x69, 0x6C, 0x71: // 'c', 's', 'i', 'l', 'q' (signed integers) + return yyjson_mut_sint(doc, num.int64Value) + case 0x43, 0x53, 0x49, 0x4C, 0x51: // 'C', 'S', 'I', 'L', 'Q' (unsigned integers) + return yyjson_mut_uint(doc, num.uint64Value) default: - throw JSONError.invalidData("Unsupported Foundation type: \(type(of: obj))") + let doubleValue = num.doubleValue + if !doubleValue.isFinite { + if options.contains(.infAndNaNAsNull) { + return yyjson_mut_null(doc) + } + if options.contains(.allowInfAndNaN) { + return yyjson_mut_real(doc, doubleValue) + } + throw invalidObjectError + } + return yyjson_mut_real(doc, doubleValue) + } + } + + @inline(__always) + private static func makeArray(_ doc: UnsafeMutablePointer) throws -> UnsafeMutablePointer { + guard let arr = yyjson_mut_arr(doc) else { + throw JSONError.invalidData("Failed to create array") + } + return arr + } + + @inline(__always) + private static func makeObject(_ doc: UnsafeMutablePointer) throws -> UnsafeMutablePointer { + guard let obj = yyjson_mut_obj(doc) else { + throw JSONError.invalidData("Failed to create object") } + return obj } } diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index 6d37fda..599c53f 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -136,4 +136,73 @@ struct RegressionTests { data.append(Self.ndjson(count: 10)) #expect(throws: JSONError.self) { try parser.parse(data) } } + + // MARK: - ReerJSONSerialization writing + + @Test func serializationWriteScalesLinearlyWithDictionaryWidth() throws { + func dictionary(_ count: Int) -> NSDictionary { + let dict = NSMutableDictionary() + for i in 0.. Date: Thu, 24 Sep 2026 19:36:12 +0800 Subject: [PATCH 04/16] chore: exclude unused test files from the test target SwiftPM warned about Info.plist, Models/instruments.json and Models/json_to_swift.rb on every build. None of them are loaded by the tests, so exclude them instead of bundling them as resources. Co-authored-by: Cursor --- Package.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Package.swift b/Package.swift index 3877685..e8202ff 100644 --- a/Package.swift +++ b/Package.swift @@ -67,6 +67,11 @@ let package = Package( .testTarget( name: "ReerJSONTests", dependencies: ["ReerJSON"], + exclude: [ + "Info.plist", + "Models/instruments.json", + "Models/json_to_swift.rb", + ], resources: [ .copy("Models/apache_builds.json"), .copy("Models/canada.json"), From a3e08a13f938f2df1be8db3617cf7e9b99bad890 Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 19:39:06 +0800 Subject: [PATCH 05/16] perf(encoder): O(1) keyed-container inserts for wide objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keyed containers insert with yyjson_mut_obj_put so that re-encoding a key replaces its value in place, but obj_put scans the whole object on every insert, making objects with many keys O(n^2) (2000 keys: 2.6 µs/key). Objects below 32 keys keep using obj_put. Once an object reaches 32 keys the encoder lazily builds a key set for it; new keys are appended with obj_add and only genuine duplicates fall back to obj_put, preserving the last-value-wins, original-position semantics. The index is shared with referencing encoders so superEncoder writes stay in sync. Co-authored-by: Cursor --- Sources/ReerJSON/JSONEncoderImpl.swift | 72 +++++++++++++++++++-- Tests/ReerJSONTests/RegressionTests.swift | 77 +++++++++++++++++++++++ 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/Sources/ReerJSON/JSONEncoderImpl.swift b/Sources/ReerJSON/JSONEncoderImpl.swift index 65c73a8..0bd0413 100644 --- a/Sources/ReerJSON/JSONEncoderImpl.swift +++ b/Sources/ReerJSON/JSONEncoderImpl.swift @@ -32,10 +32,46 @@ import yyjson import JJLISO8601DateFormatter #endif +/// Key sets for wide encoder objects, used to keep "last value wins" +/// insertion O(1) instead of scanning the whole object like +/// `yyjson_mut_obj_put` does. See `JSONEncoderImpl.putKey`. +final class ObjectKeyIndex { + static let threshold = 32 + private var keySets: [UnsafeMutablePointer: Set] = [:] + + func put( + _ key: String, + _ value: UnsafeMutablePointer, + into object: UnsafeMutablePointer, + impl: JSONEncoderImpl + ) { + let inserted = keySets[object, default: Self.existingKeys(of: object)].insert(key).inserted + if inserted { + yyjson_mut_obj_add(object, impl.wrapString(key), value) + } else { + yyjson_mut_obj_put(object, impl.wrapString(key), value) + } + } + + private static func existingKeys(of object: UnsafeMutablePointer) -> Set { + var keys = Set(minimumCapacity: Int(yyjson_mut_obj_size(object)) * 2) + var iter = yyjson_mut_obj_iter() + guard yyjson_mut_obj_iter_init(object, &iter) else { return keys } + while let keyVal = yyjson_mut_obj_iter_next(&iter) { + if let str = yyjson_mut_get_str(keyVal) { + let buffer = UnsafeRawBufferPointer(start: str, count: yyjson_mut_get_len(keyVal)) + keys.insert(String(decoding: buffer, as: UTF8.self)) + } + } + return keys + } +} + class JSONEncoderImpl: Encoder { let doc: UnsafeMutablePointer let options: ReerJSONEncoder.Options var codingPath: [CodingKey] + private var _objectKeyIndex: ObjectKeyIndex? var userInfo: [CodingUserInfoKey: Any] { options.userInfo } @@ -43,10 +79,36 @@ class JSONEncoderImpl: Encoder { var array: UnsafeMutablePointer? var object: UnsafeMutablePointer? - init(doc: UnsafeMutablePointer, codingPath: [CodingKey], options: ReerJSONEncoder.Options) { + init( + doc: UnsafeMutablePointer, + codingPath: [CodingKey], + options: ReerJSONEncoder.Options, + objectKeyIndex: ObjectKeyIndex? = nil + ) { self.doc = doc self.codingPath = codingPath self.options = options + self._objectKeyIndex = objectKeyIndex + } + + /// Shared by every encoder writing into this document's objects. + var objectKeyIndex: ObjectKeyIndex { + if let index = _objectKeyIndex { return index } + let index = ObjectKeyIndex() + _objectKeyIndex = index + return index + } + + /// Inserts `key` into a keyed-container object, replacing the value of an + /// existing equal key in place. Every insertion into such an object must + /// go through here so that `ObjectKeyIndex` stays in sync. + @inline(__always) + func putKey(_ key: String, _ value: UnsafeMutablePointer, into object: UnsafeMutablePointer) { + if yyjson_mut_obj_size(object) < ObjectKeyIndex.threshold { + yyjson_mut_obj_put(object, wrapString(key), value) + } else { + objectKeyIndex.put(key, value, into: object, impl: self) + } } @inline(__always) @@ -459,7 +521,7 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine @inline(__always) private func addToObject(key: String, value: UnsafeMutablePointer) { - yyjson_mut_obj_put(object, _strVal(key), value) + impl.putKey(key, value, into: object) } mutating func encodeNil(forKey key: Key) throws { addToObject(key: _key(key), value: yyjson_mut_null(doc)) } @@ -567,10 +629,10 @@ private class YYJSONReferencingEncoder: JSONEncoderImpl { let referencedObject: UnsafeMutablePointer init(impl: JSONEncoderImpl, key: String, codingPath: [CodingKey], object: UnsafeMutablePointer) { self.key = key; self.referencedObject = object - super.init(doc: impl.doc, codingPath: codingPath, options: impl.options) + super.init(doc: impl.doc, codingPath: codingPath, options: impl.options, objectKeyIndex: impl.objectKeyIndex) } deinit { - yyjson_mut_obj_put(referencedObject, wrapString(key), takeValue() ?? yyjson_mut_obj(doc)!) + putKey(key, takeValue() ?? yyjson_mut_obj(doc)!, into: referencedObject) } } @@ -579,7 +641,7 @@ private class YYJSONReferencingArrayEncoder: JSONEncoderImpl { let insertIndex: Int init(impl: JSONEncoderImpl, codingPath: [CodingKey], array: UnsafeMutablePointer, index: Int) { self.referencedArray = array; self.insertIndex = index - super.init(doc: impl.doc, codingPath: codingPath, options: impl.options) + super.init(doc: impl.doc, codingPath: codingPath, options: impl.options, objectKeyIndex: impl.objectKeyIndex) } deinit { yyjson_mut_arr_insert(referencedArray, takeValue() ?? yyjson_mut_obj(doc)!, insertIndex) diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index 599c53f..edfa5f0 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -186,6 +186,83 @@ struct RegressionTests { #expect(String(decoding: data, as: UTF8.self) == "[\"a\u{FFFD}b\"]") } + // MARK: - Encoder keyed containers + + private struct DynamicKey: CodingKey { + var stringValue: String + var intValue: Int? { nil } + init(_ string: String) { stringValue = string } + init(stringValue: String) { self.stringValue = stringValue } + init?(intValue: Int) { nil } + } + + private struct WideObject: Encodable { + let count: Int + var overrides: [(key: Int, value: Int)] = [] + var superKeys: [Int] = [] + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: DynamicKey.self) + for i in 0.. Date: Thu, 24 Sep 2026 19:40:51 +0800 Subject: [PATCH 06/16] fix(encoder): restore codingPath when a dictionary value throws wrapStringKeyedDictValue restored codingPath only on the success path. If a value threw and the caller recovered (e.g. `try? container.encode`), the dictionary key stayed on the path, so later errors and .custom key strategies saw a wrong coding path ([dict, value] instead of [value], unlike Foundation). Pop the key in a defer. Co-authored-by: Cursor --- Sources/ReerJSON/JSONEncoderImpl.swift | 3 +-- Tests/ReerJSONTests/RegressionTests.swift | 30 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/Sources/ReerJSON/JSONEncoderImpl.swift b/Sources/ReerJSON/JSONEncoderImpl.swift index 0bd0413..5856092 100644 --- a/Sources/ReerJSON/JSONEncoderImpl.swift +++ b/Sources/ReerJSON/JSONEncoderImpl.swift @@ -409,14 +409,13 @@ class JSONEncoderImpl: Encoder { func wrapStringKeyedDictValue(_ dict: [String: Encodable], for additionalKey: CodingKey?) throws -> UnsafeMutablePointer? { let obj = yyjson_mut_obj(doc)! - let savedPath = codingPath if let additionalKey { codingPath.append(additionalKey) } + defer { if additionalKey != nil { codingPath.removeLast() } } for (key, value) in dict { let keyVal = wrapString(key) let val = try wrapEncodable(value, for: _CodingKey(stringValue: key)!) ?? yyjson_mut_obj(doc)! yyjson_mut_obj_add(obj, keyVal, val) } - codingPath = savedPath return obj } diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index edfa5f0..9609bce 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -263,6 +263,36 @@ struct RegressionTests { #expect(value["key_1"]?.int64 == 1) } + private struct Thrower: Encodable { + func encode(to encoder: Encoder) throws { + throw EncodingError.invalidValue(0, .init(codingPath: encoder.codingPath, debugDescription: "boom")) + } + } + + private struct RecoversFromDictionaryError: Encodable { + enum CodingKeys: String, CodingKey { case dict, value } + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + _ = try? container.encode(["inner": Thrower()], forKey: .dict) + try container.encode(Thrower(), forKey: .value) + } + } + + @Test func encoderRestoresCodingPathAfterDictionaryError() throws { + func errorPath(_ encode: () throws -> Data) -> [String] { + do { + _ = try encode() + } catch let EncodingError.invalidValue(_, context) { + return context.codingPath.map(\.stringValue) + } catch {} + return [""] + } + let value = RecoversFromDictionaryError() + let foundation = errorPath { try JSONEncoder().encode(value) } + #expect(foundation == ["value"]) + #expect(errorPath { try ReerJSONEncoder().encode(value) } == foundation) + } + @Test func serializationWriteRejectsInvalidObjects() throws { let invalid = JSONError.invalidData("Invalid JSON object") let cases: [Any] = [ From 14477574c11e22ed5f1642c6bde3c9d0f20967b5 Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 19:45:04 +0800 Subject: [PATCH 07/16] fix(encoder): reuse existing array in nestedUnkeyedContainer(forKey:) nestedContainer(keyedBy:forKey:) reused an existing object for the key, but nestedUnkeyedContainer(forKey:) always inserted a new array, so a second call for the same key replaced the first array and dropped its elements ({"list":[2]} instead of Foundation's {"list":[1,2]}). Look up and reuse an existing array like Foundation does. The existing-value lookup scanned the object linearly, which made wide objects built from nested containers O(n^2); objects with 32+ keys now consult the key index first. nestedContainer and superEncoder also go through the container's _key() helper instead of calling convertedKey directly. Co-authored-by: Cursor --- Sources/ReerJSON/JSONEncoderImpl.swift | 27 +++++++++--- Tests/ReerJSONTests/RegressionTests.swift | 52 +++++++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/Sources/ReerJSON/JSONEncoderImpl.swift b/Sources/ReerJSON/JSONEncoderImpl.swift index 5856092..c00664a 100644 --- a/Sources/ReerJSON/JSONEncoderImpl.swift +++ b/Sources/ReerJSON/JSONEncoderImpl.swift @@ -39,6 +39,10 @@ final class ObjectKeyIndex { static let threshold = 32 private var keySets: [UnsafeMutablePointer: Set] = [:] + func contains(_ key: String, in object: UnsafeMutablePointer) -> Bool { + keySets[object, default: Self.existingKeys(of: object)].contains(key) + } + func put( _ key: String, _ value: UnsafeMutablePointer, @@ -111,6 +115,13 @@ class JSONEncoderImpl: Encoder { } } + func value(forKey key: String, in object: UnsafeMutablePointer) -> UnsafeMutablePointer? { + if yyjson_mut_obj_size(object) >= ObjectKeyIndex.threshold && !objectKeyIndex.contains(key, in: object) { + return nil + } + return key.withCString { yyjson_mut_obj_getn(object, $0, key.utf8.count) } + } + @inline(__always) func takeValue() -> UnsafeMutablePointer? { if let object { return object } @@ -509,7 +520,7 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine } @inline(__always) - private func _key(_ key: Key) -> String { + private func _key(_ key: some CodingKey) -> String { useDefaultKeys ? key.stringValue : impl.convertedKey(key) } @@ -550,8 +561,8 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine } mutating func nestedContainer(keyedBy keyType: NestedKey.Type, forKey key: Key) -> KeyedEncodingContainer { - let k = impl.convertedKey(key) - if let existing = k.withCString({ yyjson_mut_obj_getn(object, $0, k.utf8.count) }), yyjson_mut_is_obj(existing) { + let k = _key(key) + if let existing = impl.value(forKey: k, in: object), yyjson_mut_is_obj(existing) { return KeyedEncodingContainer(YYJSONKeyedEncodingContainer(impl: impl, codingPath: codingPath + [key], object: existing)) } let obj = yyjson_mut_obj(doc)! @@ -560,16 +571,20 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine } mutating func nestedUnkeyedContainer(forKey key: Key) -> UnkeyedEncodingContainer { + let k = _key(key) + if let existing = impl.value(forKey: k, in: object), yyjson_mut_is_arr(existing) { + return YYJSONUnkeyedEncodingContainer(impl: impl, codingPath: codingPath + [key], array: existing) + } let arr = yyjson_mut_arr(doc)! - addToObject(key: impl.convertedKey(key), value: arr) + addToObject(key: k, value: arr) return YYJSONUnkeyedEncodingContainer(impl: impl, codingPath: codingPath + [key], array: arr) } mutating func superEncoder() -> Encoder { - YYJSONReferencingEncoder(impl: impl, key: impl.convertedKey(_CodingKey.super), codingPath: codingPath + [_CodingKey.super], object: object) + YYJSONReferencingEncoder(impl: impl, key: _key(_CodingKey.super), codingPath: codingPath + [_CodingKey.super], object: object) } mutating func superEncoder(forKey key: Key) -> Encoder { - YYJSONReferencingEncoder(impl: impl, key: impl.convertedKey(key), codingPath: codingPath + [key], object: object) + YYJSONReferencingEncoder(impl: impl, key: _key(key), codingPath: codingPath + [key], object: object) } } diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index 9609bce..9b6bfd2 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -293,6 +293,58 @@ struct RegressionTests { #expect(errorPath { try ReerJSONEncoder().encode(value) } == foundation) } + private struct NestedUnkeyedTwice: Encodable { + enum CodingKeys: String, CodingKey { case list, object } + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + var first = container.nestedUnkeyedContainer(forKey: .list) + try first.encode(1) + var second = container.nestedUnkeyedContainer(forKey: .list) + try second.encode(2) + var a = container.nestedContainer(keyedBy: CodingKeys.self, forKey: .object) + try a.encode(1, forKey: .list) + var b = container.nestedContainer(keyedBy: CodingKeys.self, forKey: .object) + try b.encode(2, forKey: .object) + } + } + + @Test func encoderNestedUnkeyedContainerReusesExistingArray() throws { + let value = NestedUnkeyedTwice() + let options: JSONEncoder.OutputFormatting = [.sortedKeys] + let foundation = JSONEncoder(); foundation.outputFormatting = options + let reer = ReerJSONEncoder(); reer.outputFormatting = options + let expected = String(decoding: try foundation.encode(value), as: UTF8.self) + #expect(expected == #"{"list":[1,2],"object":{"list":1,"object":2}}"#) + let actual = String(decoding: try reer.encode(value), as: UTF8.self) + #expect(actual == expected) + } + + private struct WideNestedObject: Encodable { + let count: Int + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: DynamicKey.self) + for i in 0.. Date: Thu, 24 Sep 2026 19:46:35 +0800 Subject: [PATCH 08/16] fix(encoder): pass the container's full coding path to .custom key strategy The .custom key encoding strategy was called with the encoder's current path plus the key, but nested keyed containers do not push their key onto the encoder, so converters saw ["leaf"] where Foundation passes ["outer", "inner", "leaf"]. Convert keys with the owning container's codingPath instead. Co-authored-by: Cursor --- Sources/ReerJSON/JSONEncoderImpl.swift | 5 ++-- Tests/ReerJSONTests/RegressionTests.swift | 34 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/Sources/ReerJSON/JSONEncoderImpl.swift b/Sources/ReerJSON/JSONEncoderImpl.swift index c00664a..3f9dfc6 100644 --- a/Sources/ReerJSON/JSONEncoderImpl.swift +++ b/Sources/ReerJSON/JSONEncoderImpl.swift @@ -432,8 +432,9 @@ class JSONEncoderImpl: Encoder { // MARK: - Key encoding strategy + /// - Parameter codingPath: The path of the container that owns `key`. @inline(__always) - func convertedKey(_ key: CodingKey) -> String { + func convertedKey(_ key: CodingKey, in codingPath: [CodingKey]) -> String { switch options.keyEncodingStrategy { case .useDefaultKeys: return key.stringValue case .convertToSnakeCase: return Self._convertToSnakeCase(key.stringValue) @@ -521,7 +522,7 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine @inline(__always) private func _key(_ key: some CodingKey) -> String { - useDefaultKeys ? key.stringValue : impl.convertedKey(key) + useDefaultKeys ? key.stringValue : impl.convertedKey(key, in: codingPath) } @inline(__always) diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index 9b6bfd2..ab1f71c 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -345,6 +345,40 @@ struct RegressionTests { #expect(value["object_42"]?["v"]?.int64 == 42) } + private struct DeeplyKeyed: Encodable { + enum CodingKeys: String, CodingKey { case outer, inner, leaf } + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + var outer = container.nestedContainer(keyedBy: CodingKeys.self, forKey: .outer) + var inner = outer.nestedContainer(keyedBy: CodingKeys.self, forKey: .inner) + try inner.encode(1, forKey: .leaf) + var list = outer.nestedUnkeyedContainer(forKey: .leaf) + var element = list.nestedContainer(keyedBy: CodingKeys.self) + try element.encode(2, forKey: .leaf) + } + } + + @Test func encoderCustomKeyStrategyReceivesFullCodingPath() throws { + func paths(_ encode: (@escaping @Sendable ([CodingKey]) -> CodingKey) throws -> Void) rethrows -> [String] { + let seen = LockedState<[String]>(initialState: []) + try encode { path in + seen.withLock { $0.append(path.map { $0.intValue.map(String.init) ?? $0.stringValue }.joined(separator: ".")) } + return path.last! + } + return seen.withLock { $0 } + } + let foundation = try paths { converter in + let encoder = JSONEncoder(); encoder.keyEncodingStrategy = .custom(converter) + _ = try encoder.encode(DeeplyKeyed()) + } + let reer = try paths { converter in + let encoder = ReerJSONEncoder(); encoder.keyEncodingStrategy = .custom(converter) + _ = try encoder.encode(DeeplyKeyed()) + } + #expect(foundation.sorted() == ["outer", "outer.inner", "outer.inner.leaf", "outer.leaf", "outer.leaf.0.leaf"]) + #expect(reer.sorted() == foundation.sorted()) + } + @Test func serializationWriteRejectsInvalidObjects() throws { let invalid = JSONError.invalidData("Invalid JSON object") let cases: [Any] = [ From 5c9d699e2516546b7616109dcdec37a3510d38e3 Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 19:49:00 +0800 Subject: [PATCH 09/16] fix(decoder): reject integer literals that overflow 64 bits yyjson reads an integer literal outside the Int64/UInt64 range as a double. integer() accepted any double exactly representable in T, so -9223372036854775809 rounded to -2^63 and silently decoded as Int64.min, while Foundation throws. Only accept the double for literals with a fraction or exponent; plain integer literals are parsed exactly from the raw text (which still covers Int128/UInt128). Co-authored-by: Cursor --- Sources/ReerJSON/JSON.swift | 9 +++-- Tests/ReerJSONTests/RegressionTests.swift | 43 +++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/Sources/ReerJSON/JSON.swift b/Sources/ReerJSON/JSON.swift index 94bde43..5c0b9a2 100644 --- a/Sources/ReerJSON/JSON.swift +++ b/Sources/ReerJSON/JSON.swift @@ -169,9 +169,12 @@ extension JSON { } else if yyjson_is_sint(&convertedVal) { return T(exactly: yyjson_get_sint(&convertedVal)) } else if yyjson_is_real(&convertedVal) { - let real = unsafe_yyjson_get_real(&convertedVal) - if let value = T(exactly: real) { return value } - // try Int128/UInt128 + // An integer literal only reads as real when it overflows 64 bits; + // its rounded double must not be accepted (e.g. -2^63 - 1 → Int64.min). + if strpbrk(cString, ".eE") != nil, let value = T(exactly: unsafe_yyjson_get_real(&convertedVal)) { + return value + } + // Exact parse for integer literals, e.g. Int128/UInt128. return T(String(cString: cString)) } return nil diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index ab1f71c..213b60f 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -379,6 +379,49 @@ struct RegressionTests { #expect(reer.sorted() == foundation.sorted()) } + // MARK: - Decoder integers + + private static let integerLiterals = [ + "0", "-0", "7", "-7", "127", "128", "-128", "-129", "255", "256", "32767", "32768", "65535", "65536", + "2147483647", "2147483648", "-2147483648", "-2147483649", "4294967295", "4294967296", + "123456789012345678", "-123456789012345678", "999999999999999999", "1000000000000000000", + "9223372036854775807", "9223372036854775808", "-9223372036854775808", "-9223372036854775809", + "18446744073709551615", "18446744073709551616", "1e2", "1E2", "-1e2", "1.0", "1.5", "-0.0", "1e-2", "12e17", + ] + + private struct IntegerBox: Codable { let v: T } + + private static func compareIntegerDecoding(_: T.Type) throws { + for literal in integerLiterals { + let data = Data("[\(literal)]".utf8) + let foundation = try? JSONDecoder().decode([T].self, from: data) + let reer = try? ReerJSONDecoder().decode([T].self, from: data) + #expect(reer == foundation, "\(T.self) \(literal)") + let boxData = Data("{\"v\":\(literal)}".utf8) + let keyed = try? ReerJSONDecoder().decode(IntegerBox.self, from: boxData).v + #expect(keyed == foundation?.first, "\(T.self) keyed \(literal)") + } + } + + @Test func decoderIntegersMatchFoundation() throws { + try Self.compareIntegerDecoding(Int.self) + try Self.compareIntegerDecoding(Int8.self) + try Self.compareIntegerDecoding(Int16.self) + try Self.compareIntegerDecoding(Int32.self) + try Self.compareIntegerDecoding(Int64.self) + try Self.compareIntegerDecoding(UInt.self) + try Self.compareIntegerDecoding(UInt8.self) + try Self.compareIntegerDecoding(UInt16.self) + try Self.compareIntegerDecoding(UInt32.self) + try Self.compareIntegerDecoding(UInt64.self) + } + + @Test func decoderJSON5IntegersStillParse() throws { + let decoder = ReerJSONDecoder() + decoder.allowsJSON5 = true + #expect(try decoder.decode([Int].self, from: Data("[+1, 0x1F, -0x10, 10,]".utf8)) == [1, 31, -16, 10]) + } + @Test func serializationWriteRejectsInvalidObjects() throws { let invalid = JSONError.invalidData("Invalid JSON object") let cases: [Any] = [ From 8b78b8c12324af62db8a47e31786d58308ca28bc Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 19:50:30 +0800 Subject: [PATCH 10/16] perf(decoder): parse plain integer literals without yyjson_read_number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decoder reads numbers as raw text (needed for Decimal, Int128 and big integers), so every integer decode re-parsed its literal through the general yyjson_read_number. Literals of the form -?[0-9]{1,18} (which cannot overflow Int64) are now converted directly; everything else still takes the existing path. decode [Int] 200k: 6.28 → 5.62 ms. Co-authored-by: Cursor --- Sources/ReerJSON/JSON.swift | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Sources/ReerJSON/JSON.swift b/Sources/ReerJSON/JSON.swift index 5c0b9a2..d2cf18d 100644 --- a/Sources/ReerJSON/JSON.swift +++ b/Sources/ReerJSON/JSON.swift @@ -159,6 +159,9 @@ extension JSON { @inline(__always) func integer() -> T? { guard let cString = yyjson_get_raw(pointer) else { return nil } + if let value = Self.parsePlainInteger(cString, count: yyjson_get_len(pointer)) { + return T(exactly: value) + } var convertedVal = yyjson_val() var error = yyjson_read_err() guard let _ = yyjson_read_number(cString, &convertedVal, YYJSON_READ_ALLOW_EXT_NUMBER, nil, &error) else { @@ -180,6 +183,25 @@ extension JSON { return nil } + /// Parses `-?[0-9]{1,18}` without going through `yyjson_read_number`. + /// Anything else (fractions, exponents, JSON5 forms, 19+ digits) returns + /// nil; 18 decimal digits cannot overflow `Int64`. + @inline(__always) + private static func parsePlainInteger(_ chars: UnsafePointer, count: Int) -> Int64? { + guard count > 0, count <= 18 else { return nil } + let negative = chars[0] == 0x2D + var index = negative ? 1 : 0 + guard index < count else { return nil } + var value: Int64 = 0 + while index < count { + let digit = UInt8(bitPattern: chars[index]) &- 0x30 + guard digit < 10 else { return nil } + value = value &* 10 &+ Int64(digit) + index &+= 1 + } + return negative ? -value : value + } + var rawString: String? { guard let cString = yyjson_get_raw(pointer) else { return nil } return String(cString: cString) From 45aa0c5e4dba814b101109fdd7e7b9ba69d5ac87 Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 19:54:17 +0800 Subject: [PATCH 11/16] perf(decoder): byte-level fast path for convertFromSnakeCase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With .convertFromSnakeCase every keyed container converts all keys of the JSON object, including keys the type never reads. The Foundation algorithm used for this (split + lowercased + capitalized + joined) cost about 1 µs per key, so wide objects dominated decode time. Keys without inner underscores are now returned as is, and ASCII keys whose components are all letters or all digits are converted with byte operations. Other keys still use the original algorithm, which is kept as the reference and fuzz-tested against the fast path. The eager table is kept because snake-case conversion is not invertible, so requested keys cannot be mapped back reliably. Co-authored-by: Cursor --- Sources/ReerJSON/JSONDecoderImpl.swift | 185 ++++++++++++++++------ Tests/ReerJSONTests/RegressionTests.swift | 41 +++++ 2 files changed, 180 insertions(+), 46 deletions(-) diff --git a/Sources/ReerJSON/JSONDecoderImpl.swift b/Sources/ReerJSON/JSONDecoderImpl.swift index ab93b25..8d89758 100644 --- a/Sources/ReerJSON/JSONDecoderImpl.swift +++ b/Sources/ReerJSON/JSONDecoderImpl.swift @@ -1016,7 +1016,7 @@ private final class PreTransformKeyedDecodingContainer: KeyedDecod while let keyPtr = yyjson_obj_iter_next(&iter) { if let keyCString = yyjson_get_str(keyPtr), let valuePtr = yyjson_obj_iter_get_val(keyPtr) { let jsonKey = String(cString: keyCString) - result[Self._convertFromSnakeCase(jsonKey)]._setIfNil(to: JSON(pointer: valuePtr)) + result[SnakeCaseKeyConverter.convertFromSnakeCase(jsonKey)]._setIfNil(to: JSON(pointer: valuePtr)) } } case .custom(let converter): @@ -1343,51 +1343,6 @@ private final class PreTransformKeyedDecodingContainer: KeyedDecod } return int } - - private static func _convertFromSnakeCase(_ stringKey: String) -> String { - guard !stringKey.isEmpty else { return stringKey } - - // Find the first non-underscore character - guard let firstNonUnderscore = stringKey.firstIndex(where: { $0 != "_" }) else { - // Reached the end without finding an _ - return stringKey - } - - // Find the last non-underscore character - var lastNonUnderscore = stringKey.index(before: stringKey.endIndex) - while lastNonUnderscore > firstNonUnderscore && stringKey[lastNonUnderscore] == "_" { - stringKey.formIndex(before: &lastNonUnderscore) - } - - let keyRange = firstNonUnderscore...lastNonUnderscore - let leadingUnderscoreRange = stringKey.startIndex.. String { + var key = stringKey + switch key.withUTF8(fastConvert) { + case .unchanged: return stringKey + case .converted(let result): return result + case .unsupported: return convertFromSnakeCaseReference(stringKey) + } + } + + private enum FastResult { + case unchanged + case converted(String) + case unsupported + } + + private static let underscore = UInt8(ascii: "_") + + private static func fastConvert(_ bytes: UnsafeBufferPointer) -> FastResult { + guard let first = bytes.firstIndex(where: { $0 != underscore }) else { return .unchanged } + var last = bytes.count - 1 + while bytes[last] == underscore { last -= 1 } + + var componentCount = 0 + var index = first + while index <= last { + if bytes[index] == underscore { + index += 1 + continue + } + componentCount += 1 + var hasLetter = false, hasDigit = false + while index <= last && bytes[index] != underscore { + let byte = bytes[index] + if byte >= 0x80 { return .unsupported } + if (byte | 0x20) >= UInt8(ascii: "a") && (byte | 0x20) <= UInt8(ascii: "z") { + hasLetter = true + } else if byte >= UInt8(ascii: "0") && byte <= UInt8(ascii: "9") { + hasDigit = true + } else { + return .unsupported + } + index += 1 + } + if hasLetter && hasDigit { return .unsupported } + } + // A single component (no inner underscores) is left as is. + if componentCount <= 1 { return .unchanged } + + let result = String(unsafeUninitializedCapacity: bytes.count) { out in + var count = 0 + for i in 0.. UInt8(ascii: "9") + if isLetter { + let uppercase = isComponentStart && !isFirstComponent + out[count] = uppercase ? byte & ~0x20 : byte | 0x20 + } else { + out[count] = byte + } + count += 1 + isComponentStart = false + index += 1 + } + isFirstComponent = false + } + for i in (last + 1).. String { + guard !stringKey.isEmpty else { return stringKey } + + // Find the first non-underscore character + guard let firstNonUnderscore = stringKey.firstIndex(where: { $0 != "_" }) else { + // Reached the end without finding an _ + return stringKey + } + + // Find the last non-underscore character + var lastNonUnderscore = stringKey.index(before: stringKey.endIndex) + while lastNonUnderscore > firstNonUnderscore && stringKey[lastNonUnderscore] == "_" { + stringKey.formIndex(before: &lastNonUnderscore) + } + + let keyRange = firstNonUnderscore...lastNonUnderscore + let leadingUnderscoreRange = stringKey.startIndex.. Date: Thu, 24 Sep 2026 19:55:46 +0800 Subject: [PATCH 12/16] perf(decoder): drop redundant memchr scan in JSON.string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every decoded string was scanned for NUL bytes to choose between String(bytes:encoding:) and String(decoding:as:). For the valid UTF-8 yyjson hands back both produce the same String, embedded NULs included, so the scan only added an extra pass (decode [String] 100k x 8 B: 1.30 → 1.00 ms). stringWhenJSON5, which deliberately rejects NULs, is unchanged. Co-authored-by: Cursor --- Sources/ReerJSON/JSON.swift | 4 ---- Tests/ReerJSONTests/RegressionTests.swift | 8 ++++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Sources/ReerJSON/JSON.swift b/Sources/ReerJSON/JSON.swift index d2cf18d..7ab125f 100644 --- a/Sources/ReerJSON/JSON.swift +++ b/Sources/ReerJSON/JSON.swift @@ -78,10 +78,6 @@ extension JSON { var string: String? { guard let cString = yyjson_get_str(pointer) else { return nil } let length = yyjson_get_len(pointer) - if memchr(cString, 0, length) != nil { - let rawBuffer = UnsafeRawBufferPointer(start: cString, count: length) - return String(bytes: rawBuffer, encoding: .utf8) - } let buffer = UnsafeBufferPointer( start: UnsafeRawPointer(cString).assumingMemoryBound(to: UInt8.self), count: length diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index 1651a30..67b52fe 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -463,6 +463,14 @@ struct RegressionTests { } } + @Test func decoderStringsWithNULMatchFoundation() throws { + struct Box: Codable, Equatable { let s: String; let a: [String] } + let data = Data(#"{"s":"a\u0000b","a":["\u0000","x\u0000\u00e9\ud83d\ude00",""]}"#.utf8) + let expected = try JSONDecoder().decode(Box.self, from: data) + #expect(expected.s == "a\0b") + #expect(try ReerJSONDecoder().decode(Box.self, from: data) == expected) + } + @Test func serializationWriteRejectsInvalidObjects() throws { let invalid = JSONError.invalidData("Invalid JSON object") let cases: [Any] = [ From 8971da7843a59e47beaf86e9cb79f6e36e95b01b Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 19:58:43 +0800 Subject: [PATCH 13/16] fix(value): escape strings and render numbers correctly in description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JSONValue/JSONObject/JSONArray.description produced JSON-looking text that did not parse back and lost data: - strings and keys were not escaped (a quote or newline broke the output) and were read with String(cString:), truncating at embedded NULs; - unsigned integers above Int64.max printed as negative numbers; - raw numbers (numberAsRaw / bigNumberAsRaw) printed as null; - non-finite doubles printed as inf/nan instead of Infinity/NaN. Render directly from the yyjson value into one byte buffer, keeping the existing ": " / ", " layout. This also removes the per-level [String] arrays and joins (638 KB document: 8.1 → 2.0 ms). Co-authored-by: Cursor --- Sources/ReerJSON/Value.swift | 107 +++++++++++++++++----- Tests/ReerJSONTests/RegressionTests.swift | 22 +++++ 2 files changed, 107 insertions(+), 22 deletions(-) diff --git a/Sources/ReerJSON/Value.swift b/Sources/ReerJSON/Value.swift index 2968026..58daca8 100644 --- a/Sources/ReerJSON/Value.swift +++ b/Sources/ReerJSON/Value.swift @@ -800,25 +800,93 @@ public struct JSONValue: @unchecked Sendable { extension JSONValue: CustomStringConvertible { public var description: String { - switch storage { - case .null: - return "null" - case .bool(let b, _): - return b ? "true" : "false" - case .numberInt(let n, _): - return String(n) - case .numberDouble(let n, _): - return String(n) - case .stringPtr(let ptr, _): - return "\"\(String(cString: ptr))\"" - case .object(let ptr): - return JSONObject(value: ptr, document: document).description - case .array(let ptr): - return JSONArray(value: ptr, document: document).description + guard let rawValue else { return "null" } + return jsonDescription(of: rawValue) + } +} + +/// Renders a value as JSON with `": "` and `", "` separators. Strings are +/// escaped, so the result parses back (with `.allowInfAndNaN` for non-finite +/// numbers). +private func jsonDescription(of value: UnsafeMutablePointer) -> String { + var out: [UInt8] = [] + appendDescription(of: value, to: &out) + return String(decoding: out, as: UTF8.self) +} + +private func appendDescription(of value: UnsafeMutablePointer, to out: inout [UInt8]) { + switch yyjson_get_type(value) { + case YYJSON_TYPE_BOOL: + out.append(contentsOf: yyjson_get_bool(value) ? "true".utf8 : "false".utf8) + case YYJSON_TYPE_NUM: + if yyjson_is_uint(value) { + out.append(contentsOf: String(yyjson_get_uint(value)).utf8) + } else if yyjson_is_sint(value) { + out.append(contentsOf: String(yyjson_get_sint(value)).utf8) + } else { + let real = yyjson_get_real(value) + if real.isNaN { + out.append(contentsOf: "NaN".utf8) + } else if real.isInfinite { + out.append(contentsOf: real < 0 ? "-Infinity".utf8 : "Infinity".utf8) + } else { + out.append(contentsOf: String(real).utf8) + } } + case YYJSON_TYPE_RAW: + if let raw = yyjson_get_raw(value) { + out.append(contentsOf: UnsafeRawBufferPointer(start: raw, count: yyjson_get_len(value))) + } + case YYJSON_TYPE_STR: + if let str = yyjson_get_str(value) { + appendQuoted(UnsafeRawBufferPointer(start: str, count: yyjson_get_len(value)), to: &out) + } + case YYJSON_TYPE_ARR: + out.append(UInt8(ascii: "[")) + var iter = yyjson_arr_iter_with(value) + var first = true + while let element = yyjson_arr_iter_next(&iter) { + if !first { out.append(contentsOf: ", ".utf8) } + first = false + appendDescription(of: element, to: &out) + } + out.append(UInt8(ascii: "]")) + case YYJSON_TYPE_OBJ: + out.append(UInt8(ascii: "{")) + var iter = yyjson_obj_iter_with(value) + var first = true + while let key = yyjson_obj_iter_next(&iter) { + if !first { out.append(contentsOf: ", ".utf8) } + first = false + appendDescription(of: key, to: &out) + out.append(contentsOf: ": ".utf8) + appendDescription(of: yyjson_obj_iter_get_val(key), to: &out) + } + out.append(UInt8(ascii: "}")) + default: + out.append(contentsOf: "null".utf8) } } +private func appendQuoted(_ bytes: UnsafeRawBufferPointer, to out: inout [UInt8]) { + func hex(_ nibble: UInt8) -> UInt8 { nibble < 10 ? 0x30 + nibble : 0x57 + nibble } + out.append(UInt8(ascii: "\"")) + for byte in bytes { + switch byte { + case UInt8(ascii: "\""): out.append(contentsOf: #"\""#.utf8) + case UInt8(ascii: "\\"): out.append(contentsOf: #"\\"#.utf8) + case 0x0A: out.append(contentsOf: #"\n"#.utf8) + case 0x0D: out.append(contentsOf: #"\r"#.utf8) + case 0x09: out.append(contentsOf: #"\t"#.utf8) + case 0x08: out.append(contentsOf: #"\b"#.utf8) + case 0x0C: out.append(contentsOf: #"\f"#.utf8) + case 0..<0x20: out.append(contentsOf: [0x5C, 0x75, 0x30, 0x30, hex(byte >> 4), hex(byte & 0xF)]) + default: out.append(byte) + } + } + out.append(UInt8(ascii: "\"")) +} + // MARK: - JSON Object /// A JSON object providing key-value access. @@ -911,11 +979,7 @@ public struct JSONObjectIterator: IteratorProtocol { extension JSONObject: CustomStringConvertible { public var description: String { - var parts: [String] = [] - for (key, value) in self { - parts.append("\"\(key)\": \(value.description)") - } - return "{\(parts.joined(separator: ", "))}" + jsonDescription(of: value) } } @@ -982,8 +1046,7 @@ public struct JSONArrayIterator: IteratorProtocol { extension JSONArray: CustomStringConvertible { public var description: String { - let elements = self.map { $0.description } - return "[\(elements.joined(separator: ", "))]" + jsonDescription(of: value) } } diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index 67b52fe..803884d 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -471,6 +471,28 @@ struct RegressionTests { #expect(try ReerJSONDecoder().decode(Box.self, from: data) == expected) } + // MARK: - JSONValue.description + + @Test(arguments: [ + #"{"a\"b": "x\"y\nz\\w", "t": "\t\u0001", "u": "é😀/"}"#, + #"["a\u0000b", "", {"": [1, -2.5, true, null]}]"#, + #"[Infinity, -Infinity, NaN, 1e300, -0.0, 0.1]"#, + ]) + func jsonValueDescriptionIsParseableJSON(json: String) throws { + let value = try JSONValue(string: json, options: .allowInfAndNaN) + let reparsed = try JSONValue(string: value.description, options: .allowInfAndNaN) + #expect(try reparsed.data(options: .allowInfAndNaN) == value.data(options: .allowInfAndNaN), "\(value.description)") + } + + @Test func jsonValueDescriptionKeepsFormat() throws { + let value = try JSONValue(string: #"{"a": [1, "s", {"b": null}], "c": false}"#) + #expect(value.description == #"{"a": [1, "s", {"b": null}], "c": false}"#) + #expect(try JSONValue(string: "1.5").description == "1.5") + #expect(try JSONValue(string: "-7").description == "-7") + #expect(try JSONValue(string: "[18446744073709551615]").description == "[18446744073709551615]") + #expect(try JSONValue(string: "[1.50, 2]", options: .numberAsRaw).description == "[1.50, 2]") + } + @Test func serializationWriteRejectsInvalidObjects() throws { let invalid = JSONError.invalidData("Invalid JSON object") let cases: [Any] = [ From 47b7f9c2b1d8d20f244a59fc0a1cccf6fa41b7f3 Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 20:00:35 +0800 Subject: [PATCH 14/16] refactor(value): share document parsing between DocumentStorage and DocumentRef DocumentStorage and DocumentRef each carried a verbatim copy of the copying and in-place (INSITU + padding) parse logic, so a fix to one could silently miss the other. Move both into private helpers and keep only the ownership difference in the two types. Also drop the unused Span overload of DocumentRef.streamParse. No behavior change. Co-authored-by: Cursor --- Sources/ReerJSON/Value.swift | 178 +++++++++++++---------------------- 1 file changed, 66 insertions(+), 112 deletions(-) diff --git a/Sources/ReerJSON/Value.swift b/Sources/ReerJSON/Value.swift index 58daca8..7e32bd0 100644 --- a/Sources/ReerJSON/Value.swift +++ b/Sources/ReerJSON/Value.swift @@ -24,6 +24,66 @@ import yyjson import Foundation +// MARK: - Document Parsing (Internal) + +/// Parses JSON from `data` without modifying it. +/// +/// `YYJSON_READ_INSITU` is masked out: in-place parsing would leave the +/// document pointing into a buffer the caller still owns. +private func parseDocument(copying data: Data, options: JSONReadOptions) throws -> UnsafeMutablePointer { + var error = yyjson_read_err() + var flags = options.yyjsonFlags + flags &= ~yyjson_read_flag(YYJSON_READ_INSITU) + + if data.isEmpty { + throw JSONError.invalidJSON("Empty content") + } + + guard let doc = yyReadDocument(from: data, flags: flags, error: &error) else { + throw JSONError(parsing: error) + } + return doc +} + +/// Parses JSON in place inside `data` and takes ownership of it. +/// +/// Returns the document together with the buffer it points into, which must +/// be kept alive for as long as the document. `data` is left empty. +private func parseDocument( + consuming data: inout Data, + options: JSONReadOptions +) throws -> (UnsafeMutablePointer, Data) { + var error = yyjson_read_err() + var flags = options.yyjsonFlags + flags |= YYJSON_READ_INSITU + + if data.isEmpty { + throw JSONError.invalidJSON("Empty content") + } + + let paddingSize = Int(YYJSON_PADDING_SIZE) + let originalCount = data.count + + data.reserveCapacity(originalCount + paddingSize) + data.append(contentsOf: repeatElement(0 as UInt8, count: paddingSize)) + + let result = data.withUnsafeMutableBytes { bytes in + yyReadDocument( + mutating: bytes, + validByteCount: originalCount, + flags: flags, + error: &error + ) + } + + guard let doc = result else { + throw JSONError(parsing: error) + } + let retainedData = data + data.removeAll(keepingCapacity: false) + return (doc, retainedData) +} + // MARK: - Document Storage (Internal) /// Move-only owner for a yyjson document. @@ -46,22 +106,7 @@ internal struct DocumentStorage: ~Copyable, @unchecked Sendable { /// - options: Options for reading the JSON. /// - Throws: `JSONError` if parsing fails. init(data: Data, options: JSONReadOptions = .default) throws { - var error = yyjson_read_err() - var flags = options.yyjsonFlags - // Mask out YYJSON_READ_INSITU to prevent use-after-free issues. - // In-place parsing must use the dedicated consuming initializer. - flags &= ~yyjson_read_flag(YYJSON_READ_INSITU) - - if data.isEmpty { - throw JSONError.invalidJSON("Empty content") - } - - let result = yyReadDocument(from: data, flags: flags, error: &error) - - guard let doc = result else { - throw JSONError(parsing: error) - } - self.doc = doc + self.doc = try parseDocument(copying: data, options: options) self.retainedData = nil } @@ -77,35 +122,9 @@ internal struct DocumentStorage: ~Copyable, @unchecked Sendable { /// - options: Options for reading the JSON. /// - Throws: `JSONError` if parsing fails. init(consuming data: inout Data, options: JSONReadOptions = .default) throws { - var error = yyjson_read_err() - var flags = options.yyjsonFlags - flags |= YYJSON_READ_INSITU - - if data.isEmpty { - throw JSONError.invalidJSON("Empty content") - } - - let paddingSize = Int(YYJSON_PADDING_SIZE) - let originalCount = data.count - - data.reserveCapacity(originalCount + paddingSize) - data.append(contentsOf: repeatElement(0 as UInt8, count: paddingSize)) - - let result = data.withUnsafeMutableBytes { bytes in - yyReadDocument( - mutating: bytes, - validByteCount: originalCount, - flags: flags, - error: &error - ) - } - - guard let doc = result else { - throw JSONError(parsing: error) - } + let (doc, retainedData) = try parseDocument(consuming: &data, options: options) self.doc = doc - self.retainedData = data - data.removeAll(keepingCapacity: false) + self.retainedData = retainedData } deinit { @@ -132,53 +151,14 @@ internal final class DocumentRef: @unchecked Sendable { private let retainedData: Data? init(data: Data, options: JSONReadOptions = .default) throws { - var error = yyjson_read_err() - var flags = options.yyjsonFlags - flags &= ~yyjson_read_flag(YYJSON_READ_INSITU) - - if data.isEmpty { - throw JSONError.invalidJSON("Empty content") - } - - let result = yyReadDocument(from: data, flags: flags, error: &error) - - guard let doc = result else { - throw JSONError(parsing: error) - } - self.doc = doc + self.doc = try parseDocument(copying: data, options: options) self.retainedData = nil } init(consuming data: inout Data, options: JSONReadOptions = .default) throws { - var error = yyjson_read_err() - var flags = options.yyjsonFlags - flags |= YYJSON_READ_INSITU - - if data.isEmpty { - throw JSONError.invalidJSON("Empty content") - } - - let paddingSize = Int(YYJSON_PADDING_SIZE) - let originalCount = data.count - - data.reserveCapacity(originalCount + paddingSize) - data.append(contentsOf: repeatElement(0 as UInt8, count: paddingSize)) - - let result = data.withUnsafeMutableBytes { bytes in - yyReadDocument( - mutating: bytes, - validByteCount: originalCount, - flags: flags, - error: &error - ) - } - - guard let doc = result else { - throw JSONError(parsing: error) - } + let (doc, retainedData) = try parseDocument(consuming: &data, options: options) self.doc = doc - self.retainedData = data - data.removeAll(keepingCapacity: false) + self.retainedData = retainedData } enum StreamParseResult { @@ -186,32 +166,6 @@ internal final class DocumentRef: @unchecked Sendable { case needMoreData } - static func streamParse( - bytes: Span, - options: JSONReadOptions - ) throws -> StreamParseResult { - guard !bytes.isEmpty else { return .needMoreData } - - var error = yyjson_read_err() - var flags = options.yyjsonFlags - flags |= YYJSON_READ_STOP_WHEN_DONE - flags &= ~yyjson_read_flag(YYJSON_READ_INSITU) - - let result = yyReadDocument(from: bytes.bytes, flags: flags, error: &error) - - if let doc = result { - let consumed = yyjson_doc_get_read_size(doc) - return .success(DocumentRef(alreadyParsed: doc), consumedBytes: consumed) - } - - if error.code == YYJSON_READ_ERROR_UNEXPECTED_END - || error.code == YYJSON_READ_ERROR_EMPTY_CONTENT { - return .needMoreData - } - - throw JSONError(parsing: error) - } - static func streamParse( bytes: UnsafePointer, count: Int, From e29084aee06d3af2f880b116b310969683967a0b Mon Sep 17 00:00:00 2001 From: phoenix Date: Thu, 24 Sep 2026 20:12:21 +0800 Subject: [PATCH 15/16] test: use a larger chunk size in the mixed-size stream test Feeding a 300 KB item 7 bytes at a time re-parses it from the start on every call and took ~8 s; 997-byte chunks exercise the same boundaries. Co-authored-by: Cursor --- Tests/ReerJSONTests/RegressionTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift index 803884d..bde5ccc 100644 --- a/Tests/ReerJSONTests/RegressionTests.swift +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -101,7 +101,7 @@ struct RegressionTests { ? items.joined(separator: " \n") : "[" + items.joined(separator: ", ") + "]" let data = Data(text.utf8) - for chunkSize in [data.count, 4_096, 7] { + for chunkSize in [data.count, 4_096, 997] { var parser = JSONStreamParser(mode: mode) var values: [JSONValue] = [] var offset = 0 From 9b38f12a75516ac17e23e37abb5781f4fcabc657 Mon Sep 17 00:00:00 2001 From: openclaw Date: Fri, 25 Sep 2026 01:02:27 +0800 Subject: [PATCH 16/16] perf(encoder): keep narrow keyed containers off the width check a3e08a1 routed every keyed-container insert through putKey, which reads yyjson_mut_obj_size on each call. That check is itself cheap, but on the common narrow-object path it is pure overhead: a fresh object never reaches the index, so every insert pays a size read and a branch for nothing. Re-encoding Twitter (its user objects carry ~40 keys, so they sit just above the old threshold of 32) regressed ~22%: 0.70 ms -> 0.87 ms. Count inserts in the container instead. Narrow objects stay on the plain yyjson_mut_obj_put fast path and never call putKey; only once a container has written threshold keys does it hand over to the shared index, which discovers the earlier keys lazily. Raise the threshold to 256 so that payloads in the tens-of-keys range never build a key set at all. Verified: Twitter re-encode back to 0.706/0.707/0.708 ms over three alternating rounds (main 0.701/0.720/0.712, pr-head 0.863/0.865/0.872). All 143 tests pass. --- Sources/ReerJSON/JSONEncoderImpl.swift | 36 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/Sources/ReerJSON/JSONEncoderImpl.swift b/Sources/ReerJSON/JSONEncoderImpl.swift index 3f9dfc6..e6aff94 100644 --- a/Sources/ReerJSON/JSONEncoderImpl.swift +++ b/Sources/ReerJSON/JSONEncoderImpl.swift @@ -36,7 +36,16 @@ import JJLISO8601DateFormatter /// insertion O(1) instead of scanning the whole object like /// `yyjson_mut_obj_put` does. See `JSONEncoderImpl.putKey`. final class ObjectKeyIndex { - static let threshold = 32 + /// Objects at least this wide switch from `yyjson_mut_obj_put`'s linear + /// duplicate scan to the `ObjectKeyIndex` key set. Enabling it too early is + /// a net loss: building a `Set` for an object of a few dozen keys + /// costs more than the scan it saves. That showed up as a ~20% encoder + /// slowdown on Twitter, whose `user` objects carry 40 keys each. The win + /// only materialises for genuinely wide objects (hundreds of keys and up), + /// so the threshold sits well above the widths found in typical payloads + /// while still leaving `encoderKeyedContainerScalesLinearlyWithWidth` + /// comfortably linear. + static let threshold = 256 private var keySets: [UnsafeMutablePointer: Set] = [:] func contains(_ key: String, in object: UnsafeMutablePointer) -> Bool { @@ -106,6 +115,12 @@ class JSONEncoderImpl: Encoder { /// Inserts `key` into a keyed-container object, replacing the value of an /// existing equal key in place. Every insertion into such an object must /// go through here so that `ObjectKeyIndex` stays in sync. + /// + /// This is the *wide-object* path: it reads the object's width on every + /// call and may build a key set. Keyed containers stay off it entirely + /// until they have written `ObjectKeyIndex.threshold` keys, so narrow + /// objects (the common case) never pay for the width check. See + /// `YYJSONKeyedEncodingContainer.addToObject`. @inline(__always) func putKey(_ key: String, _ value: UnsafeMutablePointer, into object: UnsafeMutablePointer) { if yyjson_mut_obj_size(object) < ObjectKeyIndex.threshold { @@ -530,9 +545,22 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine impl.wrapString(s) } - @inline(__always) - private func addToObject(key: String, value: UnsafeMutablePointer) { - impl.putKey(key, value, into: object) + /// Number of keys this container has already inserted into `object`. + /// + /// Narrow objects (the common case) stay on the plain `yyjson_mut_obj_put` + /// fast path and never call into `impl.putKey`, so they never pay for the + /// per-insert width check that path performs. Only once the container has + /// written `ObjectKeyIndex.threshold` keys does it hand over to the shared + /// index, which discovers the earlier keys lazily and stays in sync. + private var insertedKeyCount = 0 + + private mutating func addToObject(key: String, value: UnsafeMutablePointer) { + if insertedKeyCount < ObjectKeyIndex.threshold { + insertedKeyCount += 1 + yyjson_mut_obj_put(object, _strVal(key), value) + } else { + impl.putKey(key, value, into: object) + } } mutating func encodeNil(forKey key: Key) throws { addToObject(key: _key(key), value: yyjson_mut_null(doc)) }