diff --git a/Package.swift b/Package.swift index 3877685..e8202ff 100644 --- a/Package.swift +++ b/Package.swift @@ -67,6 +67,11 @@ let package = Package( .testTarget( name: "ReerJSONTests", dependencies: ["ReerJSON"], + exclude: [ + "Info.plist", + "Models/instruments.json", + "Models/json_to_swift.rb", + ], resources: [ .copy("Models/apache_builds.json"), .copy("Models/canada.json"), diff --git a/Sources/ReerJSON/JSON.swift b/Sources/ReerJSON/JSON.swift index 94bde43..7ab125f 100644 --- a/Sources/ReerJSON/JSON.swift +++ b/Sources/ReerJSON/JSON.swift @@ -78,10 +78,6 @@ extension JSON { var string: String? { guard let cString = yyjson_get_str(pointer) else { return nil } let length = yyjson_get_len(pointer) - if memchr(cString, 0, length) != nil { - let rawBuffer = UnsafeRawBufferPointer(start: cString, count: length) - return String(bytes: rawBuffer, encoding: .utf8) - } let buffer = UnsafeBufferPointer( start: UnsafeRawPointer(cString).assumingMemoryBound(to: UInt8.self), count: length @@ -159,6 +155,9 @@ extension JSON { @inline(__always) func integer() -> T? { guard let cString = yyjson_get_raw(pointer) else { return nil } + if let value = Self.parsePlainInteger(cString, count: yyjson_get_len(pointer)) { + return T(exactly: value) + } var convertedVal = yyjson_val() var error = yyjson_read_err() guard let _ = yyjson_read_number(cString, &convertedVal, YYJSON_READ_ALLOW_EXT_NUMBER, nil, &error) else { @@ -169,14 +168,36 @@ extension JSON { } else if yyjson_is_sint(&convertedVal) { return T(exactly: yyjson_get_sint(&convertedVal)) } else if yyjson_is_real(&convertedVal) { - let real = unsafe_yyjson_get_real(&convertedVal) - if let value = T(exactly: real) { return value } - // try Int128/UInt128 + // An integer literal only reads as real when it overflows 64 bits; + // its rounded double must not be accepted (e.g. -2^63 - 1 → Int64.min). + if strpbrk(cString, ".eE") != nil, let value = T(exactly: unsafe_yyjson_get_real(&convertedVal)) { + return value + } + // Exact parse for integer literals, e.g. Int128/UInt128. return T(String(cString: cString)) } return nil } + /// Parses `-?[0-9]{1,18}` without going through `yyjson_read_number`. + /// Anything else (fractions, exponents, JSON5 forms, 19+ digits) returns + /// nil; 18 decimal digits cannot overflow `Int64`. + @inline(__always) + private static func parsePlainInteger(_ chars: UnsafePointer, count: Int) -> Int64? { + guard count > 0, count <= 18 else { return nil } + let negative = chars[0] == 0x2D + var index = negative ? 1 : 0 + guard index < count else { return nil } + var value: Int64 = 0 + while index < count { + let digit = UInt8(bitPattern: chars[index]) &- 0x30 + guard digit < 10 else { return nil } + value = value &* 10 &+ Int64(digit) + index &+= 1 + } + return negative ? -value : value + } + var rawString: String? { guard let cString = yyjson_get_raw(pointer) else { return nil } return String(cString: cString) diff --git a/Sources/ReerJSON/JSONDecoderImpl.swift b/Sources/ReerJSON/JSONDecoderImpl.swift index ab93b25..8d89758 100644 --- a/Sources/ReerJSON/JSONDecoderImpl.swift +++ b/Sources/ReerJSON/JSONDecoderImpl.swift @@ -1016,7 +1016,7 @@ private final class PreTransformKeyedDecodingContainer: KeyedDecod while let keyPtr = yyjson_obj_iter_next(&iter) { if let keyCString = yyjson_get_str(keyPtr), let valuePtr = yyjson_obj_iter_get_val(keyPtr) { let jsonKey = String(cString: keyCString) - result[Self._convertFromSnakeCase(jsonKey)]._setIfNil(to: JSON(pointer: valuePtr)) + result[SnakeCaseKeyConverter.convertFromSnakeCase(jsonKey)]._setIfNil(to: JSON(pointer: valuePtr)) } } case .custom(let converter): @@ -1343,51 +1343,6 @@ private final class PreTransformKeyedDecodingContainer: KeyedDecod } return int } - - private static func _convertFromSnakeCase(_ stringKey: String) -> String { - guard !stringKey.isEmpty else { return stringKey } - - // Find the first non-underscore character - guard let firstNonUnderscore = stringKey.firstIndex(where: { $0 != "_" }) else { - // Reached the end without finding an _ - return stringKey - } - - // Find the last non-underscore character - var lastNonUnderscore = stringKey.index(before: stringKey.endIndex) - while lastNonUnderscore > firstNonUnderscore && stringKey[lastNonUnderscore] == "_" { - stringKey.formIndex(before: &lastNonUnderscore) - } - - let keyRange = firstNonUnderscore...lastNonUnderscore - let leadingUnderscoreRange = stringKey.startIndex.. String { + var key = stringKey + switch key.withUTF8(fastConvert) { + case .unchanged: return stringKey + case .converted(let result): return result + case .unsupported: return convertFromSnakeCaseReference(stringKey) + } + } + + private enum FastResult { + case unchanged + case converted(String) + case unsupported + } + + private static let underscore = UInt8(ascii: "_") + + private static func fastConvert(_ bytes: UnsafeBufferPointer) -> FastResult { + guard let first = bytes.firstIndex(where: { $0 != underscore }) else { return .unchanged } + var last = bytes.count - 1 + while bytes[last] == underscore { last -= 1 } + + var componentCount = 0 + var index = first + while index <= last { + if bytes[index] == underscore { + index += 1 + continue + } + componentCount += 1 + var hasLetter = false, hasDigit = false + while index <= last && bytes[index] != underscore { + let byte = bytes[index] + if byte >= 0x80 { return .unsupported } + if (byte | 0x20) >= UInt8(ascii: "a") && (byte | 0x20) <= UInt8(ascii: "z") { + hasLetter = true + } else if byte >= UInt8(ascii: "0") && byte <= UInt8(ascii: "9") { + hasDigit = true + } else { + return .unsupported + } + index += 1 + } + if hasLetter && hasDigit { return .unsupported } + } + // A single component (no inner underscores) is left as is. + if componentCount <= 1 { return .unchanged } + + let result = String(unsafeUninitializedCapacity: bytes.count) { out in + var count = 0 + for i in 0.. UInt8(ascii: "9") + if isLetter { + let uppercase = isComponentStart && !isFirstComponent + out[count] = uppercase ? byte & ~0x20 : byte | 0x20 + } else { + out[count] = byte + } + count += 1 + isComponentStart = false + index += 1 + } + isFirstComponent = false + } + for i in (last + 1).. String { + guard !stringKey.isEmpty else { return stringKey } + + // Find the first non-underscore character + guard let firstNonUnderscore = stringKey.firstIndex(where: { $0 != "_" }) else { + // Reached the end without finding an _ + return stringKey + } + + // Find the last non-underscore character + var lastNonUnderscore = stringKey.index(before: stringKey.endIndex) + while lastNonUnderscore > firstNonUnderscore && stringKey[lastNonUnderscore] == "_" { + stringKey.formIndex(before: &lastNonUnderscore) + } + + let keyRange = firstNonUnderscore...lastNonUnderscore + let leadingUnderscoreRange = stringKey.startIndex..` for an object of a few dozen keys + /// costs more than the scan it saves. That showed up as a ~20% encoder + /// slowdown on Twitter, whose `user` objects carry 40 keys each. The win + /// only materialises for genuinely wide objects (hundreds of keys and up), + /// so the threshold sits well above the widths found in typical payloads + /// while still leaving `encoderKeyedContainerScalesLinearlyWithWidth` + /// comfortably linear. + static let threshold = 256 + private var keySets: [UnsafeMutablePointer: Set] = [:] + + func contains(_ key: String, in object: UnsafeMutablePointer) -> Bool { + keySets[object, default: Self.existingKeys(of: object)].contains(key) + } + + func put( + _ key: String, + _ value: UnsafeMutablePointer, + into object: UnsafeMutablePointer, + impl: JSONEncoderImpl + ) { + let inserted = keySets[object, default: Self.existingKeys(of: object)].insert(key).inserted + if inserted { + yyjson_mut_obj_add(object, impl.wrapString(key), value) + } else { + yyjson_mut_obj_put(object, impl.wrapString(key), value) + } + } + + private static func existingKeys(of object: UnsafeMutablePointer) -> Set { + var keys = Set(minimumCapacity: Int(yyjson_mut_obj_size(object)) * 2) + var iter = yyjson_mut_obj_iter() + guard yyjson_mut_obj_iter_init(object, &iter) else { return keys } + while let keyVal = yyjson_mut_obj_iter_next(&iter) { + if let str = yyjson_mut_get_str(keyVal) { + let buffer = UnsafeRawBufferPointer(start: str, count: yyjson_mut_get_len(keyVal)) + keys.insert(String(decoding: buffer, as: UTF8.self)) + } + } + return keys + } +} + class JSONEncoderImpl: Encoder { let doc: UnsafeMutablePointer let options: ReerJSONEncoder.Options var codingPath: [CodingKey] + private var _objectKeyIndex: ObjectKeyIndex? var userInfo: [CodingUserInfoKey: Any] { options.userInfo } @@ -43,10 +92,49 @@ class JSONEncoderImpl: Encoder { var array: UnsafeMutablePointer? var object: UnsafeMutablePointer? - init(doc: UnsafeMutablePointer, codingPath: [CodingKey], options: ReerJSONEncoder.Options) { + init( + doc: UnsafeMutablePointer, + codingPath: [CodingKey], + options: ReerJSONEncoder.Options, + objectKeyIndex: ObjectKeyIndex? = nil + ) { self.doc = doc self.codingPath = codingPath self.options = options + self._objectKeyIndex = objectKeyIndex + } + + /// Shared by every encoder writing into this document's objects. + var objectKeyIndex: ObjectKeyIndex { + if let index = _objectKeyIndex { return index } + let index = ObjectKeyIndex() + _objectKeyIndex = index + return index + } + + /// Inserts `key` into a keyed-container object, replacing the value of an + /// existing equal key in place. Every insertion into such an object must + /// go through here so that `ObjectKeyIndex` stays in sync. + /// + /// This is the *wide-object* path: it reads the object's width on every + /// call and may build a key set. Keyed containers stay off it entirely + /// until they have written `ObjectKeyIndex.threshold` keys, so narrow + /// objects (the common case) never pay for the width check. See + /// `YYJSONKeyedEncodingContainer.addToObject`. + @inline(__always) + func putKey(_ key: String, _ value: UnsafeMutablePointer, into object: UnsafeMutablePointer) { + if yyjson_mut_obj_size(object) < ObjectKeyIndex.threshold { + yyjson_mut_obj_put(object, wrapString(key), value) + } else { + objectKeyIndex.put(key, value, into: object, impl: self) + } + } + + func value(forKey key: String, in object: UnsafeMutablePointer) -> UnsafeMutablePointer? { + if yyjson_mut_obj_size(object) >= ObjectKeyIndex.threshold && !objectKeyIndex.contains(key, in: object) { + return nil + } + return key.withCString { yyjson_mut_obj_getn(object, $0, key.utf8.count) } } @inline(__always) @@ -347,21 +435,21 @@ class JSONEncoderImpl: Encoder { func wrapStringKeyedDictValue(_ dict: [String: Encodable], for additionalKey: CodingKey?) throws -> UnsafeMutablePointer? { let obj = yyjson_mut_obj(doc)! - let savedPath = codingPath if let additionalKey { codingPath.append(additionalKey) } + defer { if additionalKey != nil { codingPath.removeLast() } } for (key, value) in dict { let keyVal = wrapString(key) let val = try wrapEncodable(value, for: _CodingKey(stringValue: key)!) ?? yyjson_mut_obj(doc)! yyjson_mut_obj_add(obj, keyVal, val) } - codingPath = savedPath return obj } // MARK: - Key encoding strategy + /// - Parameter codingPath: The path of the container that owns `key`. @inline(__always) - func convertedKey(_ key: CodingKey) -> String { + func convertedKey(_ key: CodingKey, in codingPath: [CodingKey]) -> String { switch options.keyEncodingStrategy { case .useDefaultKeys: return key.stringValue case .convertToSnakeCase: return Self._convertToSnakeCase(key.stringValue) @@ -448,8 +536,8 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine } @inline(__always) - private func _key(_ key: Key) -> String { - useDefaultKeys ? key.stringValue : impl.convertedKey(key) + private func _key(_ key: some CodingKey) -> String { + useDefaultKeys ? key.stringValue : impl.convertedKey(key, in: codingPath) } @inline(__always) @@ -457,9 +545,22 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine impl.wrapString(s) } - @inline(__always) - private func addToObject(key: String, value: UnsafeMutablePointer) { - yyjson_mut_obj_put(object, _strVal(key), value) + /// Number of keys this container has already inserted into `object`. + /// + /// Narrow objects (the common case) stay on the plain `yyjson_mut_obj_put` + /// fast path and never call into `impl.putKey`, so they never pay for the + /// per-insert width check that path performs. Only once the container has + /// written `ObjectKeyIndex.threshold` keys does it hand over to the shared + /// index, which discovers the earlier keys lazily and stays in sync. + private var insertedKeyCount = 0 + + private mutating func addToObject(key: String, value: UnsafeMutablePointer) { + if insertedKeyCount < ObjectKeyIndex.threshold { + insertedKeyCount += 1 + yyjson_mut_obj_put(object, _strVal(key), value) + } else { + impl.putKey(key, value, into: object) + } } mutating func encodeNil(forKey key: Key) throws { addToObject(key: _key(key), value: yyjson_mut_null(doc)) } @@ -489,8 +590,8 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine } mutating func nestedContainer(keyedBy keyType: NestedKey.Type, forKey key: Key) -> KeyedEncodingContainer { - let k = impl.convertedKey(key) - if let existing = k.withCString({ yyjson_mut_obj_getn(object, $0, k.utf8.count) }), yyjson_mut_is_obj(existing) { + let k = _key(key) + if let existing = impl.value(forKey: k, in: object), yyjson_mut_is_obj(existing) { return KeyedEncodingContainer(YYJSONKeyedEncodingContainer(impl: impl, codingPath: codingPath + [key], object: existing)) } let obj = yyjson_mut_obj(doc)! @@ -499,16 +600,20 @@ private struct YYJSONKeyedEncodingContainer: KeyedEncodingContaine } mutating func nestedUnkeyedContainer(forKey key: Key) -> UnkeyedEncodingContainer { + let k = _key(key) + if let existing = impl.value(forKey: k, in: object), yyjson_mut_is_arr(existing) { + return YYJSONUnkeyedEncodingContainer(impl: impl, codingPath: codingPath + [key], array: existing) + } let arr = yyjson_mut_arr(doc)! - addToObject(key: impl.convertedKey(key), value: arr) + addToObject(key: k, value: arr) return YYJSONUnkeyedEncodingContainer(impl: impl, codingPath: codingPath + [key], array: arr) } mutating func superEncoder() -> Encoder { - YYJSONReferencingEncoder(impl: impl, key: impl.convertedKey(_CodingKey.super), codingPath: codingPath + [_CodingKey.super], object: object) + YYJSONReferencingEncoder(impl: impl, key: _key(_CodingKey.super), codingPath: codingPath + [_CodingKey.super], object: object) } mutating func superEncoder(forKey key: Key) -> Encoder { - YYJSONReferencingEncoder(impl: impl, key: impl.convertedKey(key), codingPath: codingPath + [key], object: object) + YYJSONReferencingEncoder(impl: impl, key: _key(key), codingPath: codingPath + [key], object: object) } } @@ -567,10 +672,10 @@ private class YYJSONReferencingEncoder: JSONEncoderImpl { let referencedObject: UnsafeMutablePointer init(impl: JSONEncoderImpl, key: String, codingPath: [CodingKey], object: UnsafeMutablePointer) { self.key = key; self.referencedObject = object - super.init(doc: impl.doc, codingPath: codingPath, options: impl.options) + super.init(doc: impl.doc, codingPath: codingPath, options: impl.options, objectKeyIndex: impl.objectKeyIndex) } deinit { - yyjson_mut_obj_put(referencedObject, wrapString(key), takeValue() ?? yyjson_mut_obj(doc)!) + putKey(key, takeValue() ?? yyjson_mut_obj(doc)!, into: referencedObject) } } @@ -579,7 +684,7 @@ private class YYJSONReferencingArrayEncoder: JSONEncoderImpl { let insertIndex: Int init(impl: JSONEncoderImpl, codingPath: [CodingKey], array: UnsafeMutablePointer, index: Int) { self.referencedArray = array; self.insertIndex = index - super.init(doc: impl.doc, codingPath: codingPath, options: impl.options) + super.init(doc: impl.doc, codingPath: codingPath, options: impl.options, objectKeyIndex: impl.objectKeyIndex) } deinit { yyjson_mut_arr_insert(referencedArray, takeValue() ?? yyjson_mut_obj(doc)!, insertIndex) diff --git a/Sources/ReerJSON/ReerJSONEncoder.swift b/Sources/ReerJSON/ReerJSONEncoder.swift index 50744b7..6974f28 100644 --- a/Sources/ReerJSON/ReerJSONEncoder.swift +++ b/Sources/ReerJSON/ReerJSONEncoder.swift @@ -177,11 +177,18 @@ open class ReerJSONEncoder { public init() {} + func optionsSnapshot() -> Options { + optionsLock.lock() + defer { optionsLock.unlock() } + return options + } + open func encode(_ value: T) throws -> Data { let doc = yyjson_mut_doc_new(nil)! defer { yyjson_mut_doc_free(doc) } - let encoder = JSONEncoderImpl(doc: doc, codingPath: [], options: options) + let encoder = JSONEncoderImpl(doc: doc, codingPath: [], options: optionsSnapshot()) + let outputFormatting = encoder.options.outputFormatting if let date = value as? Date { encoder.singleValue = try encoder.wrapDateValue(date, for: nil) @@ -242,7 +249,8 @@ open class ReerJSONEncoder { let doc = yyjson_mut_doc_new(nil)! defer { yyjson_mut_doc_free(doc) } - let encoder = JSONEncoderImpl(doc: doc, codingPath: [], options: options) + let encoder = JSONEncoderImpl(doc: doc, codingPath: [], options: optionsSnapshot()) + let outputFormatting = encoder.options.outputFormatting try value.encode(to: encoder, configuration: configuration) guard let root = encoder.takeValue() else { diff --git a/Sources/ReerJSON/ReerJSONSerialization.swift b/Sources/ReerJSON/ReerJSONSerialization.swift index 1a2bbb2..9c5177c 100644 --- a/Sources/ReerJSON/ReerJSONSerialization.swift +++ b/Sources/ReerJSON/ReerJSONSerialization.swift @@ -152,16 +152,16 @@ public enum ReerJSONSerialization { options.contains(.infAndNaNAsNull) || options.contains(.allowInfAndNaN) - if isTopLevelContainer { - guard isValidJSONObject(obj, allowNonFiniteNumbers: allowNonFiniteNumbers) else { - throw JSONError.invalidData("Invalid JSON object") - } - } else if isFragment { + // Containers are validated while they are converted. + if isFragment { guard options.contains(.fragmentsAllowed) else { throw JSONError.invalidData("Top-level JSON value must be an array or dictionary") } - } else { - throw JSONError.invalidData("Invalid JSON object") + if !allowNonFiniteNumbers, let number = obj as? NSNumber, !number.doubleValue.isFinite { + throw JSONError.invalidData("NaN or Infinity not allowed in JSON") + } + } else if !isTopLevelContainer { + throw invalidObjectError } guard let doc = yyjson_mut_doc_new(nil) else { @@ -233,13 +233,6 @@ public enum ReerJSONSerialization { // MARK: - Private Helpers - private static func isValidJSONObject(_ obj: Any, allowNonFiniteNumbers: Bool) -> Bool { - guard obj is NSArray || obj is NSDictionary else { - return false - } - return isValidJSONObjectRecursive(obj, allowNonFiniteNumbers: allowNonFiniteNumbers) - } - private static func isValidJSONObjectRecursive( _ obj: Any, allowNonFiniteNumbers: Bool @@ -327,80 +320,217 @@ public enum ReerJSONSerialization { return try value.data(options: writeOptions) } + private static let invalidObjectError = JSONError.invalidData("Invalid JSON object") + + /// Converts and validates a Foundation object graph in a single pass. + /// + /// Any value that `isValidJSONObject` would reject throws `invalidObjectError`. private static func foundationObjectToYYJSON( _ obj: Any, doc: UnsafeMutablePointer, options: WritingOptions ) throws -> UnsafeMutablePointer { - switch obj { - case let str as NSString: - return try yyFromString(str as String, in: doc) - - case let num as NSNumber: - let doubleValue = num.doubleValue - if doubleValue.isNaN || doubleValue.isInfinite { - if options.contains(.infAndNaNAsNull) { - return yyjson_mut_null(doc) + #if canImport(Darwin) + // Class checks on AnyObject are plain `isKindOfClass` calls, whereas + // casting `Any` goes through the much slower generic dynamic cast. + return try foundationObjectToYYJSON(object: obj as AnyObject, doc: doc, options: options) + #else + switch obj { + case let str as NSString: + return try yyFromString(str as String, in: doc) + case let num as NSNumber: + return try numberToYYJSON(num, doc: doc, options: options) + case is NSNull: + return yyjson_mut_null(doc) + case let arr as NSArray: + let jsonArr = try makeArray(doc) + for element in arr { + _ = yyjson_mut_arr_append(jsonArr, try foundationObjectToYYJSON(element, doc: doc, options: options)) } - if options.contains(.allowInfAndNaN) { - return yyjson_mut_real(doc, doubleValue) + return jsonArr + case let dict as NSDictionary: + let jsonObj = try makeObject(doc) + if options.contains(.sortedKeys) { + for (key, value) in try sortedEntries(of: dict) { + try addEntry(key: key, value: value, to: jsonObj, doc: doc, options: options) + } + } else { + for (key, value) in dict { + guard let keyString = key as? String else { throw invalidObjectError } + try addEntry(key: keyString, value: value, to: jsonObj, doc: doc, options: options) + } } - throw JSONError.invalidData("NaN or Infinity not allowed in JSON") + return jsonObj + default: + throw invalidObjectError } + #endif + } - if isBoolNumber(num) { - return yyjson_mut_bool(doc, num.boolValue) + #if canImport(Darwin) + private static func foundationObjectToYYJSON( + object: AnyObject, + doc: UnsafeMutablePointer, + options: WritingOptions + ) throws -> UnsafeMutablePointer { + if let str = object as? NSString { + return try stringToYYJSON(str, doc: doc) } - - let objCType = num.objCType.pointee - switch objCType { - case 0x63, 0x73, 0x69, 0x6C, 0x71: // 'c', 's', 'i', 'l', 'q' (signed integers) - return yyjson_mut_sint(doc, num.int64Value) - case 0x43, 0x53, 0x49, 0x4C, 0x51: // 'C', 'S', 'I', 'L', 'Q' (unsigned integers) - return yyjson_mut_uint(doc, num.uint64Value) - default: - return yyjson_mut_real(doc, doubleValue) + if let num = object as? NSNumber { + return try numberToYYJSON(num, doc: doc, options: options) } - - case is NSNull: - return yyjson_mut_null(doc) - - case let arr as NSArray: - guard let jsonArr = yyjson_mut_arr(doc) else { - throw JSONError.invalidData("Failed to create array") + if let dict = object as? NSDictionary { + let jsonObj = try makeObject(doc) + let count = CFDictionaryGetCount(dict) + try withUnsafeTemporaryAllocation(of: UnsafeRawPointer?.self, capacity: max(count * 2, 1)) { buffer in + let keys = buffer.baseAddress!, values = keys + count + CFDictionaryGetKeysAndValues(dict, keys, values) + if options.contains(.sortedKeys) { + var entries: [(key: String, nsKey: NSString, value: AnyObject)] = [] + entries.reserveCapacity(count) + for i in 0...fromOpaque(keys[i]!).takeUnretainedValue() + guard let keyString = key as? NSString else { throw invalidObjectError } + let value = Unmanaged.fromOpaque(values[i]!).takeUnretainedValue() + entries.append((keyString as String, keyString, value)) + } + entries.sort { $0.key < $1.key } + for entry in entries { + let keyVal = try stringToYYJSON(entry.nsKey, doc: doc) + let valueVal = try foundationObjectToYYJSON(object: entry.value, doc: doc, options: options) + _ = yyjson_mut_obj_add(jsonObj, keyVal, valueVal) + } + } else { + for i in 0...fromOpaque(keys[i]!).takeUnretainedValue() + guard let keyString = key as? NSString else { throw invalidObjectError } + let value = Unmanaged.fromOpaque(values[i]!).takeUnretainedValue() + let keyVal = try stringToYYJSON(keyString, doc: doc) + let valueVal = try foundationObjectToYYJSON(object: value, doc: doc, options: options) + _ = yyjson_mut_obj_add(jsonObj, keyVal, valueVal) + } + } + } + return jsonObj } - for element in arr { - let elementVal = try foundationObjectToYYJSON(element, doc: doc, options: options) - _ = yyjson_mut_arr_append(jsonArr, elementVal) + if let arr = object as? NSArray { + let jsonArr = try makeArray(doc) + let count = CFArrayGetCount(arr) + try withUnsafeTemporaryAllocation(of: UnsafeRawPointer?.self, capacity: max(count, 1)) { buffer in + CFArrayGetValues(arr, CFRange(location: 0, length: count), buffer.baseAddress!) + for element in buffer.prefix(count) { + let object = Unmanaged.fromOpaque(element!).takeUnretainedValue() + let elementVal = try foundationObjectToYYJSON(object: object, doc: doc, options: options) + _ = yyjson_mut_arr_append(jsonArr, elementVal) + } + } + return jsonArr } - return jsonArr - - case let dict as NSDictionary: - guard let jsonObj = yyjson_mut_obj(doc) else { - throw JSONError.invalidData("Failed to create object") + if object is NSNull { + return yyjson_mut_null(doc) } - - let keys: [Any] - if options.contains(.sortedKeys) { - keys = (dict.allKeys as? [String])?.sorted() ?? dict.allKeys - } else { - keys = dict.allKeys + throw invalidObjectError + } + + @inline(__always) + private static func stringToYYJSON( + _ string: NSString, + doc: UnsafeMutablePointer + ) throws -> UnsafeMutablePointer { + let cfString = string as CFString + // CFStringGetCStringPtr only succeeds for ASCII storage, where the + // UTF-16 length equals the UTF-8 byte count. + let length = CFStringGetLength(cfString) + let utf8 = CFStringBuiltInEncodings.UTF8.rawValue + if let ptr = CFStringGetCStringPtr(cfString, utf8), + let val = yyjson_mut_strncpy(doc, ptr, length) { + return val } - - for key in keys { - guard let keyString = key as? String else { - throw JSONError.invalidData("Dictionary keys must be strings") + let capacity = CFStringGetMaximumSizeForEncoding(length, utf8) + let val = withUnsafeTemporaryAllocation(of: CChar.self, capacity: max(capacity, 1)) { buffer -> UnsafeMutablePointer? in + let base = buffer.baseAddress! + var used: CFIndex = 0 + let converted = base.withMemoryRebound(to: UInt8.self, capacity: buffer.count) { + CFStringGetBytes(cfString, CFRange(location: 0, length: length), utf8, 0, false, $0, capacity, &used) } - guard let value = dict[key] else { continue } - let keyVal = try yyFromString(keyString, in: doc) - let valueVal = try foundationObjectToYYJSON(value, doc: doc, options: options) - _ = yyjson_mut_obj_put(jsonObj, keyVal, valueVal) + // A short conversion means unpaired surrogates; leave those to String bridging. + guard converted == length else { return nil } + return yyjson_mut_strncpy(doc, base, used) + } + if let val { + return val } - return jsonObj + return try yyFromString(string as String, in: doc) + } + #else + private static func addEntry( + key: String, + value: Any, + to jsonObj: UnsafeMutablePointer, + doc: UnsafeMutablePointer, + options: WritingOptions + ) throws { + let keyVal = try yyFromString(key, in: doc) + let valueVal = try foundationObjectToYYJSON(value, doc: doc, options: options) + _ = yyjson_mut_obj_add(jsonObj, keyVal, valueVal) + } + + /// Returns the entries sorted by key; throws if any key is not a string. + private static func sortedEntries(of dict: NSDictionary) throws -> [(key: String, value: Any)] { + var entries: [(key: String, value: Any)] = [] + entries.reserveCapacity(dict.count) + for (key, value) in dict { + guard let keyString = key as? String else { throw invalidObjectError } + entries.append((keyString, value)) + } + entries.sort { $0.key < $1.key } + return entries + } + #endif + @inline(__always) + private static func numberToYYJSON( + _ num: NSNumber, + doc: UnsafeMutablePointer, + options: WritingOptions + ) throws -> UnsafeMutablePointer { + if isBoolNumber(num) { + return yyjson_mut_bool(doc, num.boolValue) + } + switch num.objCType.pointee { + case 0x63, 0x73, 0x69, 0x6C, 0x71: // 'c', 's', 'i', 'l', 'q' (signed integers) + return yyjson_mut_sint(doc, num.int64Value) + case 0x43, 0x53, 0x49, 0x4C, 0x51: // 'C', 'S', 'I', 'L', 'Q' (unsigned integers) + return yyjson_mut_uint(doc, num.uint64Value) default: - throw JSONError.invalidData("Unsupported Foundation type: \(type(of: obj))") + let doubleValue = num.doubleValue + if !doubleValue.isFinite { + if options.contains(.infAndNaNAsNull) { + return yyjson_mut_null(doc) + } + if options.contains(.allowInfAndNaN) { + return yyjson_mut_real(doc, doubleValue) + } + throw invalidObjectError + } + return yyjson_mut_real(doc, doubleValue) + } + } + + @inline(__always) + private static func makeArray(_ doc: UnsafeMutablePointer) throws -> UnsafeMutablePointer { + guard let arr = yyjson_mut_arr(doc) else { + throw JSONError.invalidData("Failed to create array") + } + return arr + } + + @inline(__always) + private static func makeObject(_ doc: UnsafeMutablePointer) throws -> UnsafeMutablePointer { + guard let obj = yyjson_mut_obj(doc) else { + throw JSONError.invalidData("Failed to create object") } + return obj } } diff --git a/Sources/ReerJSON/StreamParser.swift b/Sources/ReerJSON/StreamParser.swift index 1d6d52c..b6cca23 100644 --- a/Sources/ReerJSON/StreamParser.swift +++ b/Sources/ReerJSON/StreamParser.swift @@ -94,6 +94,7 @@ public struct JSONStreamParser: Sendable { private var buffer: Data private var readOffset: Int private var arrayState: ArrayParseState + private var lastItemSize: Int = 0 /// The number of bytes buffered but not yet consumed. public var pendingByteCount: Int { @@ -157,6 +158,7 @@ public struct JSONStreamParser: Sendable { buffer.removeAll(keepingCapacity: true) readOffset = 0 arrayState = .expectOpenBracket + lastItemSize = 0 } // MARK: - Private Types @@ -303,32 +305,63 @@ public struct JSONStreamParser: Sendable { let available = buffer.count - readOffset guard available > 0 else { return nil } - // yyjson_read_opts() in non-INSITU mode allocates its own padded buffer - // and copies the input — so we do not need to add YYJSON_PADDING_SIZE - // bytes ourselves here. + // yyjson_read_opts() in non-INSITU mode copies the whole input it is + // given, so handing it everything up to the buffer end would cost + // O(remaining bytes) per item and O(K²) for K buffered items. Parse a + // window sized from the previous item instead, and only widen it when + // the window cannot decide the value on its own. Once the window + // covers all available bytes the behavior is exactly that of a + // whole-buffer parse. let startOffset = readOffset - let result: DocumentRef.StreamParseResult = try buffer.withUnsafeBytes { rawBuffer in - guard let base = rawBuffer.baseAddress else { return .needMoreData } - let bytes = base.advanced(by: startOffset).assumingMemoryBound(to: UInt8.self) - return try DocumentRef.streamParse(bytes: bytes, count: available, options: options) - } + var window = min(available, max(Self.minimumParseWindow, lastItemSize &* 2)) + while true { + let isWholeBuffer = window == available + let result: DocumentRef.StreamParseResult + do { + result = try parseWindow(at: startOffset, count: window) + } catch where !isWholeBuffer { + window = min(available, window &* 2) + continue + } - switch result { - case .needMoreData: - return nil - case .success(let doc, let consumed): - let endOffset = startOffset + consumed - - // Boundary safety: - // A successful parse that ends exactly at the buffer end may be a - // truncated numeric token (e.g. we have "12" so far but the source - // is actually "123"). yyjson cannot tell the difference. Defer the - // value until either more data arrives or the caller finalizes. - if !finalizing && endOffset >= buffer.count { + switch result { + case .needMoreData: + guard isWholeBuffer else { + window = min(available, window &* 2) + continue + } return nil + case .success(let doc, let consumed): + // A value ending exactly at the window end may continue past it + // (e.g. "12" of "123"); only a whole-buffer parse can decide. + if consumed >= window && !isWholeBuffer { + window = min(available, window &* 2) + continue + } + let endOffset = startOffset + consumed + + // Boundary safety: + // A successful parse that ends exactly at the buffer end may be a + // truncated numeric token (e.g. we have "12" so far but the source + // is actually "123"). yyjson cannot tell the difference. Defer the + // value until either more data arrives or the caller finalizes. + if !finalizing && endOffset >= buffer.count { + return nil + } + readOffset = endOffset + lastItemSize = consumed + return ParsedItem(document: doc) } - readOffset = endOffset - return ParsedItem(document: doc) + } + } + + private static let minimumParseWindow = 256 + + private func parseWindow(at offset: Int, count: Int) throws -> DocumentRef.StreamParseResult { + try buffer.withUnsafeBytes { rawBuffer in + guard let base = rawBuffer.baseAddress else { return .needMoreData } + let bytes = base.advanced(by: offset).assumingMemoryBound(to: UInt8.self) + return try DocumentRef.streamParse(bytes: bytes, count: count, options: options) } } diff --git a/Sources/ReerJSON/Value.swift b/Sources/ReerJSON/Value.swift index 2968026..7e32bd0 100644 --- a/Sources/ReerJSON/Value.swift +++ b/Sources/ReerJSON/Value.swift @@ -24,6 +24,66 @@ import yyjson import Foundation +// MARK: - Document Parsing (Internal) + +/// Parses JSON from `data` without modifying it. +/// +/// `YYJSON_READ_INSITU` is masked out: in-place parsing would leave the +/// document pointing into a buffer the caller still owns. +private func parseDocument(copying data: Data, options: JSONReadOptions) throws -> UnsafeMutablePointer { + var error = yyjson_read_err() + var flags = options.yyjsonFlags + flags &= ~yyjson_read_flag(YYJSON_READ_INSITU) + + if data.isEmpty { + throw JSONError.invalidJSON("Empty content") + } + + guard let doc = yyReadDocument(from: data, flags: flags, error: &error) else { + throw JSONError(parsing: error) + } + return doc +} + +/// Parses JSON in place inside `data` and takes ownership of it. +/// +/// Returns the document together with the buffer it points into, which must +/// be kept alive for as long as the document. `data` is left empty. +private func parseDocument( + consuming data: inout Data, + options: JSONReadOptions +) throws -> (UnsafeMutablePointer, Data) { + var error = yyjson_read_err() + var flags = options.yyjsonFlags + flags |= YYJSON_READ_INSITU + + if data.isEmpty { + throw JSONError.invalidJSON("Empty content") + } + + let paddingSize = Int(YYJSON_PADDING_SIZE) + let originalCount = data.count + + data.reserveCapacity(originalCount + paddingSize) + data.append(contentsOf: repeatElement(0 as UInt8, count: paddingSize)) + + let result = data.withUnsafeMutableBytes { bytes in + yyReadDocument( + mutating: bytes, + validByteCount: originalCount, + flags: flags, + error: &error + ) + } + + guard let doc = result else { + throw JSONError(parsing: error) + } + let retainedData = data + data.removeAll(keepingCapacity: false) + return (doc, retainedData) +} + // MARK: - Document Storage (Internal) /// Move-only owner for a yyjson document. @@ -46,22 +106,7 @@ internal struct DocumentStorage: ~Copyable, @unchecked Sendable { /// - options: Options for reading the JSON. /// - Throws: `JSONError` if parsing fails. init(data: Data, options: JSONReadOptions = .default) throws { - var error = yyjson_read_err() - var flags = options.yyjsonFlags - // Mask out YYJSON_READ_INSITU to prevent use-after-free issues. - // In-place parsing must use the dedicated consuming initializer. - flags &= ~yyjson_read_flag(YYJSON_READ_INSITU) - - if data.isEmpty { - throw JSONError.invalidJSON("Empty content") - } - - let result = yyReadDocument(from: data, flags: flags, error: &error) - - guard let doc = result else { - throw JSONError(parsing: error) - } - self.doc = doc + self.doc = try parseDocument(copying: data, options: options) self.retainedData = nil } @@ -77,35 +122,9 @@ internal struct DocumentStorage: ~Copyable, @unchecked Sendable { /// - options: Options for reading the JSON. /// - Throws: `JSONError` if parsing fails. init(consuming data: inout Data, options: JSONReadOptions = .default) throws { - var error = yyjson_read_err() - var flags = options.yyjsonFlags - flags |= YYJSON_READ_INSITU - - if data.isEmpty { - throw JSONError.invalidJSON("Empty content") - } - - let paddingSize = Int(YYJSON_PADDING_SIZE) - let originalCount = data.count - - data.reserveCapacity(originalCount + paddingSize) - data.append(contentsOf: repeatElement(0 as UInt8, count: paddingSize)) - - let result = data.withUnsafeMutableBytes { bytes in - yyReadDocument( - mutating: bytes, - validByteCount: originalCount, - flags: flags, - error: &error - ) - } - - guard let doc = result else { - throw JSONError(parsing: error) - } + let (doc, retainedData) = try parseDocument(consuming: &data, options: options) self.doc = doc - self.retainedData = data - data.removeAll(keepingCapacity: false) + self.retainedData = retainedData } deinit { @@ -132,53 +151,14 @@ internal final class DocumentRef: @unchecked Sendable { private let retainedData: Data? init(data: Data, options: JSONReadOptions = .default) throws { - var error = yyjson_read_err() - var flags = options.yyjsonFlags - flags &= ~yyjson_read_flag(YYJSON_READ_INSITU) - - if data.isEmpty { - throw JSONError.invalidJSON("Empty content") - } - - let result = yyReadDocument(from: data, flags: flags, error: &error) - - guard let doc = result else { - throw JSONError(parsing: error) - } - self.doc = doc + self.doc = try parseDocument(copying: data, options: options) self.retainedData = nil } init(consuming data: inout Data, options: JSONReadOptions = .default) throws { - var error = yyjson_read_err() - var flags = options.yyjsonFlags - flags |= YYJSON_READ_INSITU - - if data.isEmpty { - throw JSONError.invalidJSON("Empty content") - } - - let paddingSize = Int(YYJSON_PADDING_SIZE) - let originalCount = data.count - - data.reserveCapacity(originalCount + paddingSize) - data.append(contentsOf: repeatElement(0 as UInt8, count: paddingSize)) - - let result = data.withUnsafeMutableBytes { bytes in - yyReadDocument( - mutating: bytes, - validByteCount: originalCount, - flags: flags, - error: &error - ) - } - - guard let doc = result else { - throw JSONError(parsing: error) - } + let (doc, retainedData) = try parseDocument(consuming: &data, options: options) self.doc = doc - self.retainedData = data - data.removeAll(keepingCapacity: false) + self.retainedData = retainedData } enum StreamParseResult { @@ -186,32 +166,6 @@ internal final class DocumentRef: @unchecked Sendable { case needMoreData } - static func streamParse( - bytes: Span, - options: JSONReadOptions - ) throws -> StreamParseResult { - guard !bytes.isEmpty else { return .needMoreData } - - var error = yyjson_read_err() - var flags = options.yyjsonFlags - flags |= YYJSON_READ_STOP_WHEN_DONE - flags &= ~yyjson_read_flag(YYJSON_READ_INSITU) - - let result = yyReadDocument(from: bytes.bytes, flags: flags, error: &error) - - if let doc = result { - let consumed = yyjson_doc_get_read_size(doc) - return .success(DocumentRef(alreadyParsed: doc), consumedBytes: consumed) - } - - if error.code == YYJSON_READ_ERROR_UNEXPECTED_END - || error.code == YYJSON_READ_ERROR_EMPTY_CONTENT { - return .needMoreData - } - - throw JSONError(parsing: error) - } - static func streamParse( bytes: UnsafePointer, count: Int, @@ -800,23 +754,91 @@ public struct JSONValue: @unchecked Sendable { extension JSONValue: CustomStringConvertible { public var description: String { - switch storage { - case .null: - return "null" - case .bool(let b, _): - return b ? "true" : "false" - case .numberInt(let n, _): - return String(n) - case .numberDouble(let n, _): - return String(n) - case .stringPtr(let ptr, _): - return "\"\(String(cString: ptr))\"" - case .object(let ptr): - return JSONObject(value: ptr, document: document).description - case .array(let ptr): - return JSONArray(value: ptr, document: document).description + guard let rawValue else { return "null" } + return jsonDescription(of: rawValue) + } +} + +/// Renders a value as JSON with `": "` and `", "` separators. Strings are +/// escaped, so the result parses back (with `.allowInfAndNaN` for non-finite +/// numbers). +private func jsonDescription(of value: UnsafeMutablePointer) -> String { + var out: [UInt8] = [] + appendDescription(of: value, to: &out) + return String(decoding: out, as: UTF8.self) +} + +private func appendDescription(of value: UnsafeMutablePointer, to out: inout [UInt8]) { + switch yyjson_get_type(value) { + case YYJSON_TYPE_BOOL: + out.append(contentsOf: yyjson_get_bool(value) ? "true".utf8 : "false".utf8) + case YYJSON_TYPE_NUM: + if yyjson_is_uint(value) { + out.append(contentsOf: String(yyjson_get_uint(value)).utf8) + } else if yyjson_is_sint(value) { + out.append(contentsOf: String(yyjson_get_sint(value)).utf8) + } else { + let real = yyjson_get_real(value) + if real.isNaN { + out.append(contentsOf: "NaN".utf8) + } else if real.isInfinite { + out.append(contentsOf: real < 0 ? "-Infinity".utf8 : "Infinity".utf8) + } else { + out.append(contentsOf: String(real).utf8) + } + } + case YYJSON_TYPE_RAW: + if let raw = yyjson_get_raw(value) { + out.append(contentsOf: UnsafeRawBufferPointer(start: raw, count: yyjson_get_len(value))) + } + case YYJSON_TYPE_STR: + if let str = yyjson_get_str(value) { + appendQuoted(UnsafeRawBufferPointer(start: str, count: yyjson_get_len(value)), to: &out) + } + case YYJSON_TYPE_ARR: + out.append(UInt8(ascii: "[")) + var iter = yyjson_arr_iter_with(value) + var first = true + while let element = yyjson_arr_iter_next(&iter) { + if !first { out.append(contentsOf: ", ".utf8) } + first = false + appendDescription(of: element, to: &out) + } + out.append(UInt8(ascii: "]")) + case YYJSON_TYPE_OBJ: + out.append(UInt8(ascii: "{")) + var iter = yyjson_obj_iter_with(value) + var first = true + while let key = yyjson_obj_iter_next(&iter) { + if !first { out.append(contentsOf: ", ".utf8) } + first = false + appendDescription(of: key, to: &out) + out.append(contentsOf: ": ".utf8) + appendDescription(of: yyjson_obj_iter_get_val(key), to: &out) + } + out.append(UInt8(ascii: "}")) + default: + out.append(contentsOf: "null".utf8) + } +} + +private func appendQuoted(_ bytes: UnsafeRawBufferPointer, to out: inout [UInt8]) { + func hex(_ nibble: UInt8) -> UInt8 { nibble < 10 ? 0x30 + nibble : 0x57 + nibble } + out.append(UInt8(ascii: "\"")) + for byte in bytes { + switch byte { + case UInt8(ascii: "\""): out.append(contentsOf: #"\""#.utf8) + case UInt8(ascii: "\\"): out.append(contentsOf: #"\\"#.utf8) + case 0x0A: out.append(contentsOf: #"\n"#.utf8) + case 0x0D: out.append(contentsOf: #"\r"#.utf8) + case 0x09: out.append(contentsOf: #"\t"#.utf8) + case 0x08: out.append(contentsOf: #"\b"#.utf8) + case 0x0C: out.append(contentsOf: #"\f"#.utf8) + case 0..<0x20: out.append(contentsOf: [0x5C, 0x75, 0x30, 0x30, hex(byte >> 4), hex(byte & 0xF)]) + default: out.append(byte) } } + out.append(UInt8(ascii: "\"")) } // MARK: - JSON Object @@ -911,11 +933,7 @@ public struct JSONObjectIterator: IteratorProtocol { extension JSONObject: CustomStringConvertible { public var description: String { - var parts: [String] = [] - for (key, value) in self { - parts.append("\"\(key)\": \(value.description)") - } - return "{\(parts.joined(separator: ", "))}" + jsonDescription(of: value) } } @@ -982,8 +1000,7 @@ public struct JSONArrayIterator: IteratorProtocol { extension JSONArray: CustomStringConvertible { public var description: String { - let elements = self.map { $0.description } - return "[\(elements.joined(separator: ", "))]" + jsonDescription(of: value) } } diff --git a/Tests/ReerJSONTests/RegressionTests.swift b/Tests/ReerJSONTests/RegressionTests.swift new file mode 100644 index 0000000..bde5ccc --- /dev/null +++ b/Tests/ReerJSONTests/RegressionTests.swift @@ -0,0 +1,515 @@ +// +// Copyright © 2026 reers. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +import Foundation +import Testing +@testable import ReerJSON + +@Suite("Regression") +struct RegressionTests { + + @Test func encoderOptionsAreReadUnderLock() throws { + let encoder = ReerJSONEncoder() + let key = CodingUserInfoKey(rawValue: "k")! + nonisolated(unsafe) let unsafeEncoder = encoder + let group = DispatchGroup() + let writer = Thread { + for i in 0..<20_000 { + unsafeEncoder.userInfo = [key: i] + unsafeEncoder.keyEncodingStrategy = i % 2 == 0 ? .convertToSnakeCase : .useDefaultKeys + } + group.leave() + } + let reader = Thread { + for i in 0..<20_000 { + _ = try? unsafeEncoder.encode(["fooBar": i]) + } + group.leave() + } + group.enter(); group.enter() + writer.start(); reader.start() + group.wait() + } + + // MARK: - JSONStreamParser + + private static func ndjson(count: Int) -> Data { + var s = "" + for i in 0.. Void) rethrows -> Double { + var best = Double.greatestFiniteMagnitude + for _ in 0..<3 { + let start = DispatchTime.now().uptimeNanoseconds + try body() + best = min(best, Double(DispatchTime.now().uptimeNanoseconds - start)) + } + return best + } + + @Test(arguments: [JSONStreamMode.jsonLines, .jsonArray]) + func streamParserScalesLinearlyWithItemCount(mode: JSONStreamMode) throws { + func payload(_ count: Int) -> Data { + guard mode == .jsonArray else { return Self.ndjson(count: count) } + var s = String(decoding: Self.ndjson(count: count), as: UTF8.self) + s.removeLast() + return Data(("[" + s.replacingOccurrences(of: "\n", with: ",") + "]").utf8) + } + let small = payload(2_000), large = payload(16_000) + func parseAll(_ data: Data, expected: Int) throws { + var parser = JSONStreamParser(mode: mode) + let count = try parser.parse(data).count + parser.finalize().count + #expect(count == expected) + } + let smallTime = try Self.bestTime { try parseAll(small, expected: 2_000) } + let largeTime = try Self.bestTime { try parseAll(large, expected: 16_000) } + // 8x the items: linear ≈ 8x, quadratic ≈ 64x. + #expect(largeTime / smallTime < 24, "ratio \(largeTime / smallTime)") + } + + @Test(arguments: [JSONStreamMode.jsonLines, .jsonArray]) + func streamParserHandlesMixedItemSizes(mode: JSONStreamMode) throws { + let sizes = [1, 5_000, 3, 70_000, 10, 10, 300_000, 2, 1_000] + var items: [String] = [] + for (i, size) in sizes.enumerated() { + items.append("{\"i\":\(i),\"s\":\"\(String(repeating: "x", count: size))\"}") + items.append("\(i * 1_000_003)") + } + let text = mode == .jsonLines + ? items.joined(separator: " \n") + : "[" + items.joined(separator: ", ") + "]" + let data = Data(text.utf8) + for chunkSize in [data.count, 4_096, 997] { + var parser = JSONStreamParser(mode: mode) + var values: [JSONValue] = [] + var offset = 0 + while offset < data.count { + let end = min(offset + chunkSize, data.count) + values += try parser.parse(data.subdata(in: offset.. NSDictionary { + let dict = NSMutableDictionary() + for i in 0.. Data) -> [String] { + do { + _ = try encode() + } catch let EncodingError.invalidValue(_, context) { + return context.codingPath.map(\.stringValue) + } catch {} + return [""] + } + let value = RecoversFromDictionaryError() + let foundation = errorPath { try JSONEncoder().encode(value) } + #expect(foundation == ["value"]) + #expect(errorPath { try ReerJSONEncoder().encode(value) } == foundation) + } + + private struct NestedUnkeyedTwice: Encodable { + enum CodingKeys: String, CodingKey { case list, object } + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + var first = container.nestedUnkeyedContainer(forKey: .list) + try first.encode(1) + var second = container.nestedUnkeyedContainer(forKey: .list) + try second.encode(2) + var a = container.nestedContainer(keyedBy: CodingKeys.self, forKey: .object) + try a.encode(1, forKey: .list) + var b = container.nestedContainer(keyedBy: CodingKeys.self, forKey: .object) + try b.encode(2, forKey: .object) + } + } + + @Test func encoderNestedUnkeyedContainerReusesExistingArray() throws { + let value = NestedUnkeyedTwice() + let options: JSONEncoder.OutputFormatting = [.sortedKeys] + let foundation = JSONEncoder(); foundation.outputFormatting = options + let reer = ReerJSONEncoder(); reer.outputFormatting = options + let expected = String(decoding: try foundation.encode(value), as: UTF8.self) + #expect(expected == #"{"list":[1,2],"object":{"list":1,"object":2}}"#) + let actual = String(decoding: try reer.encode(value), as: UTF8.self) + #expect(actual == expected) + } + + private struct WideNestedObject: Encodable { + let count: Int + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: DynamicKey.self) + for i in 0.. CodingKey) throws -> Void) rethrows -> [String] { + let seen = LockedState<[String]>(initialState: []) + try encode { path in + seen.withLock { $0.append(path.map { $0.intValue.map(String.init) ?? $0.stringValue }.joined(separator: ".")) } + return path.last! + } + return seen.withLock { $0 } + } + let foundation = try paths { converter in + let encoder = JSONEncoder(); encoder.keyEncodingStrategy = .custom(converter) + _ = try encoder.encode(DeeplyKeyed()) + } + let reer = try paths { converter in + let encoder = ReerJSONEncoder(); encoder.keyEncodingStrategy = .custom(converter) + _ = try encoder.encode(DeeplyKeyed()) + } + #expect(foundation.sorted() == ["outer", "outer.inner", "outer.inner.leaf", "outer.leaf", "outer.leaf.0.leaf"]) + #expect(reer.sorted() == foundation.sorted()) + } + + // MARK: - Decoder integers + + private static let integerLiterals = [ + "0", "-0", "7", "-7", "127", "128", "-128", "-129", "255", "256", "32767", "32768", "65535", "65536", + "2147483647", "2147483648", "-2147483648", "-2147483649", "4294967295", "4294967296", + "123456789012345678", "-123456789012345678", "999999999999999999", "1000000000000000000", + "9223372036854775807", "9223372036854775808", "-9223372036854775808", "-9223372036854775809", + "18446744073709551615", "18446744073709551616", "1e2", "1E2", "-1e2", "1.0", "1.5", "-0.0", "1e-2", "12e17", + ] + + private struct IntegerBox: Codable { let v: T } + + private static func compareIntegerDecoding(_: T.Type) throws { + for literal in integerLiterals { + let data = Data("[\(literal)]".utf8) + let foundation = try? JSONDecoder().decode([T].self, from: data) + let reer = try? ReerJSONDecoder().decode([T].self, from: data) + #expect(reer == foundation, "\(T.self) \(literal)") + let boxData = Data("{\"v\":\(literal)}".utf8) + let keyed = try? ReerJSONDecoder().decode(IntegerBox.self, from: boxData).v + #expect(keyed == foundation?.first, "\(T.self) keyed \(literal)") + } + } + + @Test func decoderIntegersMatchFoundation() throws { + try Self.compareIntegerDecoding(Int.self) + try Self.compareIntegerDecoding(Int8.self) + try Self.compareIntegerDecoding(Int16.self) + try Self.compareIntegerDecoding(Int32.self) + try Self.compareIntegerDecoding(Int64.self) + try Self.compareIntegerDecoding(UInt.self) + try Self.compareIntegerDecoding(UInt8.self) + try Self.compareIntegerDecoding(UInt16.self) + try Self.compareIntegerDecoding(UInt32.self) + try Self.compareIntegerDecoding(UInt64.self) + } + + @Test func decoderJSON5IntegersStillParse() throws { + let decoder = ReerJSONDecoder() + decoder.allowsJSON5 = true + #expect(try decoder.decode([Int].self, from: Data("[+1, 0x1F, -0x10, 10,]".utf8)) == [1, 31, -16, 10]) + } + + // MARK: - Decoder snake case + + private struct AllKeys: Decodable { + let keys: [String: Int] + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicKey.self) + var keys: [String: Int] = [:] + for key in container.allKeys { keys[key.stringValue] = try container.decode(Int.self, forKey: key) } + self.keys = keys + } + } + + @Test func decoderSnakeCaseConversionMatchesFoundation() throws { + let jsonKeys = [ + "", "_", "___", "a", "already", "alreadyCamel", "ALLCAPS", "snake_case", "ALL_CAPS", "mixed_CaseWORD", + "_leading", "trailing_", "__both__", "a__b", "a_b_c_d", "with_1_digit", "v2_api", "x_2x", "one_22_333", + "ünïcode_kéy", "ß_straße", "emoji_😀_key", "dash-word_other", "dot.word_other", "space word_x", + "_mixed_Leading", "trailing_Mixed__", "i_d", "url_i_d", "html_url", "a_", "_a", "__a", "a__", + ] + let json = "{" + jsonKeys.enumerated().map { "\"\($1)\":\($0)" }.joined(separator: ",") + "}" + let data = Data(json.utf8) + let foundation = JSONDecoder(); foundation.keyDecodingStrategy = .convertFromSnakeCase + let reer = ReerJSONDecoder(); reer.keyDecodingStrategy = .convertFromSnakeCase + let expected = try foundation.decode(AllKeys.self, from: data).keys + let actual = try reer.decode(AllKeys.self, from: data).keys + #expect(actual == expected) + } + + @Test func snakeCaseFastPathMatchesReference() { + let alphabet: [Character] = ["a", "B", "z", "Z", "_", "_", "_", "0", "9", "-", ".", "é"] + var generator = SystemRandomNumberGenerator() + for _ in 0..<20_000 { + let length = Int.random(in: 0...12, using: &generator) + let key = String((0..