diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6316de7..38c45c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,11 @@ jobs: - name: Verify updater contract run: swift run Pesty --verify-updater + - name: Verify release memory performance + run: | + swift build -c release + scripts/test_memory_performance.sh .build/release/Pesty + - name: Build and verify release package run: | VERSION=0.0.0 BUILD="${GITHUB_RUN_NUMBER}" ./scripts/release_build.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ef28696..f39c405 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -94,6 +94,11 @@ jobs: - name: Verify updater contract run: swift run Pesty --verify-updater + - name: Verify release memory performance + run: | + swift build -c release + scripts/test_memory_performance.sh .build/release/Pesty + - name: Build ad-hoc signed release env: VERSION: ${{ steps.version.outputs.value }} diff --git a/Sources/Pesty/AppController.swift b/Sources/Pesty/AppController.swift index 9f3250f..564d336 100644 --- a/Sources/Pesty/AppController.swift +++ b/Sources/Pesty/AppController.swift @@ -461,7 +461,6 @@ final class AppController: NSObject, NSApplicationDelegate { barController = BarWindowController() } barController?.show() - store.prepareSearchIndexForPanel() store.reconcileFromDiskInBackground() } diff --git a/Sources/Pesty/Models/ClipItem.swift b/Sources/Pesty/Models/ClipItem.swift index 647cac5..9838c78 100644 --- a/Sources/Pesty/Models/ClipItem.swift +++ b/Sources/Pesty/Models/ClipItem.swift @@ -87,6 +87,28 @@ struct ClipItem: Identifiable, Codable, Equatable, Sendable { .lowercased() } + var maximumSearchableUTF8Count: Int { + let components = [customTitle, text, sourceAppName, colorHex] + .compactMap { $0 } + let rawBytes = components.reduce(into: 0) { total, value in + total += value.utf8.count + } + fileURLs.reduce(into: 0) { total, value in + total += value.utf8.count + } + let (bytesWithSeparators, additionOverflow) = rawBytes + .addingReportingOverflow(max(0, fileURLs.count - 1)) + let (sourceBytes, componentOverflow) = bytesWithSeparators + .addingReportingOverflow(components.count) + // Unicode lowercasing can expand a scalar. Reserving four times the + // source byte count keeps the mmap builder bounded without committing + // those untouched virtual pages to physical memory. + let (maximum, multiplicationOverflow) = sourceBytes + .multipliedReportingOverflow(by: 4) + return additionOverflow || componentOverflow || multiplicationOverflow + ? Int.max + : maximum + } + func sameContent(as other: ClipItem) -> Bool { guard type == other.type else { return false } switch type { diff --git a/Sources/Pesty/Store/ClipboardSearchEngine.swift b/Sources/Pesty/Store/ClipboardSearchEngine.swift index 379d417..606a484 100644 --- a/Sources/Pesty/Store/ClipboardSearchEngine.swift +++ b/Sources/Pesty/Store/ClipboardSearchEngine.swift @@ -1,12 +1,15 @@ import Foundation +import Darwin struct ClipboardSearchIndex: Sendable { let source: BarSource let contentRevision: UInt64 - // Keep the original copy-on-write array instead of duplicating every - // ClipItem struct into the index. Only normalized search bytes allocate. + // The array remains a copy-on-write view of the store. Normalized bytes + // live in an mmap-backed Data so clearing search can return all index pages + // to the OS instead of leaving them in malloc's resident-page cache. let items: [ClipItem] - let searchableBytes: [Data] + let searchableBytes: Data + let searchableRanges: [Range] } struct ClipboardSearchResult: Sendable { @@ -23,18 +26,88 @@ enum ClipboardSearchEngine { source: BarSource, contentRevision: UInt64 ) -> ClipboardSearchIndex? { - var searchableBytes: [Data] = [] - searchableBytes.reserveCapacity(items.count) - for (offset, item) in items.enumerated() { - if offset.isMultiple(of: 8), Task.isCancelled { return nil } - searchableBytes.append(Data(item.searchableText.utf8)) + var mappedCapacity = 0 + for item in items { + let (nextCapacity, overflow) = mappedCapacity.addingReportingOverflow( + item.maximumSearchableUTF8Count + ) + guard !overflow else { return nil } + mappedCapacity = nextCapacity } guard !Task.isCancelled else { return nil } + + if mappedCapacity == 0 { + return ClipboardSearchIndex( + source: source, + contentRevision: contentRevision, + items: items, + searchableBytes: Data(), + searchableRanges: Array(repeating: 0..<0, count: items.count) + ) + } + + guard let mappedBytes = mmap( + nil, + mappedCapacity, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, + -1, + 0 + ), mappedBytes != MAP_FAILED else { + return nil + } + + var ranges: [Range] = [] + ranges.reserveCapacity(items.count) + var offset = 0 + for (itemOffset, item) in items.enumerated() { + if itemOffset.isMultiple(of: 8), Task.isCancelled { + munmap(mappedBytes, mappedCapacity) + return nil + } + let searchableText = item.searchableText + let byteCount = searchableText.utf8.count + guard byteCount <= mappedCapacity - offset else { + munmap(mappedBytes, mappedCapacity) + return nil + } + let destination = mappedBytes.advanced(by: offset) + let copiedContiguously = searchableText.utf8 + .withContiguousStorageIfAvailable { bytes -> Bool in + if let baseAddress = bytes.baseAddress, !bytes.isEmpty { + memcpy(destination, baseAddress, bytes.count) + } + return true + } ?? false + if !copiedContiguously { + var byteOffset = 0 + for byte in searchableText.utf8 { + destination.storeBytes(of: byte, toByteOffset: byteOffset, + as: UInt8.self) + byteOffset += 1 + } + } + ranges.append(offset..<(offset + byteCount)) + offset += byteCount + } + guard !Task.isCancelled else { + munmap(mappedBytes, mappedCapacity) + return nil + } + + let searchableBytes = Data( + bytesNoCopy: mappedBytes, + count: offset, + deallocator: .custom { pointer, _ in + munmap(pointer, mappedCapacity) + } + ) return ClipboardSearchIndex( source: source, contentRevision: contentRevision, items: items, - searchableBytes: searchableBytes + searchableBytes: searchableBytes, + searchableRanges: ranges ) } @@ -52,8 +125,12 @@ enum ClipboardSearchEngine { for (offset, candidate) in candidates.enumerated() { if offset.isMultiple(of: 8), Task.isCancelled { return nil } guard index.items.indices.contains(candidate), - index.searchableBytes.indices.contains(candidate) else { continue } - if contains(needle, in: index.searchableBytes[candidate]) { + index.searchableRanges.indices.contains(candidate) else { continue } + if contains( + needle, + in: index.searchableBytes, + range: index.searchableRanges[candidate] + ) { matchingIndices.append(candidate) matchingItems.append(index.items[candidate]) } @@ -66,19 +143,23 @@ enum ClipboardSearchEngine { ) } - private static func contains(_ needle: Data, in haystack: Data) -> Bool { + private static func contains( + _ needle: Data, + in haystack: Data, + range: Range + ) -> Bool { guard !needle.isEmpty else { return true } - guard haystack.count >= needle.count else { return false } - guard haystack.count > cancellableChunkSize else { - return haystack.range(of: needle) != nil + guard range.count >= needle.count else { return false } + guard range.count > cancellableChunkSize else { + return haystack.range(of: needle, in: range) != nil } let overlap = max(0, needle.count - 1) - var start = 0 - while start < haystack.count { + var start = range.lowerBound + while start < range.upperBound { if Task.isCancelled { return false } - let upperBound = min(haystack.count, start + cancellableChunkSize) - let lowerBound = max(0, start - overlap) + let upperBound = min(range.upperBound, start + cancellableChunkSize) + let lowerBound = max(range.lowerBound, start - overlap) if haystack.range(of: needle, in: lowerBound..() var merged: [ClipItem] = [] for item in combined - where seen.insert(contentKey(item)).inserted { + where seen.insert(contentDigest(item)).inserted { merged.append(item) } @@ -373,7 +373,10 @@ final class ClipboardStore { searchTask = nil searchGeneration &+= 1 stripContentRevision &+= 1 + scheduleSearchIndexEviction() } else { + searchIndexEvictionTask?.cancel() + searchIndexEvictionTask = nil scheduleSearchResultsUpdate(debounce: true) } } @@ -398,7 +401,7 @@ final class ClipboardStore { @ObservationIgnored private var searchIndexTask: Task? @ObservationIgnored private var searchIndexGeneration: UInt64 = 0 @ObservationIgnored private var searchIndexBuildIsDeferred = false - @ObservationIgnored private var searchIndexPrewarmingEnabled = false + @ObservationIgnored private var searchIndexEvictionTask: Task? @ObservationIgnored private var searchCandidateCache: [String: [Int]] = [:] @ObservationIgnored private var searchCandidateCacheOrder: [String] = [] @ObservationIgnored private var searchTask: Task? @@ -579,6 +582,8 @@ final class ClipboardStore { private func invalidateSearchIndexAndRefreshResults() { searchIndexTask?.cancel() searchIndexTask = nil + searchIndexEvictionTask?.cancel() + searchIndexEvictionTask = nil searchIndexGeneration &+= 1 searchIndexBuildIsDeferred = false discardSearchIndexOffMainActor() @@ -587,30 +592,44 @@ final class ClipboardStore { filteredSearchIndices.removeAll(keepingCapacity: true) if normalizedSearchQuery(searchText).isEmpty { - if searchIndexPrewarmingEnabled { - scheduleSearchIndexBuild(deferred: true) - } + // A capture invalidates the complete normalized byte index. Do not + // immediately rebuild it while persistence is encoding the same + // history. The next real query builds a fresh index on demand. } else { scheduleSearchResultsUpdate(debounce: false) } } private func discardSearchIndexOffMainActor() { - let discardedIndex = searchIndex searchIndex = nil - guard let discardedIndex else { return } - Task.detached(priority: .utility) { - withExtendedLifetime(discardedIndex) {} - } } - func prepareSearchIndexForPanel() { - searchIndexPrewarmingEnabled = true - let expectedRevision = searchContentRevision(for: source) - guard searchIndex?.source != source - || searchIndex?.contentRevision != expectedRevision else { return } - guard searchIndexTask == nil else { return } - scheduleSearchIndexBuild(deferred: true) + private func scheduleSearchIndexEviction() { + searchIndexEvictionTask?.cancel() + guard searchIndex != nil || searchIndexTask != nil else { + searchIndexEvictionTask = nil + return + } + searchIndexEvictionTask = Task { [weak self] in + do { + try await Task.sleep(for: .seconds(3)) + } catch { + return + } + guard let self, + self.normalizedSearchQuery(self.searchText).isEmpty else { + return + } + self.searchIndexTask?.cancel() + self.searchIndexTask = nil + self.searchIndexGeneration &+= 1 + self.discardSearchIndexOffMainActor() + self.searchCandidateCache.removeAll(keepingCapacity: false) + self.searchCandidateCacheOrder.removeAll(keepingCapacity: false) + self.filteredSearchItems.removeAll(keepingCapacity: false) + self.filteredSearchIndices.removeAll(keepingCapacity: false) + self.searchIndexEvictionTask = nil + } } private func scheduleSearchResultsUpdate(debounce: Bool) { @@ -810,6 +829,13 @@ final class ClipboardStore { } } + func waitForSearchIndexForAutomatedTest() async { + guard ClipboardStore.automatedTestBase != nil else { return } + while let task = searchIndexTask { + _ = await task.value + } + } + func searchDiagnosticsForAutomatedTest() -> ( indexCount: Int, buildCount: Int, @@ -1054,6 +1080,21 @@ final class ClipboardStore { saveNow() } + func replaceHistoryForAutomatedMemoryTest(_ items: [ClipItem]) { + guard ClipboardStore.automatedTestBase != nil, + ProcessInfo.processInfo.environment["PESTY_AUTOMATED_UI_TEST"] + == "memory-seed" else { + return + } + history = items + pinboards = [] + deletionTombstones = [:] + source = .history + searchText = "" + selectedID = items.first?.id + saveNow() + } + func replaceHistoryForAutomatedSettingsCountTest(_ items: [ClipItem]) { guard ClipboardStore.automatedTestBase != nil, ProcessInfo.processInfo.environment["PESTY_AUTOMATED_UI_TEST"] @@ -1352,9 +1393,7 @@ final class ClipboardStore { let needsMaterialization = loadsFromMetadataCache && storeNeedsMaterializationBeforeReading let initialSnapshot = usesIncrementalCloudSync - ? IncrementalCloudSync.localSnapshot( - in: incrementalLocalDirectory - ) + ? incrementalCloudSync?.initialSnapshot : readSnapshot( at: loadsFromMetadataCache ? metadataCacheURL : storeURL ) @@ -2003,6 +2042,14 @@ final class ClipboardStore { } } + func requestIncrementalCompactionForAutomatedMemoryTest() { + guard ClipboardStore.automatedTestBase != nil, + ProcessInfo.processInfo.environment["PESTY_AUTOMATED_UI_TEST"] + == "memory-measure" else { return } + incrementalCompactionRequested = true + scheduleIncrementalSync() + } + private var mergeContext: ClipboardMergeContext { ClipboardMergeContext( history: history, diff --git a/Sources/Pesty/Store/IncrementalCloudSync.swift b/Sources/Pesty/Store/IncrementalCloudSync.swift index 4ac8f11..c21a2b9 100644 --- a/Sources/Pesty/Store/IncrementalCloudSync.swift +++ b/Sources/Pesty/Store/IncrementalCloudSync.swift @@ -1,4 +1,5 @@ import CryptoKit +import Darwin import Foundation import OSLog @@ -88,11 +89,13 @@ actor IncrementalCloudSync { private let checkpointsDirectory: URL private let imagesDirectory: URL private let imageRetirementGrace: TimeInterval + nonisolated let initialSnapshot: ClipboardStoreSnapshot? private let logger = Logger( subsystem: "com.bifrostproxy.pesty", category: "incremental-cloud-sync" ) private var state: State? + private var stateRequiresPersistence = false private var lastImageGarbageCollectionAt: Date? init( @@ -103,7 +106,8 @@ actor IncrementalCloudSync { self.localDirectory = localDirectory self.cloudDirectory = cloudDirectory self.imageRetirementGrace = imageRetirementGrace - stateURL = localDirectory.appendingPathComponent("state.json") + let resolvedStateURL = localDirectory.appendingPathComponent("state.json") + stateURL = resolvedStateURL outboxDirectory = localDirectory.appendingPathComponent( "outbox", isDirectory: true @@ -118,15 +122,17 @@ actor IncrementalCloudSync { ) imagesDirectory = cloudDirectory.deletingLastPathComponent() .appendingPathComponent("images", isDirectory: true) + let loaded = Self.loadPersistedState(at: resolvedStateURL) + state = loaded.state + stateRequiresPersistence = loaded.requiresPersistence + initialSnapshot = loaded.state?.snapshot } nonisolated static func localSnapshot(in localDirectory: URL) -> ClipboardStoreSnapshot? { - let url = localDirectory.appendingPathComponent("state.json") - guard let data = try? Data(contentsOf: url), - let state = try? JSONDecoder().decode(State.self, from: data), - state.formatVersion == 2 else { return nil } - return state.snapshot + loadPersistedState( + at: localDirectory.appendingPathComponent("state.json") + ).state?.snapshot } func synchronize( @@ -195,10 +201,12 @@ actor IncrementalCloudSync { allRecordsFrom expected: ClipboardStoreSnapshot ) -> Bool { let migratedHistory = Dictionary( - uniqueKeysWithValues: migrated.history.map { (contentKey($0), $0) } + uniqueKeysWithValues: migrated.history.map { + (contentVersionKey($0), $0) + } ) guard expected.history.allSatisfy({ item in - guard let actual = migratedHistory[contentKey(item)] else { + guard let actual = migratedHistory[contentVersionKey(item)] else { return false } return actual.createdAt >= item.createdAt @@ -212,7 +220,9 @@ actor IncrementalCloudSync { actual.name == board.name, actual.colorHex == board.colorHex else { return false } return board.items.allSatisfy { expectedItem in - actual.items.contains { contentKey($0) == contentKey(expectedItem) } + actual.items.contains { + contentVersionKey($0) == contentVersionKey(expectedItem) + } } }) else { return false } @@ -252,12 +262,17 @@ actor IncrementalCloudSync { @discardableResult private func loadStateIfNeeded() -> Bool { - guard state == nil else { return false } + guard state == nil else { + let requiresPersistence = stateRequiresPersistence + stateRequiresPersistence = false + return requiresPersistence + } if let decoded = try? Data(contentsOf: stateURL), - let saved = try? JSONDecoder().decode(State.self, from: decoded), + var saved = try? JSONDecoder().decode(State.self, from: decoded), saved.formatVersion == 2 { + let migrated = Self.normalizeHistoryVersionKeys(in: &saved) state = saved - return false + return migrated } state = State( deviceID: UUID(), @@ -327,12 +342,14 @@ actor IncrementalCloudSync { var result: [Record] = [] let now = Date() let oldHistory = Dictionary( - uniqueKeysWithValues: previous.history.map { (contentKey($0), $0) } + uniqueKeysWithValues: previous.history.map { + (contentVersionKey($0), $0) + } ) for item in next.history - where oldHistory[contentKey(item)] != item { + where oldHistory[contentVersionKey(item)] != item { result.append(Record( - recordedAt: oldHistory[contentKey(item)] == nil + recordedAt: oldHistory[contentVersionKey(item)] == nil ? item.createdAt : now, operation: .historyUpsert(item) @@ -554,7 +571,7 @@ actor IncrementalCloudSync { private func pullRemoteCheckpoints() async -> Bool { guard let initialState = state else { return false } - var candidates: [(CheckpointManifest, CheckpointPayload)] = [] + var candidates: [(URL, CheckpointManifest)] = [] let appliedIDs = initialState.appliedCheckpointIDs ?? [] for manifestURL in checkpointManifestURLs() { if let checkpointID = UUID( @@ -573,30 +590,34 @@ actor IncrementalCloudSync { guard let manifest = coordinatedRead( CheckpointManifest.self, from: manifestURL - ), manifest.formatVersion == 2, - let payload = await readCheckpoint( - manifest, - manifestURL: manifestURL - ) else { continue } - candidates.append((manifest, payload)) + ), manifest.formatVersion == 2 else { continue } + candidates.append((manifestURL, manifest)) } candidates.sort { - if $0.0.createdAt != $1.0.createdAt { - return $0.0.createdAt < $1.0.createdAt + if $0.1.createdAt != $1.1.createdAt { + return $0.1.createdAt < $1.1.createdAt } - return $0.0.id.uuidString < $1.0.id.uuidString + return $0.1.id.uuidString < $1.1.id.uuidString } guard var current = state else { return false } var appliedAny = false - for (manifest, payload) in candidates { + for (manifestURL, manifest) in candidates { guard !(current.appliedCheckpointIDs ?? []).contains( manifest.id ) else { continue } + guard let payload = await readCheckpoint( + manifest, + manifestURL: manifestURL + ) else { continue } applyCheckpoint(payload, manifest: manifest, to: ¤t) appliedAny = true } guard appliedAny else { return false } state = current + // JSONDecoder has released the assembled checkpoint bytes by this + // point. Return their now-empty malloc pages so a one-time first sync + // does not become the app's permanent idle footprint. + _ = malloc_zone_pressure_relief(nil, 0) return true } @@ -608,7 +629,10 @@ actor IncrementalCloudSync { var records: [Record] = [] records += payload.snapshot.history.map { item in Record( - recordedAt: payload.historyVersions[contentKey(item)] + recordedAt: historyVersion( + for: item, + in: payload.historyVersions + ) ?? item.createdAt, operation: .historyUpsert(item) ) @@ -700,9 +724,6 @@ actor IncrementalCloudSync { includedCheckpointIDs: included, retiredImageVersions: current.retiredImageVersions ?? [:] ) - guard let payloadData = try? JSONEncoder().encode(payload) else { - return (false, false) - } let checkpointDirectory = checkpointsDirectory .appendingPathComponent( current.deviceID.uuidString.lowercased(), @@ -723,26 +744,17 @@ actor IncrementalCloudSync { return (false, false) } - var chunks: [CheckpointChunk] = [] - for (index, data) in splitCheckpointPayload(payloadData).enumerated() { - let name = String(format: "chunk-%05d.part", index) - let url = checkpointDirectory.appendingPathComponent(name) - guard coordinatedWrite(data, to: url) else { - return (false, false) - } - chunks.append(CheckpointChunk( - name: name, - byteCount: data.count, - sha256: sha256(data) - )) - } + guard let streamedPayload = writeCheckpointPayload( + payload, + to: checkpointDirectory + ) else { return (false, false) } let manifest = CheckpointManifest( formatVersion: 2, id: checkpointID, deviceID: current.deviceID, createdAt: Date(), - payloadSHA256: sha256(payloadData), - chunks: chunks, + payloadSHA256: streamedPayload.sha256, + chunks: streamedPayload.chunks, coveredBatchVersions: current.appliedBatchVersions, includedCheckpointIDs: included ) @@ -754,14 +766,12 @@ actor IncrementalCloudSync { CheckpointManifest.self, from: manifestURL ), - let verifiedPayload = await readCheckpoint( + verifyCheckpointChunks( verifiedManifest, - manifestURL: manifestURL + in: checkpointDirectory ), - contains( - verifiedPayload.snapshot, - allRecordsFrom: current.snapshot - ) else { return (false, false) } + contains(payload.snapshot, allRecordsFrom: current.snapshot) + else { return (false, false) } guard var latest = state else { return (false, false) } var applied = latest.appliedCheckpointIDs ?? [] @@ -769,7 +779,7 @@ actor IncrementalCloudSync { latest.appliedCheckpointIDs = applied var verifiedRetirements = latest.verifiedRetiredImageVersions ?? [:] mergeVersions( - verifiedPayload.retiredImageVersions, + payload.retiredImageVersions, into: &verifiedRetirements ) latest.verifiedRetiredImageVersions = verifiedRetirements @@ -818,7 +828,7 @@ actor IncrementalCloudSync { for record in batch.records { switch record.operation { case .historyUpsert(let item): - let key = contentKey(item) + let key = contentVersionKey(item) guard record.recordedAt >= (state.historyVersions[key] ?? .distantPast) else { continue } if let deletion = deletionMap(state.snapshot)[ @@ -828,7 +838,7 @@ actor IncrementalCloudSync { } state.historyVersions[key] = record.recordedAt state.snapshot.history.removeAll { - contentKey($0) == key + contentVersionKey($0) == key } state.snapshot.history.append(item) state.snapshot.history.sort { $0.createdAt > $1.createdAt } @@ -918,6 +928,52 @@ actor IncrementalCloudSync { ) } + private nonisolated static func loadPersistedState( + at url: URL + ) -> (state: State?, requiresPersistence: Bool) { + guard let data = try? Data(contentsOf: url, options: .mappedIfSafe), + var saved = try? JSONDecoder().decode(State.self, from: data), + saved.formatVersion == 2 else { + return (nil, false) + } + let migrated = normalizeHistoryVersionKeys(in: &saved) + return (saved, migrated) + } + + private nonisolated static func normalizeHistoryVersionKeys( + in state: inout State + ) -> Bool { + var normalized: [String: Date] = [:] + normalized.reserveCapacity(state.historyVersions.count) + var changed = false + for (key, date) in state.historyVersions { + let normalizedKey: String + if key.utf8.count == 64, + key.utf8.allSatisfy({ byte in + (48...57).contains(byte) || (97...102).contains(byte) + }) { + normalizedKey = key + } else { + normalizedKey = digestContentKey(key) + } + normalized[normalizedKey] = max( + normalized[normalizedKey] ?? .distantPast, + date + ) + changed = changed || normalizedKey != key + } + if changed { + state.historyVersions = normalized + } + return changed + } + + private nonisolated static func digestContentKey(_ key: String) -> String { + SHA256.hash(data: Data(key.utf8)) + .map { String(format: "%02x", $0) } + .joined() + } + private func deletionMap(_ snapshot: ClipboardStoreSnapshot) -> [String: ClipDeletionTombstone] { Dictionary(uniqueKeysWithValues: (snapshot.deletions ?? []).map { @@ -942,7 +998,7 @@ actor IncrementalCloudSync { } } - private func contentKey(_ item: ClipItem) -> String { + private func legacyContentKey(_ item: ClipItem) -> String { switch item.type { case .image: "img:" + (item.imageHash ?? item.imageFileName @@ -956,10 +1012,20 @@ actor IncrementalCloudSync { } } + private func contentVersionKey(_ item: ClipItem) -> String { + contentDigest(item) + } + + private func historyVersion( + for item: ClipItem, + in versions: [String: Date] + ) -> Date? { + versions[contentVersionKey(item)] + ?? versions[legacyContentKey(item)] + } + private func contentDigest(_ item: ClipItem) -> String { - SHA256.hash(data: Data(contentKey(item).utf8)) - .map { String(format: "%02x", $0) } - .joined() + Self.digestContentKey(legacyContentKey(item)) } private func batchFileName(_ batch: Batch) -> String { @@ -1016,17 +1082,124 @@ actor IncrementalCloudSync { .filter { $0.lastPathComponent == "manifest.json" } } - private func splitCheckpointPayload(_ data: Data) -> [Data] { + private func writeCheckpointPayload( + _ payload: CheckpointPayload, + to directory: URL + ) -> (chunks: [CheckpointChunk], sha256: String)? { let chunkSize = 256_000 - guard !data.isEmpty else { return [Data()] } - var chunks: [Data] = [] - var offset = 0 - while offset < data.count { - let end = min(data.count, offset + chunkSize) - chunks.append(data.subdata(in: offset..(_ value: T) throws { + try append(encoder.encode(value)) + } + + func appendArray(_ values: [T]) throws { + try append("[") + for (index, value) in values.enumerated() { + if index > 0 { try append(",") } + try appendEncoded(value) + } + try append("]") + } + + do { + try append("{\"formatVersion\":") + try appendEncoded(payload.formatVersion) + try append(",\"snapshot\":{\"history\":") + try appendArray(payload.snapshot.history) + try append(",\"pinboards\":") + try appendArray(payload.snapshot.pinboards) + try append(",\"configuration\":") + if let configuration = payload.snapshot.configuration { + try appendEncoded(configuration) + } else { + try append("null") + } + try append(",\"deletions\":") + if let deletions = payload.snapshot.deletions { + try appendArray(deletions) + } else { + try append("null") + } + try append("},\"historyVersions\":") + try appendEncoded(payload.historyVersions) + try append(",\"boardVersions\":") + try appendEncoded(payload.boardVersions) + try append(",\"deletedBoardVersions\":") + try appendEncoded(payload.deletedBoardVersions) + try append(",\"coveredBatchVersions\":") + try appendEncoded(payload.coveredBatchVersions) + try append(",\"includedCheckpointIDs\":") + try appendEncoded(payload.includedCheckpointIDs) + try append(",\"retiredImageVersions\":") + try appendEncoded(payload.retiredImageVersions) + try append("}") + try flushChunk() + } catch { + logCheckpointError("stream", error: error) + return nil + } + + let digest = payloadHasher.finalize() + .map { String(format: "%02x", $0) } + .joined() + return (chunks, digest) + } + + private func verifyCheckpointChunks( + _ manifest: CheckpointManifest, + in directory: URL + ) -> Bool { + var payloadHasher = SHA256() + for chunk in manifest.chunks { + guard let data = coordinatedReadData( + from: directory.appendingPathComponent(chunk.name) + ), data.count == chunk.byteCount, + sha256(data) == chunk.sha256 else { return false } + payloadHasher.update(data: data) + } + let digest = payloadHasher.finalize() + .map { String(format: "%02x", $0) } + .joined() + return digest == manifest.payloadSHA256 } private func garbageCollectBatches( @@ -1232,20 +1405,112 @@ actor IncrementalCloudSync { private func persistState() { guard let state else { return } + let temporaryURL = stateURL.deletingLastPathComponent() + .appendingPathComponent(".state-\(UUID().uuidString).tmp") do { - let data = try JSONEncoder().encode(state) - try data.write(to: stateURL, options: .atomic) + try writeState(state, to: temporaryURL) try FileManager.default.setAttributes( [.posixPermissions: 0o600], - ofItemAtPath: stateURL.path + ofItemAtPath: temporaryURL.path ) + guard rename(temporaryURL.path, stateURL.path) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } } catch { + try? FileManager.default.removeItem(at: temporaryURL) logger.error( "Failed to persist incremental sync state domain=\((error as NSError).domain, privacy: .public) code=\((error as NSError).code)" ) } } + private func writeState(_ state: State, to url: URL) throws { + let fm = FileManager.default + guard fm.createFile(atPath: url.path, contents: nil) else { + throw CocoaError(.fileWriteUnknown) + } + let handle = try FileHandle(forWritingTo: url) + let encoder = JSONEncoder() + + func write(_ literal: String) throws { + try handle.write(contentsOf: Data(literal.utf8)) + } + func writeEncoded(_ value: T) throws { + try handle.write(contentsOf: encoder.encode(value)) + } + func writeArray(_ values: [T]) throws { + try write("[") + for (index, value) in values.enumerated() { + if index > 0 { try write(",") } + try writeEncoded(value) + } + try write("]") + } + + do { + try write("{\"formatVersion\":") + try writeEncoded(state.formatVersion) + try write(",\"deviceID\":") + try writeEncoded(state.deviceID) + try write(",\"nextSequence\":") + try writeEncoded(state.nextSequence) + try write(",\"snapshot\":{\"history\":") + try writeArray(state.snapshot.history) + try write(",\"pinboards\":") + try writeArray(state.snapshot.pinboards) + try write(",\"configuration\":") + if let configuration = state.snapshot.configuration { + try writeEncoded(configuration) + } else { + try write("null") + } + try write(",\"deletions\":") + if let deletions = state.snapshot.deletions { + try writeArray(deletions) + } else { + try write("null") + } + try write("},\"historyVersions\":") + try writeEncoded(state.historyVersions) + try write(",\"boardVersions\":") + try writeEncoded(state.boardVersions) + try write(",\"deletedBoardVersions\":") + try writeEncoded(state.deletedBoardVersions) + try write(",\"appliedBatchVersions\":") + try writeEncoded(state.appliedBatchVersions) + try write(",\"appliedCheckpointIDs\":") + if let ids = state.appliedCheckpointIDs { + try writeEncoded(ids) + } else { + try write("null") + } + try write(",\"retiredImageVersions\":") + if let versions = state.retiredImageVersions { + try writeEncoded(versions) + } else { + try write("null") + } + try write(",\"verifiedRetiredImageVersions\":") + if let versions = state.verifiedRetiredImageVersions { + try writeEncoded(versions) + } else { + try write("null") + } + try write(",\"batchDirectoryModificationDates\":") + if let dates = state.batchDirectoryModificationDates { + try writeEncoded(dates) + } else { + try write("null") + } + try write("}") + try handle.synchronize() + try handle.close() + } catch { + try? handle.close() + throw error + } + } + private func coordinatedWrite(_ value: T, to url: URL) -> Bool { guard let data = try? JSONEncoder().encode(value) else { return false } diff --git a/Sources/Pesty/Util/AutomatedUITestRunner.swift b/Sources/Pesty/Util/AutomatedUITestRunner.swift index 3fbe04d..e2f2563 100644 --- a/Sources/Pesty/Util/AutomatedUITestRunner.swift +++ b/Sources/Pesty/Util/AutomatedUITestRunner.swift @@ -135,6 +135,45 @@ enum AutomatedUITestRunner { let maximumDurationMilliseconds: Int } + private struct MemorySeedResult: Codable { + let phase: String + let success: Bool + let historyCount: Int + let expectedCount: Int + let textBytes: Int + } + + private struct MemoryPerformanceResult: Codable { + let phase: String + let success: Bool + let historyCount: Int + let expectedCount: Int + let searchResultCount: Int + let firstSearchMilliseconds: Int + let maximumFirstSearchMilliseconds: Int + let currentFootprintBytes: UInt64 + let averageIdleFootprintBytes: UInt64 + let peakFootprintBytes: UInt64 + let launchFootprintBytes: UInt64 + let activeSearchFootprintBytes: UInt64 + let postSearchEvictionFootprintBytes: UInt64 + let postPersistenceFootprintBytes: UInt64 + let maximumAverageIdleFootprintBytes: UInt64 + let maximumPeakFootprintBytes: UInt64 + let reorderedItemPersisted: Bool + } + + private struct MemoryFreshSyncResult: Codable { + let phase: String + let success: Bool + let historyCount: Int + let expectedCount: Int + let currentFootprintBytes: UInt64 + let peakFootprintBytes: UInt64 + let maximumCurrentFootprintBytes: UInt64 + let maximumPeakFootprintBytes: UInt64 + } + private struct SettingsRecordCountResult: Codable { let phase: String let success: Bool @@ -284,6 +323,9 @@ enum AutomatedUITestRunner { let maximumInputCadenceDelayMilliseconds: Int let inputCadenceDelayLimitMilliseconds: Int let liveMutationSearchUpdated: Bool + let liveMutationSearchIndexCount: Int + let liveMutationSearchIndexBuildCountDelta: Int + let liveMutationFirstResultMatched: Bool let unicodeSearchCorrect: Bool let caseInsensitiveSearchCorrect: Bool } @@ -488,6 +530,18 @@ enum AutomatedUITestRunner { runPerformanceTest(controller: controller, runID: runID) return } + if phase == "memory-seed" { + seedMemoryPerformanceTest(controller: controller, runID: runID) + return + } + if phase == "memory-measure" { + runMemoryPerformanceTest(controller: controller, runID: runID) + return + } + if phase == "memory-fresh-sync" { + runMemoryFreshSyncTest(controller: controller) + return + } if phase == "panel-reconciliation" { runPanelReconciliationTest(controller: controller, runID: runID) return @@ -1023,12 +1077,15 @@ enum AutomatedUITestRunner { await controller.store.waitForSearchForAutomatedTest() let liveMutationDiagnostics = controller.store.searchDiagnosticsForAutomatedTest() + let liveMutationFirstResultMatched = + controller.store.visibleItems.first?.id + == liveMutationItem.id let liveMutationSearchUpdated = - liveMutationDiagnostics.indexCount == largeHistoryCount + liveMutationDiagnostics.indexCount + == controller.store.history.count && liveMutationDiagnostics.buildCount == buildCountBeforeLiveMutation + 1 - && controller.store.visibleItems.first?.id - == liveMutationItem.id + && liveMutationFirstResultMatched controller.store.searchText = "中文搜索性能" await controller.store.waitForSearchForAutomatedTest() let unicodeSearchCorrect = @@ -1101,6 +1158,13 @@ enum AutomatedUITestRunner { inputCadenceDelayLimitMilliseconds: inputCadenceDelayLimitMilliseconds, liveMutationSearchUpdated: liveMutationSearchUpdated, + liveMutationSearchIndexCount: + liveMutationDiagnostics.indexCount, + liveMutationSearchIndexBuildCountDelta: + liveMutationDiagnostics.buildCount + - buildCountBeforeLiveMutation, + liveMutationFirstResultMatched: + liveMutationFirstResultMatched, unicodeSearchCorrect: unicodeSearchCorrect, caseInsensitiveSearchCorrect: caseInsensitiveSearchCorrect @@ -3439,6 +3503,174 @@ enum AutomatedUITestRunner { ) } + private static func seedMemoryPerformanceTest( + controller: AppController, + runID: String + ) { + let itemCount = 2_200 + let payload = String(repeating: "memory-payload-", count: 1_570) + let items = (0.. (current: UInt64, peak: UInt64)? { + var info = rusage_info_v4() + let status = withUnsafeMutablePointer(to: &info) { pointer in + let opaquePointer = UnsafeMutableRawPointer(pointer) + .assumingMemoryBound(to: rusage_info_t?.self) + return proc_pid_rusage(getpid(), RUSAGE_INFO_V4, opaquePointer) + } + guard status == 0 else { return nil } + return ( + info.ri_phys_footprint, + info.ri_lifetime_max_phys_footprint + ) + } + private static func runPerformanceTest(controller: AppController, runID: String) { let itemCount = 1_000 let checkpointIndices = [0, 249, 499, 749, 999] @@ -4423,6 +4655,43 @@ enum AutomatedUITestRunner { FileHandle.standardOutput.write(Data("\n".utf8)) } + private static func writeMemorySeed(_ result: MemorySeedResult) { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + guard let data = try? encoder.encode(result) else { return } + FileHandle.standardOutput.write( + Data("AUTOMATED_MEMORY_SEED_RESULT ".utf8) + ) + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data("\n".utf8)) + } + + private static func writeMemoryPerformance( + _ result: MemoryPerformanceResult + ) { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + guard let data = try? encoder.encode(result) else { return } + FileHandle.standardOutput.write( + Data("AUTOMATED_MEMORY_PERFORMANCE_RESULT ".utf8) + ) + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data("\n".utf8)) + } + + private static func writeMemoryFreshSync( + _ result: MemoryFreshSyncResult + ) { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + guard let data = try? encoder.encode(result) else { return } + FileHandle.standardOutput.write( + Data("AUTOMATED_MEMORY_FRESH_SYNC_RESULT ".utf8) + ) + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data("\n".utf8)) + } + private static func writePanelReconciliation( _ result: PanelReconciliationResult ) { diff --git a/scripts/test_incremental_sync.sh b/scripts/test_incremental_sync.sh index 0619de0..e895a18 100755 --- a/scripts/test_incremental_sync.sh +++ b/scripts/test_incremental_sync.sh @@ -27,6 +27,34 @@ PESTY_AUTOMATED_TEST_ID="$current_id" \ test ! -e "$cloud_dir/store.json" +# Existing 1.7.0-1.7.3 states used the complete clipboard content as each +# historyVersions key. Rewrite this isolated synthetic state into that legacy +# shape, then prove the next launch migrates every key to a fixed digest without +# losing history. +device_a_state="$test_root/device-a/sync-v2-local/state.json" +legacy_state="$test_root/legacy-state.json" +jq ' + (.historyVersions | to_entries | map(.value) | max) as $latest + | .historyVersions = reduce .snapshot.history[] as $item + ({}; .["txt:" + ($item.text // "")] = $latest) + ' "$device_a_state" >"$legacy_state" +mv "$legacy_state" "$device_a_state" + +PESTY_AUTOMATED_TEST_DATA_DIR="$cloud_dir" \ +PESTY_AUTOMATED_INCREMENTAL_SYNC=1 \ +PESTY_AUTOMATED_INCREMENTAL_LOCAL_DIR="$test_root/device-a" \ +PESTY_AUTOMATED_TEST_DEFAULTS_SUITE="$suite" \ +PESTY_AUTOMATED_UI_TEST=restart-1 \ +PESTY_AUTOMATED_TEST_ID="$current_id" \ + "$binary" + +jq -e ' + (.historyVersions | keys | length) == 4 + and all(.historyVersions | keys[]; test("^[0-9a-f]{64}$")) + and ([.snapshot.history[].text // "" + | select(startswith("pesty-auto-"))] | length) == 4 + ' "$device_a_state" >/dev/null + # A fresh device reconstructs the baseline from current checkpoints and event # batches only. Legacy monolithic stores are intentionally unsupported. PESTY_AUTOMATED_TEST_DATA_DIR="$cloud_dir" \ diff --git a/scripts/test_memory_performance.sh b/scripts/test_memory_performance.sh new file mode 100755 index 0000000..2b6af6f --- /dev/null +++ b/scripts/test_memory_performance.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +binary="${1:-.build/release/Pesty}" +if [[ ! -x "$binary" ]]; then + echo "Memory performance test binary is missing: $binary" >&2 + exit 2 +fi + +test_root="$(mktemp -d)" +cloud_dir="$test_root/cloud" +local_dir="$test_root/device" +suite="com.bifrostproxy.pesty.memory.$(date +%s).$$" +run_id="memory-$(date +%s)-$$" + +cleanup() { + defaults delete "$suite" >/dev/null 2>&1 || true + find "$test_root" -depth -delete +} +trap cleanup EXIT + +defaults write "$suite" iCloudSync -bool true +defaults write "$suite" historyLimit -int 3000 +defaults write "$suite" historyLimitUnlimited -bool false + +common_environment=( + PESTY_AUTOMATED_TEST_DATA_DIR="$cloud_dir" + PESTY_AUTOMATED_INCREMENTAL_SYNC=1 + PESTY_AUTOMATED_INCREMENTAL_LOCAL_DIR="$local_dir" + PESTY_AUTOMATED_TEST_DEFAULTS_SUITE="$suite" + PESTY_AUTOMATED_TEST_ID="$run_id" +) + +seed_output="$test_root/seed.log" +set +e +env "${common_environment[@]}" \ + PESTY_AUTOMATED_UI_TEST=memory-seed \ + "$binary" >"$seed_output" +seed_exit_code=$? +set -e +seed_result="$(sed -n '/^AUTOMATED_MEMORY_SEED_RESULT /p' \ + "$seed_output" | tail -1)" +if [[ "$seed_exit_code" -ne 0 ]] \ + || [[ -z "$seed_result" ]] \ + || [[ "$(jq -r '.success' <<<"${seed_result#* }")" != "true" ]]; then + echo "Memory seed failed." >&2 + sed -n '1,120p' "$seed_output" >&2 + exit 1 +fi + +measure_output="$test_root/measure.log" +set +e +env "${common_environment[@]}" \ + PESTY_AUTOMATED_UI_TEST=memory-measure \ + "$binary" >"$measure_output" +measure_exit_code=$? +set -e +measure_result="$(sed -n '/^AUTOMATED_MEMORY_PERFORMANCE_RESULT /p' \ + "$measure_output" | tail -1)" +if [[ -z "$measure_result" ]]; then + echo "Memory measurement did not emit a result." >&2 + sed -n '1,160p' "$measure_output" >&2 + exit 1 +fi + +measure_json="${measure_result#AUTOMATED_MEMORY_PERFORMANCE_RESULT }" +if [[ "$measure_exit_code" -ne 0 ]] \ + || [[ "$(jq -r '.success' <<<"$measure_json")" != "true" ]]; then + echo "$measure_result" + echo "Memory performance limits failed." >&2 + exit 1 +fi + +fresh_output="$test_root/fresh.log" +set +e +env "${common_environment[@]}" \ + PESTY_AUTOMATED_INCREMENTAL_LOCAL_DIR="$test_root/fresh-device" \ + PESTY_AUTOMATED_UI_TEST=memory-fresh-sync \ + "$binary" >"$fresh_output" +fresh_exit_code=$? +set -e +fresh_result="$(sed -n '/^AUTOMATED_MEMORY_FRESH_SYNC_RESULT /p' \ + "$fresh_output" | tail -1)" +if [[ "$fresh_exit_code" -ne 0 ]] \ + || [[ -z "$fresh_result" ]] \ + || [[ "$(jq -r '.success' <<<"${fresh_result#* }")" != "true" ]]; then + echo "${fresh_result:-Memory fresh-device sync emitted no result.}" + echo "Fresh-device checkpoint memory limit failed." >&2 + exit 1 +fi + +state="$local_dir/sync-v2-local/state.json" +jq -e ' + (.historyVersions | keys | length) == (.snapshot.history | length) + and all(.historyVersions | keys[]; test("^[0-9a-f]{64}$")) + ' "$state" >/dev/null +text_bytes="$(jq '[.snapshot.history[].text // "" | utf8bytelength] | add' \ + "$state")" +state_bytes="$(stat -f '%z' "$state")" +if (( text_bytes < 45000000 )); then + echo "Synthetic history is too small to represent the production workload." >&2 + exit 1 +fi +if (( state_bytes > text_bytes * 5 / 4 )); then + echo "Incremental state $state_bytes bytes duplicates too much of the $text_bytes-byte history." >&2 + exit 1 +fi + +echo "$seed_result" +echo "$measure_result" +echo "$fresh_result" +printf 'MEMORY_STATE_RESULT {"stateBytes":%s,"textBytes":%s,"historyVersionKeyBytes":%s}\n' \ + "$state_bytes" \ + "$text_bytes" \ + "$(jq '[.historyVersions | keys[] | utf8bytelength] | add' "$state")"