diff --git a/Sources/StorageAndroidCaches.swift b/Sources/StorageAndroidCaches.swift index 8e73aa5c..11bf9a97 100644 --- a/Sources/StorageAndroidCaches.swift +++ b/Sources/StorageAndroidCaches.swift @@ -33,7 +33,7 @@ enum StorageAndroidCaches { id: "android.gradle_cache/rest", category: .android, title: String(localized: "storage.cache.gradleOther", defaultValue: "Other Gradle caches"), path: caches, sizeBytes: size.bytes, sizePartial: size.partial, - lastUsed: StorageCacheSupport.modified(caches), autoCleanable: false, blocker: blocked + lastUsed: StorageCacheSupport.modified(caches), autoCleanable: false, blocker: blocked, removal: .gradleRest )) } let distributions = StorageCacheSupport.children((gradle as NSString).appendingPathComponent("wrapper/dists")) @@ -71,7 +71,7 @@ enum StorageAndroidCaches { items.append(StorageCacheSupport.item( id: "android.project_build/" + path, category: .android, title: String(localized: "storage.cache.androidBuild", defaultValue: "Android build output") + ": " + url.deletingLastPathComponent().lastPathComponent, - path: path, automatic: false, blocker: blocker, timeout: timeout + path: path, automatic: false, blocker: blocker, timeout: timeout, removal: .untrackedBuild )) } } @@ -137,16 +137,19 @@ enum StorageAndroidCaches { StorageCacheSupport.item( id: "android.system_image/" + path, category: .android, title: String(localized: "storage.cache.androidImage", defaultValue: "Android system image") + ": " + path.replacingOccurrences(of: root + "/", with: ""), - path: path, automatic: false, blocker: blocker, timeout: timeout + path: path, automatic: false, blocker: blocker, timeout: timeout, removal: .unreferencedSystemImage ) } } static func remove(_ item: StorageCacheItem, context: StorageCacheContext) -> Bool { guard blocker(context) == nil else { return false } - if item.id.hasPrefix("android.project_build/"), let path = item.path, !safeBuild(path, context: context) { return false } - if item.id.hasPrefix("android.system_image/"), !systemImages(context, timeout: 0, blocker: nil).contains(where: { $0.id == item.id && $0.path == item.path }) { return false } - if item.id == "android.gradle_cache/rest" { + switch item.removal { + case .untrackedBuild: + if let path = item.path, !safeBuild(path, context: context) { return false } + case .unreferencedSystemImage: + if !systemImages(context, timeout: 0, blocker: nil).contains(where: { $0.id == item.id && $0.path == item.path }) { return false } + case .gradleRest: guard let path = item.path, StorageCacheSupport.directory(path), let names = try? FileManager.default.contentsOfDirectory(atPath: path) else { return false } do { @@ -155,6 +158,7 @@ enum StorageAndroidCaches { } return true } catch { return false } + default: break } return StorageCacheSupport.remove(item.path) } diff --git a/Sources/StorageCacheSupport.swift b/Sources/StorageCacheSupport.swift index 5c96ae56..25506e9a 100644 --- a/Sources/StorageCacheSupport.swift +++ b/Sources/StorageCacheSupport.swift @@ -36,19 +36,39 @@ struct StorageCacheContext { let projects = WorktreeStorage.discoverRepositories(candidateDirectories: candidates) .flatMap { $0.worktrees.filter { !$0.isBare }.map(\.path) } var environment = ProcessInfo.processInfo.environment - let shell = environment["SHELL"] ?? "/bin/zsh" - if let path = run(shell, ["-l", "-c", "printf '%s' \"$PATH\""], "/", 5), !path.isEmpty, !path.contains("\n") { - environment["PATH"] = path + if let loginShellPATH { environment["PATH"] = loginShellPATH } + return Self(environment: environment, command: liveCommand(environment: environment), projectDirectories: projects) + } + + /// Resolved once per app run: a login shell runs the user's rc files, which can take seconds. + private static let loginShellPATH: String? = { + let shell = ProcessInfo.processInfo.environment["SHELL"] ?? "/bin/zsh" + guard let path = run(shell, ["-l", "-c", "printf '%s' \"$PATH\""], "/", 5), !path.isEmpty, !path.contains("\n") else { + return nil } - let commandEnvironment = environment - return Self(environment: environment, command: { executable, arguments, directory, timeout in - let result = CanonicalSubprocessRunner.run( - executable: executable, arguments: arguments, currentDirectory: directory, - timeout: timeout, stdoutLimit: 4 * 1024 * 1024, stderrLimit: 128 * 1024, - environmentOverrides: commandEnvironment - ) + return path + }() + + /// Project directories come from terminal-reported paths, so git runs with the automatic + /// hardening (`CanonicalSubprocessRunner.runAutomaticGit`). Git gets no environment + /// overrides: they would be layered over, and could undo, its hardening environment. + static func liveCommand(environment: [String: String]) -> (String, [String], String, TimeInterval) -> String? { + { executable, arguments, directory, timeout in + let stdoutLimit = 4 * 1024 * 1024 + let stderrLimit = 128 * 1024 + let result = (executable as NSString).lastPathComponent == "git" + ? CanonicalSubprocessRunner.runAutomaticGit( + arguments: arguments, currentDirectory: directory, timeout: timeout, + stdoutLimit: stdoutLimit, stderrLimit: stderrLimit, + executableURL: executable.contains("/") ? URL(fileURLWithPath: executable) : nil + ) + : CanonicalSubprocessRunner.run( + executable: executable, arguments: arguments, currentDirectory: directory, + timeout: timeout, stdoutLimit: stdoutLimit, stderrLimit: stderrLimit, + environmentOverrides: environment + ) return result.outcome == .exited && result.exitStatus == 0 ? result.stdout : nil - }, projectDirectories: projects) + } } func at(_ date: Date) -> Self { var copy = self; copy.now = date; return copy } @@ -124,12 +144,14 @@ enum StorageCacheSupport { } static func item( id: String, category: StorageCategoryID, title: String, path: String, - automatic: Bool, blocker: String?, timeout: TimeInterval, lastUsed: Date? = nil + automatic: Bool, blocker: String?, timeout: TimeInterval, lastUsed: Date? = nil, + removal: StorageCacheRemoval = .directory ) -> StorageCacheItem { let size = timeout > 0 ? WorktreeStorage.directorySize(atPath: path, timeout: timeout) : (bytes: Int64(0), partial: true) return StorageCacheItem(id: id, category: category, title: title, path: path, sizeBytes: size.bytes, sizePartial: size.partial, - lastUsed: lastUsed ?? modified(path), autoCleanable: automatic, blocker: blocker) + lastUsed: lastUsed ?? modified(path), autoCleanable: automatic, blocker: blocker, + removal: removal) } static func remove(_ path: String?) -> Bool { guard let path, directory(path) else { return false } diff --git a/Sources/StorageCategories.swift b/Sources/StorageCategories.swift index 6cdd6959..1286e5d7 100644 --- a/Sources/StorageCategories.swift +++ b/Sources/StorageCategories.swift @@ -21,6 +21,26 @@ struct StorageCacheItem: Equatable, Sendable { let autoCleanable: Bool /// Why it must stay right now (for example "xcode_running"); nil means removable. let blocker: String? + /// Set by the scan. `clean` uses the strategy of a fresh scan, never a caller-supplied one. + let removal: StorageCacheRemoval +} + +/// How `clean` removes an item, decided when the item is scanned. +enum StorageCacheRemoval: Equatable, Sendable { + /// Delete the item's directory. + case directory + /// Delete the directory once no simulator is booted. + case simulatorCaches + /// `xcrun simctl delete unavailable`, only for the real user home. + case unavailableSimulators + /// Delete every child of the Gradle caches directory except build caches and transforms. + case gradleRest + /// Delete the directory once it still holds no tracked, dotenv or nested `.git` files. + case untrackedBuild + /// Delete the directory once no AVD references it. + case unreferencedSystemImage + /// Run the package manager with these arguments; without the tool, clear the cache's contents. + case packageTool([String]) } struct StorageCacheCleanOutcome: Equatable, Sendable { @@ -76,17 +96,22 @@ enum StorageCategories { ) -> StorageCacheCleanOutcome { var outcome = StorageCacheCleanOutcome() var seen = Set() - for item in items where seen.insert(item.id).inserted { + let requested = items.filter { seen.insert($0.id).inserted } + let categories = StorageCategoryID.allCases.filter { category in requested.contains { $0.category == category } } + let scanned = Dictionary( + scan(categories: categories, sizeWalkTimeout: 0, context: context.at(now)).map { ($0.id, $0) }, + uniquingKeysWith: { first, _ in first } + ) + for item in requested { // IDs are selectors, never authority to delete a caller-supplied path. - let current = scan(categories: [item.category], sizeWalkTimeout: 0, context: context.at(now)).first { $0.id == item.id } - guard let current, current.path == item.path else { + guard let current = scanned[item.id], current.category == item.category, current.path == item.path else { outcome.kept.append(.init(item: item, reason: "no_longer_eligible")) continue } let fresh = StorageCacheItem( id: current.id, category: current.category, title: current.title, path: current.path, sizeBytes: item.sizeBytes, sizePartial: item.sizePartial, lastUsed: current.lastUsed, - autoCleanable: current.autoCleanable, blocker: current.blocker + autoCleanable: current.autoCleanable, blocker: current.blocker, removal: current.removal ) if let blocker = fresh.blocker { outcome.kept.append(.init(item: fresh, reason: blocker)) @@ -124,7 +149,7 @@ enum StorageCategories { private static func measuredSize(_ item: StorageCacheItem) -> Int64 { guard let path = item.path else { return item.sizeBytes } guard StorageCacheSupport.directory(path) else { return 0 } - return item.id == "android.gradle_cache/rest" + return item.removal == .gradleRest ? StorageAndroidCaches.sizeOfRest(path, timeout: 20).bytes : WorktreeStorage.directorySize(atPath: path, timeout: 20).bytes } diff --git a/Sources/StoragePackageCaches.swift b/Sources/StoragePackageCaches.swift index ba700856..6b93ee71 100644 --- a/Sources/StoragePackageCaches.swift +++ b/Sources/StoragePackageCaches.swift @@ -80,22 +80,26 @@ enum StoragePackageCaches { id: "packages." + cache.id, category: .packages, title: cache.id.hasPrefix("yarn_berry") ? "Yarn Berry" : cache.manager, path: cache.path, automatic: cache.manager == "pnpm" && context.tool("pnpm") != nil || StorageCacheSupport.idle(date, now: context.now), - blocker: blocker(manager: cache.manager, context: context), timeout: timeout, lastUsed: date + blocker: blocker(manager: cache.manager, context: context), timeout: timeout, lastUsed: date, + removal: .packageTool(cleanArguments(cache)) ) } } + private static func cleanArguments(_ cache: Cache) -> [String] { + switch cache.manager { + case "bun": ["pm", "cache", "rm"] + case "npm": ["cache", "clean", "--force"] + case "pnpm": ["store", "prune"] + default: cache.id == "yarn_berry" ? ["cache", "clean", "--mirror"] : ["cache", "clean"] + } + } + static func remove(_ item: StorageCacheItem, context: StorageCacheContext) -> Bool { - guard let cache = caches(context).first(where: { "packages." + $0.id == item.id && $0.path == item.path }), + guard case .packageTool(let arguments) = item.removal, + let cache = caches(context).first(where: { "packages." + $0.id == item.id && $0.path == item.path }), blocker(manager: cache.manager, context: context) == nil else { return false } guard context.tool(cache.manager) != nil else { return StorageCacheSupport.clearContents(cache.path) } - let arguments: [String] - switch cache.manager { - case "bun": arguments = ["pm", "cache", "rm"] - case "npm": arguments = ["cache", "clean", "--force"] - case "pnpm": arguments = ["store", "prune"] - default: arguments = item.id == "packages.yarn_berry" ? ["cache", "clean", "--mirror"] : ["cache", "clean"] - } return context.execute(cache.manager, arguments, timeout: 120) != nil } } diff --git a/Sources/StorageXcodeCaches.swift b/Sources/StorageXcodeCaches.swift index f8f866eb..e3760899 100644 --- a/Sources/StorageXcodeCaches.swift +++ b/Sources/StorageXcodeCaches.swift @@ -48,7 +48,7 @@ enum StorageXcodeCaches { items.append(StorageCacheSupport.item( id: "xcode.simulator_caches", category: .xcode, title: String(localized: "storage.cache.simulatorCaches", defaultValue: "Simulator caches"), - path: caches, automatic: true, blocker: simulatorBlocker, timeout: timeout + path: caches, automatic: true, blocker: simulatorBlocker, timeout: timeout, removal: .simulatorCaches )) } if let unavailable = devices(context, unavailable: true) { @@ -61,7 +61,8 @@ enum StorageXcodeCaches { id: "xcode.unavailable_simulators", category: .xcode, title: String(localized: "storage.cache.unavailableSimulators", defaultValue: "Unavailable simulators"), path: nil, sizeBytes: sizes.reduce(0) { $0 + $1.bytes }, sizePartial: sizes.contains { $0.partial }, - lastUsed: paths.compactMap(StorageCacheSupport.modified).max(), autoCleanable: true, blocker: simulatorBlocker + lastUsed: paths.compactMap(StorageCacheSupport.modified).max(), autoCleanable: true, blocker: simulatorBlocker, + removal: .unavailableSimulators )) } } @@ -81,10 +82,10 @@ enum StorageXcodeCaches { static func remove(_ item: StorageCacheItem, context: StorageCacheContext) -> Bool { guard !running(context) else { return false } - if item.id == "xcode.unavailable_simulators" || item.id == "xcode.simulator_caches" { + if item.removal == .simulatorCaches || item.removal == .unavailableSimulators { guard let all = devices(context, unavailable: false), !all.contains(where: { $0.state == "Booted" }) else { return false } } - if item.id == "xcode.unavailable_simulators" { + if item.removal == .unavailableSimulators { // simctl always targets the real user device set; injected homes must never invoke it. guard context.home == WorktreeStorage.resolvedPath(FileManager.default.homeDirectoryForCurrentUser.path) else { return false } return context.execute("xcrun", ["simctl", "delete", "unavailable"], timeout: 120) != nil diff --git a/Sources/TerminalController+Storage.swift b/Sources/TerminalController+Storage.swift index 54e77be0..11b98a39 100644 --- a/Sources/TerminalController+Storage.swift +++ b/Sources/TerminalController+Storage.swift @@ -3,13 +3,17 @@ // snapshot the directories open in workspaces. import Foundation +private enum StorageTimestamp { + /// ISO8601DateFormatter is thread-safe; the handlers run on the socket thread. + static let formatter = ISO8601DateFormatter() +} + extension TerminalController { nonisolated func v2StorageScan(params: [String: Any]) -> V2CallResult { let openDirectories = v2MainSync { WorktreeStorage.openWorkspaceDirectories() } let entries = WorktreeStorage.scanAll(openDirectories: openDirectories) let policy = WorktreeRemovalPolicy.automatic(gracePeriodHours: ProgramaWorktreeSettings.autoRemoveAfterHours()) let now = Date() - let formatter = ISO8601DateFormatter() var payloads: [[String: Any]] = [] var totalBytes: Int64 = 0 @@ -26,7 +30,7 @@ extension TerminalController { "branch": v2OrNull(entry.branch), "size_bytes": entry.sizeBytes, "size_partial": entry.sizePartial, - "last_modified": v2OrNull(entry.lastModified.map { formatter.string(from: $0) }), + "last_modified": v2OrNull(entry.lastModified.map(StorageTimestamp.formatter.string(from:))), "programa_created": entry.isProgramaCreated, "dirty": v2OrNull(entry.isDirty), "ignored_env_files": v2OrNull(entry.hasIgnoredEnvFiles), @@ -127,7 +131,7 @@ extension TerminalController { "path": v2OrNull(item.path), "size_bytes": item.sizeBytes, "size_partial": item.sizePartial, - "last_used": v2OrNull(item.lastUsed.map { ISO8601DateFormatter().string(from: $0) }), + "last_used": v2OrNull(item.lastUsed.map(StorageTimestamp.formatter.string(from:))), "auto_cleanable": item.autoCleanable, "blocker": v2OrNull(item.blocker), ] diff --git a/programaTests/StorageCategoriesTests.swift b/programaTests/StorageCategoriesTests.swift index 85317d14..55b90988 100644 --- a/programaTests/StorageCategoriesTests.swift +++ b/programaTests/StorageCategoriesTests.swift @@ -472,4 +472,62 @@ final class StorageCategoriesTests: XCTestCase { XCTAssertTrue(exists(referenced), "the tool retains packages still in use") XCTAssertFalse(exists(orphan)) } + + func testCleaningSeveralItemsOfOneCategoryScansThatCategoryOnce() throws { + let caches = try ["First", "Second", "Third"].map { + try directory("Library/Developer/Xcode/DerivedData/\($0)-abcd", age: 30 * day) + } + var deviceQueries = 0 + var fixture = context() + fixture.command = { executable, arguments, _, _ in + if executable == "xcrun", arguments.starts(with: ["simctl", "list"]) { deviceQueries += 1 } + return nil + } + let items = StorageCategories.scan(categories: [.xcode], context: fixture) + let queriesPerScan = deviceQueries + XCTAssertGreaterThan(queriesPerScan, 0) + + deviceQueries = 0 + let preview = StorageCategories.clean(items: items, automaticOnly: false, dryRun: true, now: now, context: fixture) + XCTAssertEqual(preview.removed.count, caches.count) + XCTAssertEqual(deviceQueries, queriesPerScan, "a dry run scans the category once, not once per item") + + deviceQueries = 0 + let outcome = StorageCategories.clean(items: items, automaticOnly: false, dryRun: false, now: now, context: fixture) + XCTAssertEqual(Set(outcome.removed.compactMap(\.path)), Set(caches.map(\.path))) + XCTAssertEqual(deviceQueries, queriesPerScan, "removal scans the category once, not once per item") + for cache in caches { XCTAssertFalse(exists(cache)) } + } + + func testProjectBuildGitCheckRunsWithAutomaticGitHardening() throws { + let log = home.appendingPathComponent("git-invocations.log") + let git = try directory("bin").appendingPathComponent("git") + try """ + #!/bin/sh + { printf '%s\\n' "$@"; env | grep '^GIT_CONFIG_' | sort; echo '@@end'; } >> '\(log.path)' + exit 0 + """.write(to: git, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: git.path) + let repo = try directory("repo") + try "plugins {}\n".write(to: repo.appendingPathComponent("build.gradle"), atomically: true, encoding: .utf8) + let build = try directory("repo/build") + var fixture = context(projects: [repo.path]) + fixture.tools = ["git": git.path] + fixture.command = StorageCacheContext.liveCommand(environment: [:]) + + let items = StorageCategories.scan(categories: [.android], context: fixture) + + XCTAssertNotNil(try? item(build, in: items), "the fake git reports no tracked files") + let invocations = try String(contentsOf: log, encoding: .utf8) + .components(separatedBy: "@@end\n").filter { !$0.isEmpty } + .map { $0.split(separator: "\n").map(String.init) } + let listFiles = try XCTUnwrap(invocations.first { $0.contains("ls-files") }) + let options = CanonicalSubprocessRunner.automaticGitOptions + XCTAssertTrue(options.contains("core.fsmonitor=false")) + XCTAssertEqual(Array(listFiles.prefix(options.count + 3)), options + ["ls-files", "-z", "--"]) + XCTAssertEqual(listFiles.count > options.count + 3 ? WorktreeStorage.resolvedPath(listFiles[options.count + 3]) : nil, build.path) + let fsmonitorKey = try XCTUnwrap(listFiles.first { $0.hasPrefix("GIT_CONFIG_KEY_") && $0.hasSuffix("=core.fsmonitor") }) + let index = fsmonitorKey.dropFirst("GIT_CONFIG_KEY_".count).prefix { $0 != "=" } + XCTAssertTrue(listFiles.contains("GIT_CONFIG_VALUE_\(index)=false")) + } } diff --git a/programaTests/StorageHubTests.swift b/programaTests/StorageHubTests.swift index 49538f42..f608d0f5 100644 --- a/programaTests/StorageHubTests.swift +++ b/programaTests/StorageHubTests.swift @@ -194,7 +194,7 @@ final class StorageHubTests: XCTestCase { private func cache(id: String, title: String, bytes: Int64) -> StorageCacheItem { StorageCacheItem( id: id, category: .xcode, title: title, path: "/fixtures/\(id)", sizeBytes: bytes, - sizePartial: false, lastUsed: nil, autoCleanable: false, blocker: nil + sizePartial: false, lastUsed: nil, autoCleanable: false, blocker: nil, removal: .directory ) } }