Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions server/services/arduino-compiler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@ export class ArduinoCompiler {
await writeOutputToCache(this.defaultBinaryStorageDir, sketchHash, cliOutput).catch(() => undefined);
}
await runHexCacheCleanup(hexCacheDir);
await runHexCacheCleanup(this.defaultBinaryStorageDir);
}

return { cliOutput, cliErrors, parsedErrors };
Expand Down
5 changes: 4 additions & 1 deletion server/services/compiler/cache-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ export async function runHexCacheCleanup(
): Promise<void> {
try {
const entries = await readdir(hexCacheDir);
const files: Array<{ path: string; size: number; atimeMs: number }> = [];
const files: Array<{ path: string; sidecarPath: string; size: number; atimeMs: number }> = [];
let totalSize = 0;

for (const entry of entries) {
Expand All @@ -136,6 +136,7 @@ export async function runHexCacheCleanup(
totalSize += fileStat.size;
files.push({
path: fullPath,
sidecarPath: join(hexCacheDir, `${entry.slice(0, -".hex".length)}.output.txt`),
size: fileStat.size,
atimeMs: fileStat.atimeMs || fileStat.mtimeMs,
});
Expand All @@ -150,6 +151,8 @@ export async function runHexCacheCleanup(
for (const file of files) {
if (totalSize <= maxBytes) break;
await rm(file.path, { force: true });
// The output sidecar is useless without its binary and must not outlive it.
await rm(file.sidecarPath, { force: true });
totalSize -= file.size;
}
} catch (error) {
Expand Down
12 changes: 12 additions & 0 deletions server/services/workers/compile-worker-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,18 @@ export async function evictLruEntries(
}
}

/**
* Directories under the build cache that grow per sketch hash and need LRU eviction.
* The instant-binary store (written by ArduinoCompiler) is included so it cannot grow unbounded.
*/
export function buildCacheEvictionTargets(
buildCacheDir: string,
hexCacheDir: string,
coreCacheBuildPath: string,
): string[] {
return [hexCacheDir, join(buildCacheDir, "binaries"), coreCacheBuildPath];
}

/**
* LRU cleanup of build cache directories, debounced via marker file.
*/
Expand Down
3 changes: 2 additions & 1 deletion server/services/workers/compile-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import {
buildSketchHash,
checkBinaryExists,
checkFileExists,
buildCacheEvictionTargets,
cleanupCacheLru,
ensureDirectories,
execArduinoCliJson,
Expand Down Expand Up @@ -170,7 +171,7 @@ async function buildCoreFingerprint(task: CompileRequestPayload, fqbn: string):
}

async function cleanupCacheLruLocal(): Promise<void> {
await cleanupCacheLru(BUILD_CACHE_DIR, [HEX_CACHE_DIR, CORE_CACHE_BUILD_PATH], buildCacheMaxBytes);
await cleanupCacheLru(BUILD_CACHE_DIR, buildCacheEvictionTargets(BUILD_CACHE_DIR, HEX_CACHE_DIR, CORE_CACHE_BUILD_PATH), buildCacheMaxBytes);
}

async function acquireCoreCache(coreReadyMarker: string, coreLockPath: string, coreFingerprint: string): Promise<{ coreCacheWarm: boolean; acquiredCoreLock: boolean; activeBuildCachePath: string }> {
Expand Down
14 changes: 14 additions & 0 deletions tests/server/services/compiler/cache-manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,20 @@ describe("cache-manager cache cleanup", () => {
await expect(readFile(paths[2])).resolves.toEqual(Buffer.alloc(3, 3));
});

it("removes the output sidecar together with an evicted HEX entry", async () => {
const hexCacheDir = await createHexCacheDirectory();
const oldHex = join(hexCacheDir, "old.hex");
await writeFile(oldHex, Buffer.alloc(4, 1));
await writeFile(join(hexCacheDir, "old.output.txt"), "old output", "utf8");
await writeFile(join(hexCacheDir, "new.hex"), Buffer.alloc(4, 2));
await writeFile(join(hexCacheDir, "new.output.txt"), "new output", "utf8");
await utimes(oldHex, new Date("2020-01-01T00:00:00Z"), new Date("2020-01-01T00:00:00Z"));

await runHexCacheCleanup(hexCacheDir, 4);

await expect(readdir(hexCacheDir)).resolves.toEqual(["new.hex", "new.output.txt"]);
});

it("does not remove HEX entries or sidecars when the cache is within the limit", async () => {
const hexCacheDir = await createHexCacheDirectory();
await writeFile(join(hexCacheDir, "sketch.hex"), Buffer.alloc(4, 1));
Expand Down
11 changes: 11 additions & 0 deletions tests/server/workers/compile-worker-utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
acquireCoreCacheLock,
collectDirectoryRecords,
evictLruEntries,
buildCacheEvictionTargets,
cleanupCacheLru,
ensureDirectories,
execArduinoCliJson,
Expand Down Expand Up @@ -390,3 +391,13 @@ describe("execArduinoCliJson", () => {
expect(result).toBeNull();
});
});

describe("buildCacheEvictionTargets", () => {
it("covers the instant-binary store written by ArduinoCompiler", () => {
expect(buildCacheEvictionTargets("/cache", "/cache/hex-cache", "/core/build-cache")).toEqual([
"/cache/hex-cache",
"/cache/binaries",
"/core/build-cache",
]);
});
});
Loading