diff --git a/CLAUDE.md b/CLAUDE.md index eff0e13..c0c5fe6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -315,14 +315,44 @@ build machinery for compatibility that nobody needs. leaves (`DatabaseVault.RoomOf`). - **Lambdas saved before keep working, untouched on disk.** A version or a feature is read in the layout it was saved in and translated as it is read - (`LambdaSource.Parse`: an envelope `"version": 1` is the old layout, 2 is - today's) - assets at the top move below `resources/`, `.lambda/docs/`, + (`LambdaSource.Parse`: an envelope `"version": 1` is the first layout, 2 + the second - today's names, every resource inside it - and 3 today's) - + assets at the top move below `resources/`, `.lambda/docs/`, `.lambda/tests/` and `.lambda/build/` to `docs/`, `tests/` and `build/`; C# that was in `.lambda/` was never compiled, so it keeps a `.txt` ending (`tests/Smoke.cs.txt`) rather than be compiled into a lambda online. Never rewrite stored versions for it. Their git history keeps its commits; the move is a commit of its own on top (see Git). A top-level `assets/` or `.lambda/` is refused with a hint where it goes now. +- **Resources are kept once, by what they hold - decided.** A version or a + feature names each resource by the SHA-256 of what it holds (`stored` and + `length` in the envelope) and the lambda's `IResourceStore` + (`{data}/code/{id}/resources/`) keeps it, once for all its versions, + features and previews. Carried inside, every save and every deployment of a + site took several copies of all of it as strings and base64 in one process + for every lambda: a fifteen megabyte site spiked the server by hundreds. So: + - **Never carry a resource as text when it can be bytes.** What an archive or + a push holds becomes a file with `LambdaFile.FromBytes`, and goes on as + files to the overloads that take them (`IMetaService.Save`, + `IFeatureService.Save`, `CheckAsync`) - never `LambdaSource.Serialize`d + into one text on its way there. + - **The two writers keep them** (`LambdaHistory.Append`, + `FeatureService.Save`, both with `LambdaSource.Keep`) before anything that + names them is written, so a version only ever names what is kept. + - **What needs what a resource holds reads it through the store**: + `LambdaVersionContent.Files()`, `FeatureContent.Files()`, + `IStorageService.ReadFiles`, or `LambdaSource.Parse(code, resources)`. + Parsed without one, a stored resource can be named, counted and compared + and throws when read, rather than reading as nothing. Going online copies + a resource out of the store file to file (`Materialize`), and an archive + streams it. + - Versions of the second layout are read as they are, resources inside, and + never rewritten. + - What nothing names any more goes with the version or the feature that + named it last (`DeleteVersion`, `DeleteFeature`), and only once it is an + hour old: the resources of a save are kept before what names them is + written, under a turn a pruning does not hold. An envelope that cannot be + read stops the collection rather than naming nothing. - Names: a C# file's own name as C# names (dots allowed), its folders as the rest of the code; a resource as files that are served always were (an extension, no dot names, six folders deep); the rest of the @@ -756,6 +786,10 @@ rules that matter: interface (`IMetaService.GetFacts`, `ISourceService.Star`), not taken from the unit. An exception is an entry in the test with its reason; today only the composition root (`Application`) has one. +- **The collector conserves memory - decided.** Server GC, with + `System.GC.ConserveMemory` set in the project file: every lambda shares the + process, and what one does now and then (a large version saved, an archive + packed) left the high water mark of large objects held for all of them. - Compiled lambdas run in the server process. The `CodeGuard` raises the cost of misbehaving; **it is not a sandbox.** Do not describe it as one. - Database: SQLite, EF Core, migrated by Evolve from `Data/Migrations/V__*.sql`. @@ -819,7 +853,9 @@ method is called - so the code is written the way it runs: binding with Roslyn, packing, copying a workspace or a database. Not in many small asynchronous steps (`File.ReadAllTextAsync` hops out and back per buffer), and not in place. A version's file is read and written in place: - it is bounded by its tier's allowance, and saving one is rare. `Task.Run` is + it names its resources rather than carrying them, is bounded by its tier's + allowance, and saving one is rare; one that still carries them is read from + its file as it is parsed, not into a string first. `Task.Run` is refused, so every hop is marked in `Offload` and can be given a scheduler of its own there; work started in the background rather than awaited (the scheduler's jobs, seeding the demos) is not a hop and suppresses RS0030 diff --git a/README.md b/README.md index 850b811..248a735 100644 --- a/README.md +++ b/README.md @@ -383,13 +383,28 @@ read in the layout it was saved in, so the old ones are translated as they are read - `site/app.js` becomes `resources/site/app.js`, `.lambda/docs/product.md` becomes `docs/product.md`, `.lambda/build/package.json` becomes `build/package.json` - and saved in the new one when they are next saved (an -envelope with `"version": 2`, `LambdaSource.Parse`). C# that was in `.lambda/` +envelope with `"version": 3` today, `LambdaSource.Parse`). C# that was in `.lambda/` was never compiled and would be now, so it keeps what it holds under a name that is not: `.lambda/tests/Smoke.cs` becomes `tests/Smoke.cs.txt`. A feature's files are read the same way. In git, a lambda's history keeps the commits its clones already have; the move is a commit of its own on top of each (see [Working with git](#working-with-git)). +**Resources are kept once, apart from the versions.** A version names its +resources by the SHA-256 of what they hold rather than carrying them, and the +lambda keeps each of them once, below `{data}/code/{id}/resources/`, for every +version, feature and preview that ships it unchanged. Going online copies them +out of there, file to file, and an archive is packed by streaming them, so +nothing reads a site's worth of pages into the server's memory to save it or +put it online - which, when versions carried their resources as text and +base64 in one string, took a few hundred megabytes of a process every lambda +shares for a site of fifteen. A version saved before carries its resources +inside, as an envelope of `"version": 2` did, and is read as it is - nothing +on disk is rewritten - until it is saved again. A resource that no version, +feature or preview names any more is removed with the version or the feature +that named it last, once it is an hour old: the resources of a save are kept +before what names them is written. + ### What it is built from Some of a lambda may be made by a build tool rather than written as it is @@ -1146,7 +1161,9 @@ its own (`IDatabaseVault`, `ISecretVault`, `IDomainRegistry`, `ILogBook`, maps the DTOs. `LambdaGuard` (who may change a lambda - not a demo) and `VersionInput` (what a version may hold) are rules other services apply too. - **Storage** (`Services/Storage`) - the code itself, on the file system, one - file per version. Never in the database. + file per version, and the resources of a lambda's versions and features in a + store of its own, each once, by what it holds (`ResourceStore`). Never in the + database. - **Data** (`Services/Data`) - which kinds of data a lambda keeps, switched on and off by its owner. Only the choice is stored: a lambda without a row for a kind has its default, so a kind added to `DataKinds` needs no migration to @@ -1439,7 +1456,11 @@ Both are for the operator only, behind the administration token (see in the editor, and nobody sees everybody's. What it is for is the shape of the memory curve. A managed heap that climbs across gen 2 collections is a leak; committed bytes climbing while the managed heap stays -flat is the heap keeping pages it could return, which is not. +flat is the heap keeping pages it could return, which is not. The collector is +asked to conserve memory (`System.GC.ConserveMemory` in the project file), so +after a spike - a large version saved, an archive packed - it compacts the +large object heap and gives back what it no longer needs, rather than keeping +the high water mark of the busiest lambda for all of them. ## TLS @@ -1502,18 +1523,19 @@ version may come to, its code and its resources together, is a [limit](#limits) of the lambda's tier. How many files there are is not limited, and neither is the number of C# files. -Budget the disk for the premium tier. A version is kept whole, its files as -text or base64 in JSON, so each version saved at the limit takes up to half as -much again on disk - 184 MB for 128 MB of pictures - and fifty of them are -kept by default: at the defaults, a premium lambda that is saved over and over -at its limit can come to around 9 GB of history. - -Budget the memory too. A version is read and written whole: with the 128 MB -the premium tier allows, saving one took the server to 2.3 GB and deploying it -to 2.8 GB. Even the 32 MB of a free lambda took a server idling at 350 MB to -950 MB to save, and anybody can create a free lambda. Set what a version may -come to, for both tiers, in the panel's **Limits** to what the machine can -carry until resources are stored apart from the versions. +Budget the disk for the premium tier. A version keeps its code whole, as text +in JSON, and names its resources, which the lambda's store keeps once each, as +the bytes they are: a version costs what it changed, and fifty of them are +kept by default. A version saved before resources were kept apart carries +them inside, as text or base64, up to half as much again as they are. + +Budget the memory too. Saving a version reads the archive or the request it +came in, and its resources go into the store file by file; going online copies +them out. A site of 15 MB in 1,768 files took the server 237 MB higher to save +and deploy while versions carried their resources, and 46 MB since; its first +request after a restart, 111 MB and 37. Set what a version may come to, for +both tiers, in the panel's **Limits** to what the machine can carry - the +request a version arrives in is still read whole. ### More than one hostname diff --git a/src/GenHTTP.Lambda/Api/CodeResource.cs b/src/GenHTTP.Lambda/Api/CodeResource.cs index 481d0e2..53d1292 100644 --- a/src/GenHTTP.Lambda/Api/CodeResource.cs +++ b/src/GenHTTP.Lambda/Api/CodeResource.cs @@ -28,7 +28,7 @@ public sealed class CodeResource(IMetaService meta) [ResourceMethod(Method.Post, "lambdas/:privateKey/code/check")] public async ValueTask Check(string privateKey, CodeRequest request) { - var outcome = await meta.CheckAsync(privateKey, VersionResource.Serialize(request.Files)); + var outcome = await meta.CheckAsync(privateKey, VersionResource.Checked(request.Files)); return new CompilationResponse(outcome.Success, outcome.Diagnostics); } diff --git a/src/GenHTTP.Lambda/Api/FeatureResource.cs b/src/GenHTTP.Lambda/Api/FeatureResource.cs index 4119dc8..ebce7b0 100644 --- a/src/GenHTTP.Lambda/Api/FeatureResource.cs +++ b/src/GenHTTP.Lambda/Api/FeatureResource.cs @@ -78,7 +78,7 @@ public FeatureContentResponse Get(string privateKey, string feature, string? fol { var found = features.Get(privateKey, feature); - return new FeatureContentResponse(Describe(found.Feature), VersionResource.Below(LambdaSource.Parse(found.Code), folder)); + return new FeatureContentResponse(Describe(found.Feature), VersionResource.Below(found.Files(), folder)); } /// @@ -132,7 +132,7 @@ public async ValueTask Change(string privateKey, string fe { var read = features.Get(privateKey, feature); - var files = LambdaChanges.Apply(LambdaSource.Parse(read.Code), request.Files, request.Remove, request.Edits); + var files = LambdaChanges.Apply(read.Files(), request.Files, request.Remove, request.Edits); // made on top of what was read, and refused if something else was saved meanwhile return await SaveAsync(privateKey, feature, files, deploy, request.Specification, request.Change, read.Feature.Revision); @@ -150,7 +150,7 @@ public IResponse GetArchive(string privateKey, string feature, string? layout, I var found = features.Get(privateKey, feature); - var zip = LambdaArchive.Pack(LambdaSource.Parse(found.Code), laid); + var zip = LambdaArchive.Pack(found.Files(), laid); logger.LogInformation("Downloaded feature '{Feature}' of lambda {Lambda} layout {Layout}", found.Feature.Name, meta.PublicKeyOf(privateKey), laid); @@ -181,7 +181,7 @@ public async ValueTask PutArchive(string privateKey, strin var tier = Enum.Parse(lambda.Tier); var files = await VersionResource.UnpackAsync(body, limits.BuildOf(tier), laid, - LambdaSource.Parse(features.Get(privateKey, feature).Code)); + features.Get(privateKey, feature).Files()); return await SaveAsync(privateKey, feature, files, deploy, specification, change); } @@ -328,7 +328,7 @@ public async ValueTask> Merge(string privateKey, st private async ValueTask SaveAsync(string privateKey, string feature, IReadOnlyList? files, bool? deploy, string? specification, string? change, int? after = null) { - var saved = features.Save(privateKey, feature, VersionResource.Serialize(files), new VersionNote(specification, change), after); + var saved = features.Save(privateKey, feature, VersionResource.Checked(files), new VersionNote(specification, change), after); var publicKey = meta.PublicKeyOf(privateKey); diff --git a/src/GenHTTP.Lambda/Api/LambdaResource.cs b/src/GenHTTP.Lambda/Api/LambdaResource.cs index 34925a2..c7b539b 100644 --- a/src/GenHTTP.Lambda/Api/LambdaResource.cs +++ b/src/GenHTTP.Lambda/Api/LambdaResource.cs @@ -171,7 +171,7 @@ public async ValueTask Export(string privateKey, IRequest request) { using var stream = File.Create(target); - ProjectPacker.Pack(exported, LambdaSource.Parse(content.Code), stream, database); + ProjectPacker.Pack(exported, content.Files(), stream, database); return target; } diff --git a/src/GenHTTP.Lambda/Api/Mcp/McpTools.cs b/src/GenHTTP.Lambda/Api/Mcp/McpTools.cs index 4c1c3b4..30efa25 100644 --- a/src/GenHTTP.Lambda/Api/Mcp/McpTools.cs +++ b/src/GenHTTP.Lambda/Api/Mcp/McpTools.cs @@ -614,7 +614,7 @@ private async ValueTask WriteAsync(JsonObject arguments, string orig // what it held before the save replaces it var had = Text(arguments, "feature") is { } feature - ? LambdaSource.Parse(features.Get(privateKey, feature).Code) + ? features.Get(privateKey, feature).Files() : meta.Require(privateKey).LatestVersion != null ? Api.VersionResource.Latest(meta, privateKey) : []; var saved = McpProtocol.Adding(await SaveAsync(arguments, files, origin), "dropped", Dropped(had, files)); @@ -676,7 +676,7 @@ private async ValueTask ChangeAsync(JsonObject arguments, string ori { var content = features.Get(privateKey, feature); - current = LambdaSource.Parse(content.Code); + current = content.Files(); read = content.Feature.Revision; } @@ -720,18 +720,16 @@ private async ValueTask SaveAsync(JsonObject arguments, IReadOnlyLis var note = new VersionNote(Text(arguments, "specification"), Text(arguments, "change"), VersionOrigins.Agent); - var code = LambdaSource.Serialize(files); - // said where the next save is decided, and as a request rather than a // refusal - like the note below: the code matters more var documentation = Unwritten(files); if (Text(arguments, "feature") is { } feature) { - return await SaveFeatureAsync(arguments, privateKey, feature, code, files.Count, note, origin, read, documentation); + return await SaveFeatureAsync(arguments, privateKey, feature, files, note, origin, read, documentation); } - var version = meta.Save(privateKey, code, note); + var version = meta.Save(privateKey, files, note); logger.LogInformation("Saved lambda {Lambda} version {Version} files {Files}", meta.PublicKeyOf(privateKey), version.Version, files.Count); @@ -761,7 +759,7 @@ private async ValueTask SaveAsync(JsonObject arguments, IReadOnlyLis * would have to read the whole lambda back to find out whether the * line compiles. The version is saved either way. */ - var outcome = await meta.CheckAsync(privateKey, code); + var outcome = await meta.CheckAsync(privateKey, files); return McpProtocol.Say(new { @@ -782,12 +780,12 @@ private async ValueTask SaveAsync(JsonObject arguments, IReadOnlyLis /// Stores the files as the feature's, and puts its preview online or /// compiles it if the arguments ask for that. /// - private async ValueTask SaveFeatureAsync(JsonObject arguments, string privateKey, string feature, string code, int files, VersionNote note, + private async ValueTask SaveFeatureAsync(JsonObject arguments, string privateKey, string feature, IReadOnlyList files, VersionNote note, string origin, int? read, string? documentation) { - var saved = features.Save(privateKey, feature, code, note, read); + var saved = features.Save(privateKey, feature, files, note, read); - logger.LogInformation("Saved feature '{Feature}' of lambda {Lambda} files {Files}", saved.Name, meta.PublicKeyOf(privateKey), files); + logger.LogInformation("Saved feature '{Feature}' of lambda {Lambda} files {Files}", saved.Name, meta.PublicKeyOf(privateKey), files.Count); if (Flag(arguments, "deploy") == true) { @@ -796,7 +794,7 @@ private async ValueTask SaveFeatureAsync(JsonObject arguments, strin if (Flag(arguments, "check") == true) { - var outcome = await meta.CheckAsync(privateKey, code); + var outcome = await meta.CheckAsync(privateKey, files); return McpProtocol.Say(new { @@ -831,7 +829,7 @@ private async ValueTask CheckAsync(JsonObject arguments) var privateKey = Required(arguments, "privateKey"); - var outcome = await meta.CheckAsync(privateKey, LambdaSource.Serialize(files)); + var outcome = await meta.CheckAsync(privateKey, files); logger.LogInformation("Checked code of lambda {Lambda} files {Files} compiles {Compiles}", meta.PublicKeyOf(privateKey), files.Count, outcome.Success); @@ -1446,7 +1444,7 @@ private async ValueTask DeployFeatureAsync(string privateKey, string // links that leave the preview for the live lambda, said while there is still time var lambda = meta.Get(privateKey); - var files = LambdaSource.Parse((features.Get(privateKey, feature)).Code); + var files = features.Get(privateKey, feature).Files(); var warning = lambda != null ? Leaks(files, lambda.PublicKey) : null; @@ -1594,7 +1592,7 @@ private JsonObject Read(JsonObject arguments, string origin) { feature = features.Get(privateKey, wanted); - files = LambdaSource.Parse(feature.Code); + files = feature.Files(); } else { @@ -1604,7 +1602,7 @@ private JsonObject Read(JsonObject arguments, string origin) { content = meta.GetVersion(privateKey, which); - files = LambdaSource.Parse(content.Code); + files = content.Files(); } } @@ -2320,7 +2318,7 @@ private JsonObject Guide(string origin) => McpProtocol.Say(new serving = "return Layout.Create().Add(\"api\", api).Add(Resources.App(\"web\"));", files = "Resources.Files(\"media\") serves a folder as plain files, straight from the directory - the fastest way the server has to send a file. Use it for what is not the app's pages: pictures, downloads, media.", contentTypes = "Inferred from the file extension.", - size = "Every version keeps its own copy of its resources and is read whole to be saved and deployed, so a large file that is data rather than program - a model, a dataset, video, a library of pictures - belongs in the workspace, where it is kept once. They count towards what a version may come to, with its code - see limits.", + size = "Resources count towards what a version may come to, with its code - see limits - however many versions ship them unchanged, so a large file that is data rather than program - a model, a dataset, video, a library of pictures - belongs in the workspace, where it does not.", names = "Below resources/: letters, digits, dashes, underscores, dots and slashes, and an extension to serve it by. No name starting with a dot, no .." }, workspace = new diff --git a/src/GenHTTP.Lambda/Api/VersionResource.cs b/src/GenHTTP.Lambda/Api/VersionResource.cs index d65d76c..b5fda32 100644 --- a/src/GenHTTP.Lambda/Api/VersionResource.cs +++ b/src/GenHTTP.Lambda/Api/VersionResource.cs @@ -74,7 +74,7 @@ public IResponse GetArchive(string privateKey, int version, string? layout, IReq var content = meta.GetVersion(privateKey, version); - var zip = LambdaArchive.Pack(LambdaSource.Parse(content.Code), laid); + var zip = LambdaArchive.Pack(content.Files(), laid); logger.LogInformation("Downloaded lambda {Lambda} version {Version} layout {Layout}", lambda.PublicKey, version, laid); @@ -157,7 +157,7 @@ private async ValueTask> SaveAsync(string privateKe { var note = new VersionNote(specification, change, VersionOrigins.Api); - var version = meta.Save(privateKey, Serialize(files), note); + var version = meta.Save(privateKey, Checked(files), note); var publicKey = meta.PublicKeyOf(privateKey); @@ -208,14 +208,14 @@ internal static IReadOnlyList Latest(IMetaService meta, string priva return []; } - return LambdaSource.Parse((meta.GetVersion(privateKey, latest)).Code); + return meta.GetVersion(privateKey, latest).Files(); } internal static VersionResponse Describe(LambdaVersionInfo version) => new(version.Version, version.Created, version.Specification, version.Change, version.Origin); internal static VersionContentResponse Describe(LambdaVersionContent content, string? folder = null) - => new(content.Version, content.Created, content.Specification, content.Change, content.Origin, Below(LambdaSource.Parse(content.Code), folder)); + => new(content.Version, content.Created, content.Specification, content.Change, content.Origin, Below(content.Files(), folder)); /// /// The files below a folder, or all of them when none is named. @@ -224,16 +224,21 @@ internal static IReadOnlyList Below(IReadOnlyList files, => string.IsNullOrEmpty(folder) ? files : [.. files.Where(f => f.Name.StartsWith(folder, StringComparison.Ordinal))]; /// - /// Turns what was submitted into the single blob a version is stored as. + /// What was submitted, checked before it goes anywhere. /// - internal static string Serialize(IReadOnlyList? files) + /// + /// Handed on as the files they are rather than made one text first: + /// the resources of an archive go into the lambda's store as the bytes + /// they were read as. + /// + internal static IReadOnlyList Checked(IReadOnlyList? files) { if (LambdaSource.Validate(files) is { } complaint) { throw LambdaException.Invalid(complaint); } - return LambdaSource.Serialize(files!); + return files!; } } diff --git a/src/GenHTTP.Lambda/Configuration/LambdaOptions.cs b/src/GenHTTP.Lambda/Configuration/LambdaOptions.cs index ad47049..03918c2 100644 --- a/src/GenHTTP.Lambda/Configuration/LambdaOptions.cs +++ b/src/GenHTTP.Lambda/Configuration/LambdaOptions.cs @@ -163,10 +163,10 @@ public sealed record LambdaOptions /// code and its resources together. /// /// - /// Mind what it costs: every version keeps its own copy of all of it, as - /// base64 in JSON for what is not text, so a version saved at the limit - /// takes nearly half as much again on disk, of - /// them are kept, and a version is read and written whole, in memory. + /// Mind what it costs: a version keeps its code whole and names its + /// resources, which the lambda's store keeps once each, so a version costs + /// the disk what it changed - of them are kept - + /// and the request it arrives in is read whole, in memory. /// public long BuildBytes { get; init; } = 32L * 1024 * 1024; diff --git a/src/GenHTTP.Lambda/GenHTTP.Lambda.csproj b/src/GenHTTP.Lambda/GenHTTP.Lambda.csproj index 11d4753..c403ded 100644 --- a/src/GenHTTP.Lambda/GenHTTP.Lambda.csproj +++ b/src/GenHTTP.Lambda/GenHTTP.Lambda.csproj @@ -26,6 +26,19 @@ $(Features);runtime-async=on + + + + + diff --git a/src/GenHTTP.Lambda/Services/Deployment/DeploymentService.cs b/src/GenHTTP.Lambda/Services/Deployment/DeploymentService.cs index 0965754..b0ed562 100644 --- a/src/GenHTTP.Lambda/Services/Deployment/DeploymentService.cs +++ b/src/GenHTTP.Lambda/Services/Deployment/DeploymentService.cs @@ -76,11 +76,14 @@ public DeploymentService(IStorageService storage, ServerRegistry servers, ISecre #region Functionality - public async ValueTask ValidateAsync(string code, long? lambdaId = null, WorkspaceLimits? limits = null, CancellationToken cancellation = default) + public ValueTask ValidateAsync(string code, long? lambdaId = null, WorkspaceLimits? limits = null, CancellationToken cancellation = default) + => ValidateAsync(LambdaSource.Parse(code, Storage.Resources(lambdaId ?? 0)), lambdaId, limits, cancellation); + + public async ValueTask ValidateAsync(IReadOnlyList files, long? lambdaId = null, WorkspaceLimits? limits = null, CancellationToken cancellation = default) { var id = lambdaId ?? 0; - var request = new CompilationRequest(LambdaSource.Parse(code), Storage.GetWorkspace(id), Storage.GetResourceDirectory(id), + var request = new CompilationRequest(files, Storage.GetWorkspace(id), Storage.GetResourceDirectory(id), Storage.GetAssemblyDirectory(id), $"check_{id}", false, limits ?? WorkspaceLimits.Standard); await _compiling.WaitAsync(cancellation); @@ -105,25 +108,25 @@ public async ValueTask ValidateAsync(string code, long? lamb } public ValueTask ActivateAsync(long lambdaId, int version, WorkspaceLimits limits, CancellationToken cancellation = default) - => ActivateAsync(new Slot(lambdaId, null), version, limits, () => Storage.Read(lambdaId, version), cancellation); + => ActivateAsync(new Slot(lambdaId, null), version, limits, () => Storage.ReadFiles(lambdaId, version), cancellation); public ValueTask ResolveAsync(long lambdaId, int version, WorkspaceLimits limits, CancellationToken cancellation = default) - => ResolveAsync(new Slot(lambdaId, null), version, limits, () => Storage.Read(lambdaId, version), cancellation); + => ResolveAsync(new Slot(lambdaId, null), version, limits, () => Storage.ReadFiles(lambdaId, version), cancellation); public ValueTask PreviewAsync(long lambdaId, long featureId, int preview, string code, WorkspaceLimits limits, CancellationToken cancellation = default) - => ActivateAsync(new Slot(lambdaId, featureId), preview, limits, () => code, cancellation); + => ActivateAsync(new Slot(lambdaId, featureId), preview, limits, () => LambdaSource.Parse(code, Storage.Resources(lambdaId)), cancellation); public ValueTask ResolvePreviewAsync(long lambdaId, long featureId, int preview, WorkspaceLimits limits, CancellationToken cancellation = default) - => ResolveAsync(new Slot(lambdaId, featureId), preview, limits, () => Storage.ReadPreview(lambdaId, featureId), cancellation); + => ResolveAsync(new Slot(lambdaId, featureId), preview, limits, () => Storage.ReadPreviewFiles(lambdaId, featureId), cancellation); /// /// Builds what a slot is to serve and makes it what it serves. /// /// Which build this is: the version, or the deployment of a preview /// Where the code comes from - private async ValueTask ActivateAsync(Slot slot, int stamp, WorkspaceLimits limits, Func read, + private async ValueTask ActivateAsync(Slot slot, int stamp, WorkspaceLimits limits, Func?> read, CancellationToken cancellation) { if (IsDeployed(slot, stamp, limits)) @@ -168,20 +171,19 @@ private async ValueTask ActivateAsync(Slot slot, int stamp, /// Reads, writes out and compiles what a slot is to serve - under the lock /// and away from the reactor, see the activation above. /// - private async ValueTask BuildAsync(Slot slot, int stamp, WorkspaceLimits limits, Func read, + private async ValueTask BuildAsync(Slot slot, int stamp, WorkspaceLimits limits, Func?> read, CancellationToken cancellation) { - var code = read(); + // straight from where it is stored, the resources named rather than read + var files = read(); - if (code == null) + if (files == null) { return CompilationOutcome.Failed(slot.FeatureId == null ? $"Version {stamp} of this lambda does not exist anymore." : "The preview of this feature has nothing to serve anymore."); } - var files = LambdaSource.Parse(code); - // what this version ships is written out before it is compiled, so the // handler it returns is serving the resources of the version going // online rather than whatever the last one left behind @@ -268,13 +270,13 @@ private void Restore(Slot slot) try { - var code = slot.FeatureId is { } feature - ? Storage.ReadPreview(slot.LambdaId, feature) - : Storage.Read(slot.LambdaId, running.Stamp); + var files = slot.FeatureId is { } feature + ? Storage.ReadPreviewFiles(slot.LambdaId, feature) + : Storage.ReadFiles(slot.LambdaId, running.Stamp); - if (code != null) + if (files != null) { - Materialize(slot, LambdaSource.Parse(code)); + Materialize(slot, files); } } catch (Exception e) @@ -283,7 +285,7 @@ private void Restore(Slot slot) } } - private async ValueTask ResolveAsync(Slot slot, int stamp, WorkspaceLimits limits, Func read, + private async ValueTask ResolveAsync(Slot slot, int stamp, WorkspaceLimits limits, Func?> read, CancellationToken cancellation) { if (_deployed.TryGetValue(slot, out var existing) && existing.Stamp == stamp && existing.Limits == limits) @@ -329,6 +331,11 @@ private bool IsDeployed(Slot slot, int stamp, WorkspaceLimits limits) /// rather than lingering because nothing overwrote it. Names were checked /// before they got here, and the path is resolved against the root again /// anyway - a file that would land outside is skipped rather than trusted. + /// + /// A resource the lambda's store keeps is copied out of it, file to file, + /// without being read into the server - which a site of a few thousand + /// pages, written out on every deployment and on the first request after + /// a restart, would otherwise be, all of it, every time. /// private void Materialize(Slot slot, IReadOnlyList files) { @@ -364,7 +371,14 @@ private void Materialize(Slot slot, IReadOnlyList files) Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllBytes(path, file.Bytes); + if (file.Stored is { } stored) + { + stored.CopyTo(path); + } + else + { + File.WriteAllBytes(path, file.Bytes); + } } } catch (Exception e) diff --git a/src/GenHTTP.Lambda/Services/Deployment/IDeploymentService.cs b/src/GenHTTP.Lambda/Services/Deployment/IDeploymentService.cs index 55aa925..b027b78 100644 --- a/src/GenHTTP.Lambda/Services/Deployment/IDeploymentService.cs +++ b/src/GenHTTP.Lambda/Services/Deployment/IDeploymentService.cs @@ -28,6 +28,11 @@ public interface IDeploymentService /// What the lambda may keep in its workspace, the standard limits if not given ValueTask ValidateAsync(string code, long? lambdaId = null, WorkspaceLimits? limits = null, CancellationToken cancellation = default); + /// + /// Checks whether the files compile, without deploying them. + /// + ValueTask ValidateAsync(IReadOnlyList files, long? lambdaId = null, WorkspaceLimits? limits = null, CancellationToken cancellation = default); + /// /// Compiles and prepares the given version so it is ready to serve requests. /// diff --git a/src/GenHTTP.Lambda/Services/Deployment/Model/LambdaArchive.cs b/src/GenHTTP.Lambda/Services/Deployment/Model/LambdaArchive.cs index e0da3c1..3a7b9a7 100644 --- a/src/GenHTTP.Lambda/Services/Deployment/Model/LambdaArchive.cs +++ b/src/GenHTTP.Lambda/Services/Deployment/Model/LambdaArchive.cs @@ -51,6 +51,11 @@ public static class LambdaArchive /// /// Packs the given files into a zip archive. /// + /// + /// Every file is streamed into its entry from wherever it is - a resource + /// straight out of the lambda's store - rather than copied into an array + /// of its own first. + /// public static byte[] Pack(IReadOnlyList files, ArchiveLayout layout = ArchiveLayout.Lambda) { using var buffer = new MemoryStream(); @@ -59,16 +64,25 @@ public static byte[] Pack(IReadOnlyList files, ArchiveLayout layout { foreach (var file in files) { - var (name, content) = layout == ArchiveLayout.Lambda ? (file.Name, file.Bytes) : InProject(file); + var snippet = layout == ArchiveLayout.Project && file.Name == LambdaSource.EntryName; - var entry = zip.CreateEntry(name, CompressionLevel.Optimal); + var entry = zip.CreateEntry(layout == ArchiveLayout.Lambda ? file.Name : InProject(file), CompressionLevel.Optimal); // fixed, so the same version always packs to the same bytes entry.LastWriteTime = Timestamp; using var target = entry.Open(); - target.Write(content); + if (snippet) + { + target.Write(Encoding.UTF8.GetBytes(ProjectSnippet.ForRepository(file.Code))); + } + else + { + using var source = file.Open(); + + source.CopyTo(target); + } } } @@ -76,13 +90,11 @@ public static byte[] Pack(IReadOnlyList files, ArchiveLayout layout } /// - /// A file of the lambda where a clone has it: the snippet in the class - /// it is the body of, the rest where puts it. + /// Where a clone has a file of the lambda: the snippet in the class it is + /// the body of, the rest where puts it. /// - private static (string Path, byte[] Content) InProject(LambdaFile file) - => file.Name == LambdaSource.EntryName - ? (ProjectPaths.Snippet, Encoding.UTF8.GetBytes(ProjectSnippet.ForRepository(file.Code))) - : (ProjectPaths.Of(file.Name), file.Bytes); + private static string InProject(LambdaFile file) + => file.Name == LambdaSource.EntryName ? ProjectPaths.Snippet : ProjectPaths.Of(file.Name); #endregion @@ -367,6 +379,13 @@ private static async ValueTask BufferAsync(Stream content, long ma private static LambdaFile ToFile(string name, byte[] bytes) { + // a resource stays the bytes it was read as until the lambda's store + // has them, rather than becoming a string of twice their size + if (LambdaSource.IsResource(name)) + { + return LambdaFile.FromBytes(name, bytes); + } + if (TryText(bytes, out var text)) { return new LambdaFile(name, text); diff --git a/src/GenHTTP.Lambda/Services/Deployment/Model/LambdaSource.cs b/src/GenHTTP.Lambda/Services/Deployment/Model/LambdaSource.cs index d6d7699..6f7547b 100644 --- a/src/GenHTTP.Lambda/Services/Deployment/Model/LambdaSource.cs +++ b/src/GenHTTP.Lambda/Services/Deployment/Model/LambdaSource.cs @@ -1,6 +1,10 @@ using System.Buffers.Text; +using System.Security.Cryptography; +using System.Text; +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Unicode; namespace GenHTTP.Lambda.Services.Deployment.Model; @@ -9,11 +13,108 @@ namespace GenHTTP.Lambda.Services.Deployment.Model; /// kept with it - or one of its resources, which it reads and serves while it /// runs. /// -/// What it is called, which is also what diagnostics name -/// Its contents, base64 when says so -/// "base64" for a file that is not text, absent otherwise -public sealed record LambdaFile(string Name, [property: JsonConverter(typeof(LongStringConverter))] string Code, string? Encoding = null) +/// +/// What a file holds is in one of three places, and is only turned into the +/// others when somebody asks for it: the text or the base64 it was sent as +/// (), the bytes it was read as from an archive or a push, +/// or - for a resource of a saved version - the lambda's resource store, +/// which keeps every resource once, by what it holds (). +/// +/// A version of a website ships its pages as resources, thousands of them. +/// Carried as strings, every save and every deployment of one took several +/// copies of them at twice their size, as text and as base64, in one string +/// the size of all of them together - a few hundred megabytes, all at once, +/// for a site of fifteen, in a process every lambda shares. So the bytes stay +/// bytes until the store has them, and a version names them rather than +/// carries them. +/// +public sealed record LambdaFile { + private static readonly UTF8Encoding Text = new(false, false); + + private string? _code; + + private byte[]? _bytes; + + /// What it is called, which is also what diagnostics name + /// Its contents, base64 when says so + /// "base64" for a file that is not text, absent otherwise + [JsonConstructor] + public LambdaFile(string name, string code, string? encoding = null) + { + Name = name; + Encoding = encoding; + _code = code; + } + + private LambdaFile(string name, string? encoding, byte[]? bytes, StoredContent? stored) + { + Name = name; + Encoding = encoding; + _bytes = bytes; + Stored = stored; + } + + /// + /// A resource as it was read, kept as the bytes it is: text where it is + /// UTF-8 without a NUL, base64 otherwise - once somebody asks for it as either. + /// + /// + /// A byte order mark is not part of the text, as it never was when a + /// resource was read into a string. + /// + /// True for a file that stays what it was before - not text - whatever its bytes look like now + public static LambdaFile FromBytes(string name, byte[] bytes, bool binary = false) + { + if (!binary && Utf8.IsValid(bytes) && !bytes.AsSpan().Contains((byte)0)) + { + return new LambdaFile(name, null, bytes.AsSpan().StartsWith(Preamble) ? bytes[Preamble.Length..] : bytes, null); + } + + return new LambdaFile(name, "base64", bytes, null); + } + + /// + /// A resource as the lambda's resource store keeps it. + /// + /// "base64" where it is not text, as it was when it was kept + public static LambdaFile FromStore(string name, StoredContent stored, string? encoding) + => new(name, encoding, null, stored); + + private static ReadOnlySpan Preamble => [0xEF, 0xBB, 0xBF]; + + /// What it is called, which is also what diagnostics name. + public string Name { get; init; } + + /// + /// What it holds, base64 when says so - made from + /// its bytes, or read from the store, the first time it is asked for. + /// + /// + /// Given anew - an edit of the text, say - it is what the file holds from + /// then on, and the bytes or the stored copy it was before are let go. + /// + [JsonConverter(typeof(LongStringConverter))] + public string Code + { + get => _code ??= Decode(); + init + { + _code = value; + _bytes = null; + Stored = null; + } + } + + /// "base64" for a file that is not text, absent otherwise. + public string? Encoding { get; init; } + + /// + /// Where the lambda's resource store keeps it - for a resource of a saved + /// version or feature, and nothing for a file sent or read with what it holds. + /// + [JsonIgnore] + public StoredContent? Stored { get; private init; } /// Whether this is C# the lambda is compiled from. /// @@ -35,11 +136,131 @@ public sealed record LambdaFile(string Name, [property: JsonConverter(typeof(Lon [JsonIgnore] public bool IsResource => LambdaSource.IsResource(Name); - /// The bytes of a file, whatever it was sent as. + /// + /// Whether what it holds is text sent with it - base64 that still has to + /// be checked, say - rather than bytes it was read as or a stored copy. + /// + [JsonIgnore] + public bool IsSent => _bytes == null && Stored == null; + + /// The bytes of a file, whatever it was sent, read or kept as. + /// A copy, made anew every time it is asked for - streams it. + [JsonIgnore] + public byte[] Bytes + { + get + { + if (_bytes != null) + { + return _bytes; + } + + if (Stored is { } stored) + { + return stored.ReadAll(); + } + + return Encoding == "base64" ? Convert.FromBase64String(Code) : Text.GetBytes(Code); + } + } + + /// How many bytes it holds - decoded rather than as sent - without making any of them. [JsonIgnore] - public byte[] Bytes => Encoding == "base64" - ? Convert.FromBase64String(Code) - : System.Text.Encoding.UTF8.GetBytes(Code); + public long Length + { + get + { + if (_bytes != null) + { + return _bytes.Length; + } + + if (Stored is { } stored) + { + return stored.Length; + } + + // a file that is not the base64 it claims to be is caught by + // Validate; here it simply counts for nothing + if (Encoding == "base64") + { + return Base64.IsValid(_code!, out var decoded) ? decoded : 0; + } + + return Text.GetByteCount(_code!); + } + } + + /// The SHA-256 of what it holds, in lowercase hex - the name the resource store keeps it under. + [JsonIgnore] + public string Sha256 => Stored?.Sha256 ?? Convert.ToHexStringLower(SHA256.HashData(Bytes)); + + /// Reads what the file holds, from wherever it is. + public Stream Open() => Stored is { } stored ? stored.Open() : new MemoryStream(Bytes, false); + + private string Decode() + { + var bytes = _bytes ?? Stored?.ReadAll() ?? []; + + return Encoding == "base64" ? Convert.ToBase64String(bytes) : Text.GetString(bytes); + } + + /// + /// The same file: of the same name and encoding, holding the same. + /// + /// + /// Wherever either holds it, so a resource read from the store is the + /// same as the one it was kept from. + /// + public bool Equals(LambdaFile? other) + { + if (other is null) + { + return false; + } + + if (ReferenceEquals(this, other)) + { + return true; + } + + if (Name != other.Name || Encoding != other.Encoding) + { + return false; + } + + if (Stored != null && other.Stored != null) + { + return Stored.Sha256 == other.Stored.Sha256; + } + + if (IsSent && other.IsSent) + { + return _code == other._code; + } + + return Length == other.Length && Sha256 == other.Sha256; + } + + public override int GetHashCode() => HashCode.Combine(Name, Encoding); + + /// + /// Says which file it is - not what it holds, which a file of the store + /// would have to be read for, and a large one would fill the log. + /// + private bool PrintMembers(StringBuilder builder) + { + builder.Append("Name = ").Append(Name); + + if (Encoding != null) + { + builder.Append(", Encoding = ").Append(Encoding); + } + + builder.Append(", Length = ").Append(Length); + + return true; + } } @@ -151,21 +372,36 @@ public static bool IsCompiled(string? name) #endregion + /// + /// Written without escaping what only matters inside HTML: an envelope is + /// stored and read back, never put into a page, and the default escapes + /// every <, >, & and ' of a file in six characters - a third more + /// for a version that keeps a site's pages in its code. + /// private static readonly JsonSerializerOptions Format = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - PropertyNameCaseInsensitive = true + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; /// - /// Marks the stored text as an envelope in today's layout. + /// Marks the stored text as an envelope in today's layout, whose + /// resources are named by what they hold and kept in the resource store. /// /// /// No C# begins this way, so a blob that starts with it was written by this /// and one that does not is code from before there was more than one file - /// - or an envelope of the first layout, which starts with its own marker. + /// - or an envelope of an earlier layout, which starts with its own marker. /// - private const string Marker = "{\"version\":2,\"files\":"; + private const string Marker = "{\"version\":3,\"files\":"; + + /// + /// Marks an envelope written in the second layout: today's names, with + /// what every file holds in the envelope itself, resources included. + /// + private const string SecondMarker = "{\"version\":2,\"files\":"; /// /// Marks an envelope written in the first layout: the files to serve at @@ -176,26 +412,38 @@ public static bool IsCompiled(string? name) #region Functionality /// - /// Reads stored text as the files it holds, in today's layout. + /// Reads stored text as the files it holds, in today's layout - without + /// what the resources hold, which can be named, counted and compared + /// like this but not read. + /// + public static IReadOnlyList Parse(string? stored) => Parse(stored, null); + + /// + /// Reads stored text as the files it holds, in today's layout, its + /// resources read from the store of the lambda it belongs to. /// - public static IReadOnlyList Parse(string? stored) + /// + /// Nothing written in an earlier layout is ever rewritten: it is read in + /// today's, as it is read. An envelope of the second layout still carries + /// what its resources hold, and keeps carrying it until it is saved again. + /// + /// The resource store of the lambda the text belongs to + public static IReadOnlyList Parse(string? stored, IResourceStore? resources) { var text = stored ?? string.Empty; - var current = text.StartsWith(Marker, StringComparison.Ordinal); + var layout = LayoutOf(text); - if (!current && !text.StartsWith(FirstMarker, StringComparison.Ordinal)) + if (layout == 0) { return [new LambdaFile(EntryName, text)]; } try { - var envelope = JsonSerializer.Deserialize(text, Format); - - if (envelope?.Files is { Count: > 0 } files) + if (JsonSerializer.Deserialize(text, Format) is { Files.Count: > 0 } envelope) { - return current ? files : [.. files.Select(f => f with { Name = Moved(f.Name) })]; + return Files(envelope, layout, resources); } } catch (JsonException) @@ -207,6 +455,75 @@ public static IReadOnlyList Parse(string? stored) return [new LambdaFile(EntryName, text)]; } + /// + /// Reads what a version was stored as straight from where it is stored, + /// rather than as a string first. + /// + /// + /// An envelope of the second layout carries every resource of a version, + /// a site's worth of them: read as one string, that string alone took + /// twice what it held, beside the files read out of it. + /// + /// What was stored, which has to be seekable + /// The resource store of the lambda it belongs to + public static IReadOnlyList Read(Stream stored, IResourceStore? resources) + { + var start = new byte[Marker.Length]; + + var layout = LayoutOf(Encoding.ASCII.GetString(start, 0, stored.ReadAtLeast(start, start.Length, throwOnEndOfStream: false))); + + stored.Position = 0; + + if (layout != 0) + { + try + { + if (JsonSerializer.Deserialize(stored, Format) is { Files.Count: > 0 } envelope) + { + return Files(envelope, layout, resources); + } + } + catch (JsonException) + { + // somebody's code that only starts like an envelope, as above + } + + stored.Position = 0; + } + + using var reader = new StreamReader(stored, leaveOpen: true); + + return [new LambdaFile(EntryName, reader.ReadToEnd())]; + } + + /// + /// The hashes of the resources stored text names: what the resource store + /// has to keep for it. + /// + /// + /// Only today's layout names any. Text of an earlier one carries what its + /// resources hold and is not read at all, however large it is. + /// + /// An envelope of today's layout that cannot be read throws rather than + /// naming nothing: what it names would be taken from the store. + /// + /// Stored text that starts as an envelope of today's layout and is not one + public static IReadOnlyList References(Stream stored) + { + var start = new byte[Marker.Length]; + + if (LayoutOf(Encoding.ASCII.GetString(start, 0, stored.ReadAtLeast(start, start.Length, throwOnEndOfStream: false))) != 3) + { + return []; + } + + stored.Position = 0; + + var envelope = JsonSerializer.Deserialize(stored, Format) ?? throw new JsonException("The envelope is empty."); + + return [.. envelope.Files.Where(f => f.Stored != null).Select(f => f.Stored!)]; + } + /// /// Writes the files as the single blob a version is stored as. /// @@ -214,6 +531,10 @@ public static IReadOnlyList Parse(string? stored) /// A lambda that is still one file is stored as that file and nothing else, /// so the common case stays readable on disk and anything reading an older /// version cannot tell the difference. + /// + /// A resource the store keeps is named by its hash; one it does not keep + /// yet is written with what it holds, as it was before there was a store + /// - which is what is for, before a version is written. /// public static string Serialize(IReadOnlyList files) { @@ -222,7 +543,52 @@ public static string Serialize(IReadOnlyList files) return files[0].Code; } - return JsonSerializer.Serialize(new Envelope(2, files), Format); + var stored = files.Select(f => f.Stored is { } kept + ? new StoredFile(f.Name, null, f.Encoding, kept.Sha256, kept.Length) + : new StoredFile(f.Name, f.Code, f.Encoding)) + .ToList(); + + return JsonSerializer.Serialize(new Envelope(3, stored), Format); + } + + /// + /// Puts what the resources hold into the resource store of the lambda + /// they are saved for, and gives back the files as it keeps them. + /// + /// + /// Before anything is written that names them, so that a version names + /// only what is kept. A resource the store keeps already - read from a + /// version of this lambda, or the same as one - is kept once, however + /// many versions ship it. + /// + /// The resource store of the lambda the files are saved for + public static IReadOnlyList Keep(IReadOnlyList files, IResourceStore resources) + { + var kept = new List(files.Count); + + foreach (var file in files) + { + if (!file.IsResource) + { + kept.Add(file); + continue; + } + + if (file.Stored is { } stored && resources.Has(stored.Sha256)) + { + kept.Add(ReferenceEquals(stored.Store, resources) ? file : LambdaFile.FromStore(file.Name, stored with { Store = resources }, file.Encoding)); + continue; + } + + if (file.Stored is { Store: null }) + { + throw new InvalidOperationException($"'{file.Name}' names a resource as {file.Stored.Sha256}, which the lambda it is saved for does not keep."); + } + + kept.Add(LambdaFile.FromStore(file.Name, resources.Keep(file.Bytes), file.Encoding)); + } + + return kept; } /// @@ -230,8 +596,10 @@ public static string Serialize(IReadOnlyList files) /// was written in. /// /// - /// Compared as they are first, which is the answer nearly always: only a - /// blob of the first layout beside one written since is read to tell. + /// Compared as they are first, which is the answer nearly always. Read + /// otherwise - without the resource store, which is not needed to tell: + /// a resource it keeps is compared by its hash, and one written with what + /// it holds by the hash of that. /// public static bool Same(string? stored, string? other) { @@ -245,7 +613,60 @@ public static bool Same(string? stored, string? other) return false; } - return Serialize(Parse(stored)) == Serialize(Parse(other)); + var files = Parse(stored); + + var others = Parse(other); + + return files.Count == others.Count && files.Zip(others).All(pair => pair.First.Equals(pair.Second)); + } + + /// + /// Which layout stored text starts in: 3 for today's, 2 and 1 for the + /// earlier envelopes, 0 for code from before there were several files. + /// + private static int LayoutOf(string text) + { + if (text.StartsWith(Marker, StringComparison.Ordinal)) + { + return 3; + } + + if (text.StartsWith(SecondMarker, StringComparison.Ordinal)) + { + return 2; + } + + return text.StartsWith(FirstMarker, StringComparison.Ordinal) ? 1 : 0; + } + + /// + /// The files of an envelope, named as today's layout names them. + /// + private static IReadOnlyList Files(Envelope envelope, int layout, IResourceStore? resources) + { + var files = new List(envelope.Files.Count); + + foreach (var file in envelope.Files) + { + var name = layout == 1 ? Moved(file.Name) : file.Name; + + if (file.Stored == null) + { + files.Add(new LambdaFile(name, file.Code ?? string.Empty, file.Encoding)); + } + else if (StoredContent.IsHash(file.Stored)) + { + files.Add(LambdaFile.FromStore(name, new StoredContent(file.Stored, file.Length ?? 0, resources), file.Encoding)); + } + else + { + // a hash becomes part of a path in the store, and what is not + // one could name a file outside it + throw new InvalidOperationException($"'{name}' is stored as '{file.Stored}', which is not the hash of a resource."); + } + } + + return files; } /// @@ -308,17 +729,7 @@ public static long Size(IEnumerable files) /// /// How many bytes one file comes to, decoded. /// - public static long Size(LambdaFile file) - { - if (file.Encoding == "base64") - { - // a file that is not the base64 it claims to be is caught by - // Validate; here it simply counts for nothing - return Base64.IsValid(file.Code, out var decoded) ? decoded : 0; - } - - return System.Text.Encoding.UTF8.GetByteCount(file.Code); - } + public static long Size(LambdaFile file) => file.Length; /// /// Checks the shape of what was submitted, and says what is wrong with it. @@ -350,8 +761,9 @@ public static long Size(LambdaFile file) return $"'{file.Name}' asks for encoding '{file.Encoding}'. Only 'base64' is understood; leave it out for text."; } - // the same test Convert.FromBase64String makes, without the copy - if (file.Encoding == "base64" && !Base64.IsValid(file.Code)) + // the same test Convert.FromBase64String makes, without the copy - + // of what was sent as base64, since bytes read or kept are bytes + if (file.Encoding == "base64" && file.IsSent && !Base64.IsValid(file.Code)) { return $"'{file.Name}' says it is base64 and is not."; } @@ -608,7 +1020,14 @@ public static bool IsValidName(string? name) #region Types - private sealed record Envelope(int Version, IReadOnlyList Files); + private sealed record Envelope(int Version, IReadOnlyList Files); + + /// + /// A file as an envelope holds it: with what it holds, or - a resource of + /// today's layout - with the hash and the length it is kept under. + /// + private sealed record StoredFile(string Name, [property: JsonConverter(typeof(LongStringConverter))] string? Code = null, string? Encoding = null, + string? Stored = null, long? Length = null); #endregion diff --git a/src/GenHTTP.Lambda/Services/Deployment/Model/ResourceStore.cs b/src/GenHTTP.Lambda/Services/Deployment/Model/ResourceStore.cs new file mode 100644 index 0000000..4d49b83 --- /dev/null +++ b/src/GenHTTP.Lambda/Services/Deployment/Model/ResourceStore.cs @@ -0,0 +1,78 @@ +namespace GenHTTP.Lambda.Services.Deployment.Model; + +/// +/// Where the resources of one lambda are kept: each once, under the SHA-256 +/// of what it holds, for every version and feature of the lambda that ships it. +/// +/// +/// A version names its resources by their hash rather than carrying them, so +/// that a website of a few thousand pages is a small file of code and names, +/// and its pages are files - copied where it goes online, never read into a +/// string, and kept once however many versions ship them unchanged. +/// +public interface IResourceStore +{ + + /// + /// Keeps what a resource holds, and says what it is kept as. + /// + StoredContent Keep(ReadOnlySpan content); + + /// + /// Whether something is kept under the hash. + /// + bool Has(string sha256); + + /// + /// Reads what is kept under the hash. + /// + Stream Open(string sha256); + + /// + /// Writes what is kept under the hash into a file, replacing it. + /// + void CopyTo(string sha256, string path); + +} + +/// +/// A resource as a lambda's resource store keeps it. +/// +/// The hash of what it holds, in lowercase hex - the name it is kept under +/// How many bytes it holds +/// +/// Where it is kept. Absent where the files were read without the store, which +/// can still name, count and compare them, and say so when asked for more. +/// +public sealed record StoredContent(string Sha256, long Length, IResourceStore? Store) +{ + + public Stream Open() => Available.Open(Sha256); + + public byte[] ReadAll() + { + using var source = Open(); + + using var target = new MemoryStream(checked((int)Length)); + + source.CopyTo(target); + + return target.ToArray(); + } + + public void CopyTo(string path) => Available.CopyTo(Sha256, path); + + private IResourceStore Available + => Store ?? throw new InvalidOperationException($"The resource kept as {Sha256} was read without the store that keeps it, which has to be handed to LambdaSource.Parse to read what it holds."); + + /// + /// Whether a text is a hash a resource can be kept under: 64 lowercase hex digits. + /// + /// + /// Checked before a hash becomes part of a path, so that what an envelope + /// says can never name a file outside the store. + /// + public static bool IsHash(string? text) + => text is { Length: 64 } && text.All(c => c is >= '0' and <= '9' or >= 'a' and <= 'f'); + +} diff --git a/src/GenHTTP.Lambda/Services/Features/FeatureModel.cs b/src/GenHTTP.Lambda/Services/Features/FeatureModel.cs index 9c3a0f0..3af8425 100644 --- a/src/GenHTTP.Lambda/Services/Features/FeatureModel.cs +++ b/src/GenHTTP.Lambda/Services/Features/FeatureModel.cs @@ -1,3 +1,5 @@ +using System.Text.Json.Serialization; + using GenHTTP.Lambda.Services.Deployment.Model; using GenHTTP.Lambda.Services.Meta.Model; @@ -54,7 +56,16 @@ int Revision /// A feature with the files it holds. /// /// Every file, as the single blob a version is stored as -public sealed record FeatureContent(FeatureInfo Feature, string Code); +/// Where the lambda keeps the resources the blob names, to read what they hold +public sealed record FeatureContent(FeatureInfo Feature, string Code, [property: JsonIgnore] IResourceStore? Resources = null) +{ + + /// + /// The files, what the resources hold read from the store where somebody asks for it. + /// + public IReadOnlyList Files() => LambdaSource.Parse(Code, Resources); + +} /// /// What a new feature is to be. diff --git a/src/GenHTTP.Lambda/Services/Features/FeatureService.cs b/src/GenHTTP.Lambda/Services/Features/FeatureService.cs index 512b7ec..fa29e7e 100644 --- a/src/GenHTTP.Lambda/Services/Features/FeatureService.cs +++ b/src/GenHTTP.Lambda/Services/Features/FeatureService.cs @@ -102,7 +102,7 @@ public FeatureContent Get(string privateKey, string feature) var code = storage.ReadFeature(lambda.Id, entity.Id) ?? throw LambdaException.NotFound($"The files of the feature '{entity.Name}' are no longer available."); - return new FeatureContent(Describe(entity, Newest(database, lambda.Id)), code); + return new FeatureContent(Describe(entity, Newest(database, lambda.Id)), code, storage.Resources(lambda.Id)); } public (long LambdaId, long FeatureId) Require(string privateKey, string feature, bool editable) @@ -313,8 +313,11 @@ public FeatureInfo Update(string privateKey, string feature, FeatureUpdate updat } public FeatureInfo Save(string privateKey, string feature, string code, VersionNote? note = null, int? after = null) + => Save(privateKey, feature, VersionInput.Validate(code), note, after); + + public FeatureInfo Save(string privateKey, string feature, IReadOnlyList files, VersionNote? note = null, int? after = null) { - var files = VersionInput.Validate(code); + VersionInput.Validate(files); var (database, lambda, entity, turn) = Locked(privateKey, feature); @@ -334,7 +337,9 @@ public FeatureInfo Save(string privateKey, string feature, string code, VersionN // version may hold from the start rather than refused when merged VersionInput.ValidateAllowance(files, lambda.Tier, limits); - storage.WriteFeature(lambda.Id, entity.Id, code); + // its resources go into the lambda's store, which every feature of it + // shares with the versions, and the feature names them + storage.WriteFeature(lambda.Id, entity.Id, LambdaSource.Serialize(LambdaSource.Keep(files, storage.Resources(lambda.Id)))); entity.Specification = VersionInput.Tidy(note?.Specification, VersionNote.MaxSpecification) ?? entity.Specification; entity.Change = VersionInput.Tidy(note?.Change, VersionNote.MaxChange) ?? entity.Change; diff --git a/src/GenHTTP.Lambda/Services/Features/IFeatureService.cs b/src/GenHTTP.Lambda/Services/Features/IFeatureService.cs index b44ed70..341a515 100644 --- a/src/GenHTTP.Lambda/Services/Features/IFeatureService.cs +++ b/src/GenHTTP.Lambda/Services/Features/IFeatureService.cs @@ -49,6 +49,12 @@ public interface IFeatureService /// FeatureInfo Save(string privateKey, string feature, string code, VersionNote? note = null, int? after = null); + /// + /// Saves files into the feature - what an archive, a push or a request + /// holds - without making them one text first. + /// + FeatureInfo Save(string privateKey, string feature, IReadOnlyList files, VersionNote? note = null, int? after = null); + /// /// Puts the feature's files online at its own address, with its own copy /// of the data. The lambda itself is not touched. diff --git a/src/GenHTTP.Lambda/Services/Git/GitHistory.cs b/src/GenHTTP.Lambda/Services/Git/GitHistory.cs index 3015734..f412c59 100644 --- a/src/GenHTTP.Lambda/Services/Git/GitHistory.cs +++ b/src/GenHTTP.Lambda/Services/Git/GitHistory.cs @@ -5,6 +5,7 @@ using GenHTTP.Lambda.Services.Features; using GenHTTP.Lambda.Services.Meta; using GenHTTP.Lambda.Services.Meta.Model; +using GenHTTP.Lambda.Services.Storage; using Microsoft.Extensions.Logging; @@ -36,7 +37,7 @@ namespace GenHTTP.Lambda.Services.Git; /// away from the reactor: the first read of a lambda with a long history /// reads every version. /// -public sealed class GitHistory(GitStore store, IMetaService meta, IFeatureService features, ILogger logger) +public sealed class GitHistory(GitStore store, IMetaService meta, IFeatureService features, IStorageService storage, ILogger logger) { /// @@ -98,7 +99,7 @@ public async ValueTask SyncAsync(GitLambda lambda, bool withFeatures) var previous = parent != null ? store.ReadCommit(lambda.Id, GitObjectId.Parse(parent)) : null; - var made = await CommitAsync(lambda, LambdaSource.Parse(code), parent != null ? [parent] : [], previous, + var made = await CommitAsync(lambda, LambdaSource.Parse(code, storage.Resources(lambda.Id)), parent != null ? [parent] : [], previous, OneLine(version.Change) ?? $"Version {version.Version}", version.Created, version.Version); index.Versions[version.Version] = made; @@ -236,7 +237,7 @@ private async ValueTask SyncAsync(GitLambda lambda, GitIndex index, Featur ? $"Brings version {feature.Base} into the feature '{feature.Name}'" : OneLine(feature.Change) ?? feature.Name; - var commit = await CommitAsync(lambda, LambdaSource.Parse(code), parents, previous, message, feature.Modified, null); + var commit = await CommitAsync(lambda, LambdaSource.Parse(code, storage.Resources(lambda.Id)), parents, previous, message, feature.Modified, null); index.Features[feature.Key] = new FeatureTip(commit, feature.Revision, feature.Base); @@ -272,7 +273,7 @@ private async ValueTask MoveAsync(GitLambda lambda, GitIndex index, stri ? "Moves assets/ to resources/, where the platform keeps what a lambda reads and serves now" : "Brings the platform's files up to date with how it lays a lambda out now"; - var made = await CommitAsync(lambda, LambdaSource.Parse(code()), [tip], null, message, DateTime.UtcNow, null); + var made = await CommitAsync(lambda, LambdaSource.Parse(code(), storage.Resources(lambda.Id)), [tip], null, message, DateTime.UtcNow, null); index.Moved[tip] = made; diff --git a/src/GenHTTP.Lambda/Services/Git/GitPushes.cs b/src/GenHTTP.Lambda/Services/Git/GitPushes.cs index 1eef886..101a583 100644 --- a/src/GenHTTP.Lambda/Services/Git/GitPushes.cs +++ b/src/GenHTTP.Lambda/Services/Git/GitPushes.cs @@ -193,7 +193,7 @@ private async ValueTask MainAsync(GitLambda lambda, GitIndex index, GitReference // a version is only made of what compiles, as when a feature is merged var tip = steps[^1]; - var check = await meta.CheckAsync(lambda.PrivateKey, LambdaSource.Serialize(tip.Read.Files!)); + var check = await meta.CheckAsync(lambda.PrivateKey, tip.Read.Files!); if (!check.Success) { @@ -214,7 +214,7 @@ private async ValueTask MainAsync(GitLambda lambda, GitIndex index, GitReference try { - version = meta.Save(lambda.PrivateKey, LambdaSource.Serialize(step.Read.Files!), new VersionNote(specification, change, VersionOrigins.Git), after); + version = meta.Save(lambda.PrivateKey, step.Read.Files!, new VersionNote(specification, change, VersionOrigins.Git), after); } catch (LambdaException e) when (saved.Count > 0) { @@ -366,8 +366,6 @@ private async ValueTask FeatureAsync(GitLambda lambda, GitIndex index, GitRefere return; } - var code = LambdaSource.Serialize(read.Files); - var (change, _) = Message(commits.Find(update.NewId)!); var started = feature == null; @@ -383,7 +381,7 @@ private async ValueTask FeatureAsync(GitLambda lambda, GitIndex index, GitRefere try { - feature = features.Save(lambda.PrivateKey, feature.Key, code, new VersionNote(Change: feature.Change == null ? change : null, Origin: VersionOrigins.Git), + feature = features.Save(lambda.PrivateKey, feature.Key, read.Files, new VersionNote(Change: feature.Change == null ? change : null, Origin: VersionOrigins.Git), after: feature.Revision); } catch (LambdaException) when (started) diff --git a/src/GenHTTP.Lambda/Services/Git/ProjectTree.cs b/src/GenHTTP.Lambda/Services/Git/ProjectTree.cs index 5765dd5..4546fc9 100644 --- a/src/GenHTTP.Lambda/Services/Git/ProjectTree.cs +++ b/src/GenHTTP.Lambda/Services/Git/ProjectTree.cs @@ -213,6 +213,15 @@ public static async ValueTask ReadAsync(IReadOnlyList tree, I files.Add(new LambdaFile(name, text)); stored.Add(new StoredFile(name, entry.Path, blob, false)); } + else if (LambdaSource.IsResource(name)) + { + // the bytes it is, until the lambda's store has them - binary + // where it was before, whatever its bytes look like now + var file = LambdaFile.FromBytes(name, content, known is { Binary: true } && known.Blob == blob); + + files.Add(file); + stored.Add(new StoredFile(name, entry.Path, blob, file.Encoding == "base64")); + } else if (known is { Binary: true } && known.Blob == blob || !TryText(content, out var asText)) { files.Add(new LambdaFile(name, Convert.ToBase64String(content), "base64")); diff --git a/src/GenHTTP.Lambda/Services/Meta/DemoSeeder.cs b/src/GenHTTP.Lambda/Services/Meta/DemoSeeder.cs index 88fc7f1..e7a66d5 100644 --- a/src/GenHTTP.Lambda/Services/Meta/DemoSeeder.cs +++ b/src/GenHTTP.Lambda/Services/Meta/DemoSeeder.cs @@ -4,6 +4,7 @@ using GenHTTP.Lambda.Data.Entities; using GenHTTP.Lambda.Services.Data; using GenHTTP.Lambda.Services.Databases; +using GenHTTP.Lambda.Services.Deployment.Model; using GenHTTP.Lambda.Services.Meta.Model; using GenHTTP.Lambda.Services.Secrets; @@ -125,12 +126,16 @@ private async ValueTask SeedAsync(LambdaDemo demo, CancellationToken cance var current = CurrentCode(demo.Key, lambda); - if (current == wanted && lambda?.ActiveVersion != null) + // compared by what the files hold: a saved version names its resources + // by their hash, where the template carries them + var same = LambdaSource.Same(current, wanted); + + if (same && lambda?.ActiveVersion != null) { return false; } - if (current != wanted) + if (!same) { var change = current == null ? "Set up as a demo" : "Brought up to date with its template"; diff --git a/src/GenHTTP.Lambda/Services/Meta/IMetaService.cs b/src/GenHTTP.Lambda/Services/Meta/IMetaService.cs index d9611aa..c6e8063 100644 --- a/src/GenHTTP.Lambda/Services/Meta/IMetaService.cs +++ b/src/GenHTTP.Lambda/Services/Meta/IMetaService.cs @@ -68,11 +68,26 @@ public interface IMetaService /// LambdaVersionInfo Save(string privateKey, string code, VersionNote? note = null, int? after = null); + /// + /// Saves files as the next version - what an archive, a push or a request + /// holds - without making them one text first. + /// + /// + /// The resources go into the lambda's store as the bytes they were read + /// as, rather than as a copy of them twice their size, in base64. + /// + LambdaVersionInfo Save(string privateKey, IReadOnlyList files, VersionNote? note = null, int? after = null); + /// /// Compiles the given code without deploying it. /// ValueTask CheckAsync(string privateKey, string code, CancellationToken cancellation = default); + /// + /// Checks whether files would compile as a version of the lambda, without saving them. + /// + ValueTask CheckAsync(string privateKey, IReadOnlyList files, CancellationToken cancellation = default); + /// /// Builds the given version (or the latest one) and makes it the active deployment. /// diff --git a/src/GenHTTP.Lambda/Services/Meta/LambdaHistory.cs b/src/GenHTTP.Lambda/Services/Meta/LambdaHistory.cs index 432e76e..767c17c 100644 --- a/src/GenHTTP.Lambda/Services/Meta/LambdaHistory.cs +++ b/src/GenHTTP.Lambda/Services/Meta/LambdaHistory.cs @@ -1,5 +1,6 @@ using GenHTTP.Lambda.Data; using GenHTTP.Lambda.Data.Entities; +using GenHTTP.Lambda.Services.Deployment.Model; using GenHTTP.Lambda.Services.Meta.Model; using GenHTTP.Lambda.Services.Settings; using GenHTTP.Lambda.Services.Storage; @@ -42,6 +43,17 @@ public sealed class LambdaHistory(IStorageService storage, ILimitsService limits /// go from both. /// public LambdaVersionInfo Append(LambdaDbContext database, LambdaEntity lambda, string code, DateTime now, VersionNote note) + => Append(database, lambda, LambdaSource.Parse(code, storage.Resources(lambda.Id)), now, note); + + /// + /// Adds the files as the next version of the lambda, and saves. + /// + /// + /// The resources go into the lambda's store first, and the version names + /// them: a resource the store keeps already - shipped unchanged from the + /// version before - is not kept again. + /// + public LambdaVersionInfo Append(LambdaDbContext database, LambdaEntity lambda, IReadOnlyList files, DateTime now, VersionNote note) { var version = database.Deployments.Where(d => d.LambdaId == lambda.Id) .Max(d => (int?)d.Version) + 1 ?? 1; @@ -50,7 +62,7 @@ public LambdaVersionInfo Append(LambdaDbContext database, LambdaEntity lambda, s var change = VersionInput.Tidy(note.Change, VersionNote.MaxChange); - storage.Write(lambda.Id, version, code); + storage.Write(lambda.Id, version, LambdaSource.Serialize(LambdaSource.Keep(files, storage.Resources(lambda.Id)))); database.Deployments.Add(new DeploymentEntity { diff --git a/src/GenHTTP.Lambda/Services/Meta/MetaService.cs b/src/GenHTTP.Lambda/Services/Meta/MetaService.cs index 93f801e..e76ad15 100644 --- a/src/GenHTTP.Lambda/Services/Meta/MetaService.cs +++ b/src/GenHTTP.Lambda/Services/Meta/MetaService.cs @@ -347,7 +347,7 @@ public LambdaVersionContent GetVersion(string privateKey, int version) var code = Storage.Read(lambda.Id, version) ?? throw LambdaException.NotFound($"The code of version {version} is no longer available."); - return new LambdaVersionContent(deployment.Version, deployment.Created, code, deployment.Specification, deployment.Change, deployment.Origin); + return new LambdaVersionContent(deployment.Version, deployment.Created, code, deployment.Specification, deployment.Change, deployment.Origin, Storage.Resources(lambda.Id)); } public IReadOnlyList GetActivations(string privateKey) @@ -430,8 +430,11 @@ public LambdaCounts Count() #region Editing public LambdaVersionInfo Save(string privateKey, string code, VersionNote? note = null, int? after = null) + => Save(privateKey, VersionInput.Validate(code), note, after); + + public LambdaVersionInfo Save(string privateKey, IReadOnlyList files, VersionNote? note = null, int? after = null) { - var files = VersionInput.Validate(code); + VersionInput.Validate(files); using var database = Databases.CreateDbContext(); @@ -456,16 +459,19 @@ public LambdaVersionInfo Save(string privateKey, string code, VersionNote? note VersionInput.ValidateAllowance(files, lambda.Tier, Limits); - var version = History.Append(database, lambda, code, DateTime.UtcNow, note); + var version = History.Append(database, lambda, files, DateTime.UtcNow, note); Logger.LogInformation("Saved lambda #{LambdaId} version {Version}", lambda.Id, version.Version); return version; } - public async ValueTask CheckAsync(string privateKey, string code, CancellationToken cancellation = default) + public ValueTask CheckAsync(string privateKey, string code, CancellationToken cancellation = default) + => CheckAsync(privateKey, VersionInput.Validate(code), cancellation); + + public async ValueTask CheckAsync(string privateKey, IReadOnlyList files, CancellationToken cancellation = default) { - var files = VersionInput.Validate(code); + VersionInput.Validate(files); long id; @@ -480,7 +486,7 @@ public async ValueTask CheckAsync(string privateKey, string (id, limits) = (lambda.Id, WorkspaceOf(database, lambda)); } - return await Deployments.ValidateAsync(code, id, limits, cancellation); + return await Deployments.ValidateAsync(files, id, limits, cancellation); } /// diff --git a/src/GenHTTP.Lambda/Services/Meta/Model/LambdaModel.cs b/src/GenHTTP.Lambda/Services/Meta/Model/LambdaModel.cs index 5bb026d..6f444d7 100644 --- a/src/GenHTTP.Lambda/Services/Meta/Model/LambdaModel.cs +++ b/src/GenHTTP.Lambda/Services/Meta/Model/LambdaModel.cs @@ -1,3 +1,5 @@ +using System.Text.Json.Serialization; + using GenHTTP.Lambda.Data.Entities; using GenHTTP.Lambda.Services.Deployment.Model; @@ -54,7 +56,18 @@ public sealed record LambdaVersionInfo(int Version, DateTime Created, string? Sp /// /// A stored version, including the code itself. /// -public sealed record LambdaVersionContent(int Version, DateTime Created, string Code, string? Specification = null, string? Change = null, string? Origin = null); +/// Every file, as the single blob a version is stored as +/// Where the lambda keeps the resources the blob names, to read what they hold +public sealed record LambdaVersionContent(int Version, DateTime Created, string Code, string? Specification = null, string? Change = null, string? Origin = null, + [property: JsonIgnore] IResourceStore? Resources = null) +{ + + /// + /// The files, what the resources hold read from the store where somebody asks for it. + /// + public IReadOnlyList Files() => LambdaSource.Parse(Code, Resources); + +} /// /// Why a version was written, as whoever wrote it tells it. diff --git a/src/GenHTTP.Lambda/Services/Meta/VersionInput.cs b/src/GenHTTP.Lambda/Services/Meta/VersionInput.cs index 9dbd971..928be4d 100644 --- a/src/GenHTTP.Lambda/Services/Meta/VersionInput.cs +++ b/src/GenHTTP.Lambda/Services/Meta/VersionInput.cs @@ -27,14 +27,22 @@ public static IReadOnlyList Validate(string? code) throw LambdaException.Invalid("The code must not be empty."); } - var files = LambdaSource.Parse(code); + return Validate(LambdaSource.Parse(code)); + } + /// + /// Checks files sent or read as they are - what an archive or a push + /// holds - the way checks text. + /// + /// The files, for the checks that do need to know whose they are + public static IReadOnlyList Validate(IReadOnlyList? files) + { if (LambdaSource.Validate(files) is { } complaint) { throw LambdaException.Invalid(complaint); } - return files; + return files!; } /// @@ -48,8 +56,9 @@ public static IReadOnlyList Validate(string? code) /// what the premium tier allows, so whoever reads it knows there is more. /// A feature is held to the same, since what it holds becomes a version. /// - /// Counted in bytes as they are, whatever a file is - every version - /// carries its own copy of all of it, and is read and written whole. + /// Counted in bytes as they are, whatever a file is - all of it, however + /// much of it the lambda's store keeps already for the versions before: + /// any version may be the one put online, and it ships all of it. /// public static void ValidateAllowance(IReadOnlyList files, LambdaTier tier, ILimitsService limits) { diff --git a/src/GenHTTP.Lambda/Services/Source/SourceService.cs b/src/GenHTTP.Lambda/Services/Source/SourceService.cs index 51935ca..3c2cf6a 100644 --- a/src/GenHTTP.Lambda/Services/Source/SourceService.cs +++ b/src/GenHTTP.Lambda/Services/Source/SourceService.cs @@ -243,7 +243,7 @@ public SourceListing List(string? search, SourceOrder order, int skip, int take) var code = storage.Read(row.Id, version) ?? throw LambdaException.NotFound($"The code of version {version} is missing."); - var files = LambdaSource.Parse(code); + var files = LambdaSource.Parse(code, storage.Resources(row.Id)); // the names the code reads, which it says itself - never what is // stored, and never a value diff --git a/src/GenHTTP.Lambda/Services/Storage/FileSystemStorageService.cs b/src/GenHTTP.Lambda/Services/Storage/FileSystemStorageService.cs index 23759a2..b17711d 100644 --- a/src/GenHTTP.Lambda/Services/Storage/FileSystemStorageService.cs +++ b/src/GenHTTP.Lambda/Services/Storage/FileSystemStorageService.cs @@ -1,5 +1,6 @@ using GenHTTP.Lambda.Configuration; using GenHTTP.Lambda.Infrastructure; +using GenHTTP.Lambda.Services.Deployment.Model; using Microsoft.Extensions.Logging; @@ -10,6 +11,11 @@ namespace GenHTTP.Lambda.Services.Storage; /// every lambda a private directory below {data}/workspaces/{id}. /// /// +/// The resources of all versions and features of a lambda are kept below +/// {data}/code/{id}/resources, each once and named by what it holds +/// (), and the versions name them. What nothing +/// names any more is removed with a version or a feature. +/// /// A feature keeps everything of its own below {data}/features/{id}/{feature}: /// files.json as it is being worked on, preview.json as its /// preview was deployed, its copy of the workspace in workspace and @@ -89,6 +95,10 @@ public void Write(long lambdaId, int version, string code) return File.ReadAllText(file); } + public IReadOnlyList? ReadFiles(long lambdaId, int version) => ReadFilesIfThere(lambdaId, GetFile(lambdaId, version)); + + public IResourceStore Resources(long lambdaId) => new ResourceStore(GetResourceStoreDirectory(lambdaId)); + public void DeleteVersion(long lambdaId, int version) { var file = GetFile(lambdaId, version); @@ -98,6 +108,9 @@ public void DeleteVersion(long lambdaId, int version) File.Delete(file); } + // what only this version named + CollectResources(lambdaId); + // and the project its published source was packed into, which no // visitor can ask for any more var packed = Path.Combine(Options.SourceDirectory, lambdaId.ToString()); @@ -183,6 +196,9 @@ public void WritePreview(long lambdaId, long featureId, string code) public string? ReadPreview(long lambdaId, long featureId) => ReadIfThere(Path.Combine(GetFeatureDirectory(lambdaId, featureId), "preview.json")); + public IReadOnlyList? ReadPreviewFiles(long lambdaId, long featureId) + => ReadFilesIfThere(lambdaId, Path.Combine(GetFeatureDirectory(lambdaId, featureId), "preview.json")); + /// /// Copied beside the copy it replaces and swapped in once complete, so a /// copy that fails halfway leaves the one there was. On a thread of its @@ -226,6 +242,9 @@ await Offload.Run(() => public void DeleteFeature(long lambdaId, long featureId) { Remove(GetFeatureDirectory(lambdaId, featureId)); + + // what only the feature named, its preview included + CollectResources(lambdaId); } public IEnumerable<(long LambdaId, long FeatureId)> ListFeatures() @@ -297,6 +316,25 @@ private static void WriteWhole(string file, string code) File.Move(staging, file, true); } + /// + /// Reads stored files straight from their file, the resources from the + /// lambda's store - without the string an envelope that carries them is. + /// + private IReadOnlyList? ReadFilesIfThere(long lambdaId, string file) + { + try + { + using var stored = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + + return LambdaSource.Read(stored, Resources(lambdaId)); + } + catch (Exception e) when (e is FileNotFoundException or DirectoryNotFoundException) + { + // removed between looking and reading, or never there + return null; + } + } + private static string? ReadIfThere(string file) { try @@ -312,10 +350,91 @@ private static void WriteWhole(string file, string code) #endregion + #region Resources + + /// + /// How long a resource nothing names is kept all the same. + /// + /// + /// The resources of a save are kept before the version or the feature + /// that names them is written, and a version is removed under the lambda's + /// turn while a feature is saved under its own - so what was kept a moment + /// ago may be named a moment from now. An hour is longer than any save. + /// + private static readonly TimeSpan ResourceGrace = TimeSpan.FromHours(1); + + /// + /// Removes the resources none of the lambda's versions, features and + /// previews names any more. + /// + /// + /// Anything that cannot be read leaves everything kept: a name that is + /// missed would take a resource somebody still serves. + /// + private void CollectResources(long lambdaId) + { + var store = GetResourceStoreDirectory(lambdaId); + + if (!Directory.Exists(store)) + { + return; + } + + try + { + var named = new HashSet(StringComparer.Ordinal); + + foreach (var version in Directory.EnumerateFiles(GetCodeDirectory(lambdaId), "v*.cs")) + { + Name(version, named); + } + + var features = Path.Combine(Options.FeatureDirectory, lambdaId.ToString()); + + if (Directory.Exists(features)) + { + foreach (var feature in Directory.EnumerateDirectories(features)) + { + Name(Path.Combine(feature, "files.json"), named); + Name(Path.Combine(feature, "preview.json"), named); + } + } + + var (files, bytes) = new ResourceStore(store).Collect(named, DateTime.UtcNow - ResourceGrace); + + if (files > 0) + { + Logger.LogInformation("Removed {Files} resources of lambda #{LambdaId} no version or feature names any more ({Bytes} bytes)", files, lambdaId, bytes); + } + } + catch (Exception e) when (e is IOException or System.Text.Json.JsonException) + { + Logger.LogWarning(e, "Kept the resources of lambda #{LambdaId}: what names them could not be read", lambdaId); + } + } + + private static void Name(string file, HashSet named) + { + try + { + using var stored = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + + named.UnionWith(LambdaSource.References(stored)); + } + catch (Exception e) when (e is FileNotFoundException or DirectoryNotFoundException) + { + // not there, so it names nothing + } + } + + #endregion + #region Helpers private string GetCodeDirectory(long lambdaId) => Path.Combine(Options.CodeDirectory, lambdaId.ToString()); + private string GetResourceStoreDirectory(long lambdaId) => Path.Combine(GetCodeDirectory(lambdaId), "resources"); + private string GetWorkspaceDirectory(long lambdaId) => Path.Combine(Options.WorkspaceDirectory, lambdaId.ToString()); private string GetFeatureDirectory(long lambdaId, long featureId) diff --git a/src/GenHTTP.Lambda/Services/Storage/IStorageService.cs b/src/GenHTTP.Lambda/Services/Storage/IStorageService.cs index 016a705..da728f9 100644 --- a/src/GenHTTP.Lambda/Services/Storage/IStorageService.cs +++ b/src/GenHTTP.Lambda/Services/Storage/IStorageService.cs @@ -1,3 +1,5 @@ +using GenHTTP.Lambda.Services.Deployment.Model; + namespace GenHTTP.Lambda.Services.Storage; /// @@ -15,8 +17,29 @@ public interface IStorageService /// /// Reads the code of the given version, if it exists. /// + /// + /// As it was stored: its resources named by what they hold, which + /// keeps - or, for a version saved before there + /// was a store, with what they hold. + /// string? Read(long lambdaId, int version); + /// + /// Reads the files of the given version, if it exists, straight from where + /// it is stored and with its resources read from the lambda's store. + /// + IReadOnlyList? ReadFiles(long lambdaId, int version); + + /// + /// Where the resources of a lambda are kept: each once, by what it holds, + /// for all its versions and features. + /// + /// + /// What is no longer named by a version, a feature or a preview of the + /// lambda goes when a version or a feature is removed. + /// + IResourceStore Resources(long lambdaId); + /// /// Removes a single version of a lambda. /// @@ -91,6 +114,13 @@ public interface IStorageService /// string? ReadPreview(long lambdaId, long featureId); + /// + /// Reads the files the preview of a feature was deployed with, if it was, + /// straight from where they are stored and with the resources read from + /// the lambda's store. + /// + IReadOnlyList? ReadPreviewFiles(long lambdaId, long featureId); + /// /// Replaces the feature's copy of the workspace with a fresh copy of the /// lambda's own. diff --git a/src/GenHTTP.Lambda/Services/Storage/ResourceStore.cs b/src/GenHTTP.Lambda/Services/Storage/ResourceStore.cs new file mode 100644 index 0000000..903be2c --- /dev/null +++ b/src/GenHTTP.Lambda/Services/Storage/ResourceStore.cs @@ -0,0 +1,129 @@ +using System.Security.Cryptography; + +using GenHTTP.Lambda.Services.Deployment.Model; + +namespace GenHTTP.Lambda.Services.Storage; + +/// +/// Keeps the resources of one lambda as files below {data}/code/{id}/resources, +/// each named by the SHA-256 of what it holds, in a folder of its first two digits. +/// +/// +/// Written through a file beside it and moved into place, so that a name in +/// the store always holds all of what it names - whoever finds it there may +/// copy it out at once. +/// +internal sealed class ResourceStore(string root) : IResourceStore +{ + + #region Functionality + + public StoredContent Keep(ReadOnlySpan content) + { + var sha256 = Convert.ToHexStringLower(SHA256.HashData(content)); + + var path = PathOf(sha256); + + if (File.Exists(path)) + { + // kept already, and named again: young again too, so that a + // collection does not take it before what names it is written + File.SetLastWriteTimeUtc(path, DateTime.UtcNow); + } + else + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + + var staging = $"{path}.{Guid.NewGuid():N}.keeping"; + + using (var file = new FileStream(staging, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + file.Write(content); + } + + try + { + File.Move(staging, path, false); + } + catch (IOException) when (File.Exists(path)) + { + // the same, kept by another save in the meantime + File.Delete(staging); + } + } + + return new StoredContent(sha256, content.Length, this); + } + + public bool Has(string sha256) => StoredContent.IsHash(sha256) && File.Exists(PathOf(sha256)); + + public Stream Open(string sha256) => new FileStream(PathOf(sha256), FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + + public void CopyTo(string sha256, string path) => File.Copy(PathOf(sha256), path, true); + + /// + /// Removes what nothing names any more and was last named before the + /// given time, and says how much that was. + /// + /// + /// The time is a grace for what is being saved right now: its resources + /// are kept before the version that names them is written, and a + /// collection in between must not take them. + /// + public (int Files, long Bytes) Collect(IReadOnlySet named, DateTime before) + { + if (!Directory.Exists(root)) + { + return (0, 0); + } + + int files = 0; + + long bytes = 0; + + foreach (var file in new DirectoryInfo(root).EnumerateFiles("*", SearchOption.AllDirectories)) + { + // a file still being kept is not one yet, and one left behind by + // a save that broke off is a name nothing could know + var orphan = file.Name.EndsWith(".keeping", StringComparison.Ordinal); + + if (!orphan && named.Contains(file.Name)) + { + continue; + } + + if (file.LastWriteTimeUtc >= before) + { + continue; + } + + try + { + var length = file.Length; + + file.Delete(); + + files++; + bytes += length; + } + catch (IOException) + { + // being read right now; the next collection takes it + } + } + + return (files, bytes); + } + + #endregion + + #region Helpers + + private string PathOf(string sha256) + => StoredContent.IsHash(sha256) + ? Path.Combine(root, sha256[..2], sha256) + : throw new ArgumentException($"'{sha256}' is not the hash of a resource.", nameof(sha256)); + + #endregion + +} diff --git a/tests/GenHTTP.Lambda.Tests/Lambdas/LayoutMigrationTests.cs b/tests/GenHTTP.Lambda.Tests/Lambdas/LayoutMigrationTests.cs index af2a441..ed21048 100644 --- a/tests/GenHTTP.Lambda.Tests/Lambdas/LayoutMigrationTests.cs +++ b/tests/GenHTTP.Lambda.Tests/Lambdas/LayoutMigrationTests.cs @@ -116,7 +116,7 @@ public async Task ALambdaKeptInTheFirstLayoutRunsAsItDid() var stored = await File.ReadAllTextAsync(VersionFile(fixture, lambda, 3)); - StringAssert.StartsWith(stored, "{\"version\":2,", "what is saved now is saved in today's layout"); + StringAssert.StartsWith(stored, "{\"version\":3,", "what is saved now is saved in today's layout"); StringAssert.StartsWith(await File.ReadAllTextAsync(VersionFile(fixture, lambda, 2)), "{\"version\":1,", "and what was saved then is left as it was"); } diff --git a/tests/GenHTTP.Lambda.Tests/Lambdas/ResourceStoreTests.cs b/tests/GenHTTP.Lambda.Tests/Lambdas/ResourceStoreTests.cs new file mode 100644 index 0000000..64a1971 --- /dev/null +++ b/tests/GenHTTP.Lambda.Tests/Lambdas/ResourceStoreTests.cs @@ -0,0 +1,314 @@ +using System.IO.Compression; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +using GenHTTP.Lambda.Api.Model; +using GenHTTP.Lambda.Services.Deployment.Model; +using GenHTTP.Lambda.Tests.Git; +using GenHTTP.Lambda.Tests.Infrastructure; + +using GenHTTP.Testing; + +namespace GenHTTP.Lambda.Tests.Lambdas; + +/// +/// The resources of a lambda kept once each, by what they hold, and named by +/// the versions and features that ship them - rather than carried inside +/// every one of them. +/// +[TestClass] +public sealed class ResourceStoreTests +{ + + [TestMethod] + public async Task AVersionNamesItsResourcesRatherThanCarryingThem() + { + await using var fixture = await LambdaFixture.CreateAsync(); + + var lambda = await fixture.CreateLambdaAsync("pages"); + + var page = Encoding.UTF8.GetBytes("

a page of a site

"); + + // not text: a NUL, and bytes no UTF-8 has + byte[] logo = [0x89, 0x50, 0x4E, 0x47, 0x00, 0xFF, 0x10, 0x00]; + + using (var uploaded = await UploadAsync(fixture, lambda.PrivateKey, Zip(("lambda.cs", "return Resources.App(\"web\");"u8.ToArray()), + ("resources/web/index.html", page), ("resources/web/logo.png", logo)))) + { + Assert.AreEqual(HttpStatusCode.Created, uploaded.StatusCode, await uploaded.Content.ReadAsStringAsync()); + } + + var deployed = await fixture.DeployAsync(lambda.PrivateKey); + + Assert.IsTrue(deployed.Success, string.Join("; ", deployed.Diagnostics.Select(d => d.Message))); + + Assert.AreEqual("

a page of a site

", await fixture.CallAsync("http://pages.localhost/"), "served from where it went online"); + + using (var served = await fixture.GetAsync("http://pages.localhost/logo.png")) + { + CollectionAssert.AreEqual(logo, await served.Content.ReadAsByteArrayAsync(), "and served as the bytes it is"); + } + + var stored = await File.ReadAllTextAsync(VersionFile(fixture, lambda, 2)); + + Assert.IsFalse(stored.Contains("a page of a site"), "the version names the page rather than carrying it"); + StringAssert.Contains(stored, $"\"stored\":\"{Hash(page)}\""); + + CollectionAssert.AreEquivalent(new[] { Hash(page), Hash(logo) }, Kept(fixture, lambda), "and the store keeps what it holds, by its hash"); + + // read back as they were sent - by the editor, by an agent, in an archive + var files = (await fixture.VersionAsync(lambda, 2)).Files; + + Assert.AreEqual("

a page of a site

", files.Single(f => f.Name == "resources/web/index.html").Code); + + var picture = files.Single(f => f.Name == "resources/web/logo.png"); + + Assert.AreEqual("base64", picture.Encoding); + CollectionAssert.AreEqual(logo, Convert.FromBase64String(picture.Code)); + + using var archive = await fixture.GetAsync($"/api/v1/lambdas/{lambda.PrivateKey}/versions/2/zip"); + + using var zip = new ZipArchive(new MemoryStream(await archive.Content.ReadAsByteArrayAsync())); + + CollectionAssert.AreEqual(page, Read(zip, "resources/web/index.html")); + CollectionAssert.AreEqual(logo, Read(zip, "resources/web/logo.png")); + } + + [TestMethod] + public async Task AResourceShippedUnchangedIsKeptOnce() + { + await using var fixture = await LambdaFixture.CreateAsync(); + + var lambda = await fixture.CreateLambdaAsync(); + + var style = Encoding.UTF8.GetBytes("body { color: teal }"); + + foreach (var says in new[] { "one", "two" }) + { + using var uploaded = await UploadAsync(fixture, lambda.PrivateKey, Zip(("lambda.cs", Encoding.UTF8.GetBytes(Repository.Says(says))), + ("resources/site.css", style))); + + Assert.AreEqual(HttpStatusCode.Created, uploaded.StatusCode, await uploaded.Content.ReadAsStringAsync()); + } + + CollectionAssert.AreEqual(new[] { Hash(style) }, Kept(fixture, lambda), "however many versions ship it"); + + foreach (var version in new[] { 2, 3 }) + { + StringAssert.Contains(await File.ReadAllTextAsync(VersionFile(fixture, lambda, version)), Hash(style), $"version {version} names it"); + } + } + + [TestMethod] + public async Task AVersionSavedWithItsResourcesInsideRunsAsItDid() + { + await using var fixture = await LambdaFixture.CreateAsync(); + + var lambda = await fixture.CreateLambdaAsync("inside"); + + await fixture.SaveAsync(lambda, "Adds a page", new LambdaFile("lambda.cs", "return Resources.App(\"web\");"), + new LambdaFile("resources/web/index.html", "

a placeholder

")); + + // put back as the second layout stored a version: every resource inside it + var envelope = JsonSerializer.Serialize(new + { + version = 2, + files = new object[] + { + new { name = "lambda.cs", code = "return Resources.App(\"web\");" }, + new { name = "resources/web/index.html", code = "

kept inside

" } + } + }, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + + await File.WriteAllTextAsync(VersionFile(fixture, lambda, 2), envelope); + + var deployed = await fixture.DeployAsync(lambda.PrivateKey); + + Assert.IsTrue(deployed.Success, string.Join("; ", deployed.Diagnostics.Select(d => d.Message))); + + Assert.AreEqual("

kept inside

", await fixture.CallAsync("http://inside.localhost/"), "what it carries is what it serves"); + + using (var changed = await fixture.SendAsync(HttpMethod.Post, $"/api/v1/lambdas/{lambda.PrivateKey}/versions/changes?deploy=true", + new VersionChangeRequest(null, null, [new FileEdit("resources/web/index.html", "kept inside", "kept apart")]))) + { + Assert.AreEqual(HttpStatusCode.Created, changed.StatusCode, await changed.Content.ReadAsStringAsync()); + } + + Assert.AreEqual("

kept apart

", await fixture.CallAsync("http://inside.localhost/")); + + StringAssert.Contains(await File.ReadAllTextAsync(VersionFile(fixture, lambda, 3)), $"\"stored\":\"{Hash("

kept apart

"u8.ToArray())}\"", + "a version saved now names its resources"); + + Assert.AreEqual(envelope, await File.ReadAllTextAsync(VersionFile(fixture, lambda, 2)), "and the one saved then is left as it was"); + + // and a resource the store keeps is edited like any other: read from there, kept again + using (var edited = await fixture.SendAsync(HttpMethod.Post, $"/api/v1/lambdas/{lambda.PrivateKey}/versions/changes?deploy=true", + new VersionChangeRequest(null, null, [new FileEdit("resources/web/index.html", "kept apart", "kept apart, and edited")]))) + { + Assert.AreEqual(HttpStatusCode.Created, edited.StatusCode, await edited.Content.ReadAsStringAsync()); + } + + Assert.AreEqual("

kept apart, and edited

", await fixture.CallAsync("http://inside.localhost/")); + } + + [TestMethod] + public async Task AResourceNoVersionNamesAnyMoreIsRemovedWithTheLast() + { + await using var fixture = await LambdaFixture.CreateAsync(o => o with { MaxVersions = 2 }); + + var lambda = await fixture.CreateLambdaAsync(); + + await SaveNoteAsync(fixture, lambda, "one"); + await SaveNoteAsync(fixture, lambda, "two"); + + // kept a while ago, rather than a moment ago by a save not written yet + Age(fixture, lambda); + + // the newest two are kept, so the version that named "one" goes + await SaveNoteAsync(fixture, lambda, "three"); + + CollectionAssert.AreEquivalent(new[] { Hash("two"u8.ToArray()), Hash("three"u8.ToArray()) }, Kept(fixture, lambda), + "what a version kept still names stays, and what none names goes"); + } + + [TestMethod] + public async Task AResourceOnlyAFeatureShippedGoesWithTheFeature() + { + await using var fixture = await LambdaFixture.CreateAsync(); + + var lambda = await fixture.CreateLambdaAsync("drafty"); + + await fixture.SaveAsync(lambda, "Adds a page", new LambdaFile("lambda.cs", "return Resources.App(\"web\");"), + new LambdaFile("resources/web/index.html", "

online

")); + + var feature = await fixture.CreateFeatureAsync(lambda, "Redo"); + + using (var request = fixture.Host.GetRequest($"/api/v1/lambdas/{lambda.PrivateKey}/features/{feature.Key}/zip?deploy=true", HttpMethod.Put)) + { + request.Content = new ByteArrayContent(Zip(("lambda.cs", "return Resources.App(\"web\");"u8.ToArray()), + ("resources/web/index.html", "

in the draft

"u8.ToArray()))); + request.Content.Headers.ContentType = new("application/zip"); + + using var put = await fixture.Host.GetResponseAsync(request); + + Assert.AreEqual(HttpStatusCode.OK, put.StatusCode, await put.Content.ReadAsStringAsync()); + } + + using (var preview = await fixture.GetAsync($"/features/{feature.Key}/")) + { + Assert.AreEqual("

in the draft

", await preview.GetContentAsync(), "the preview serves what the feature keeps in the lambda's store"); + } + + CollectionAssert.AreEquivalent(new[] { Hash("

online

"u8.ToArray()), Hash("

in the draft

"u8.ToArray()) }, Kept(fixture, lambda)); + + Age(fixture, lambda); + + using (var deleted = await fixture.SendAsync(HttpMethod.Delete, $"/api/v1/lambdas/{lambda.PrivateKey}/features/{feature.Key}")) + { + Assert.AreEqual(HttpStatusCode.NoContent, deleted.StatusCode, await deleted.Content.ReadAsStringAsync()); + } + + CollectionAssert.AreEqual(new[] { Hash("

online

"u8.ToArray()) }, Kept(fixture, lambda), "what only the feature shipped goes with it"); + } + + [TestMethod] + public async Task ADemoSeededAgainIsNotSavedAgain() + { + await using var fixture = await LambdaFixture.CreateAsync(); + + await fixture.SeedDemosAsync(); + + var before = fixture.Meta.Get("demo-crud")!.LatestVersion; + + // as on every start: its version names the resources its template carries + await fixture.SeedDemosAsync(); + + Assert.AreEqual(before, fixture.Meta.Get("demo-crud")!.LatestVersion, "a demo that is what its template holds is left as it is"); + + StringAssert.Contains(await File.ReadAllTextAsync(Path.Combine(fixture.Options.CodeDirectory, fixture.Meta.GetId("demo-crud")!.ToString()!, $"v{before}.cs")), + "\"stored\":", "and keeps its resources in its store like any lambda"); + } + + #region Helpers + + private static async Task SaveNoteAsync(LambdaFixture fixture, LambdaResponse lambda, string note) + { + using var uploaded = await UploadAsync(fixture, lambda.PrivateKey, Zip(("lambda.cs", "return Resources.App(\"web\");"u8.ToArray()), + ("resources/web/note.txt", Encoding.UTF8.GetBytes(note)))); + + Assert.AreEqual(HttpStatusCode.Created, uploaded.StatusCode, await uploaded.Content.ReadAsStringAsync()); + } + + private static string StoreOf(LambdaFixture fixture, LambdaResponse lambda) + => Path.Combine(fixture.Options.CodeDirectory, fixture.Meta.GetId(lambda.PrivateKey)!.ToString()!, "resources"); + + private static string VersionFile(LambdaFixture fixture, LambdaResponse lambda, int version) + => Path.Combine(fixture.Options.CodeDirectory, fixture.Meta.GetId(lambda.PrivateKey)!.ToString()!, $"v{version}.cs"); + + /// + /// The hashes the lambda's store keeps something under. + /// + private static string[] Kept(LambdaFixture fixture, LambdaResponse lambda) + { + var store = StoreOf(fixture, lambda); + + return Directory.Exists(store) ? [.. Directory.EnumerateFiles(store, "*", SearchOption.AllDirectories).Select(f => Path.GetFileName(f)).Order(StringComparer.Ordinal)] : []; + } + + /// + /// Makes everything the store keeps two hours old, past the grace a save + /// in progress is given. + /// + private static void Age(LambdaFixture fixture, LambdaResponse lambda) + { + foreach (var file in Directory.EnumerateFiles(StoreOf(fixture, lambda), "*", SearchOption.AllDirectories)) + { + File.SetLastWriteTimeUtc(file, DateTime.UtcNow.AddHours(-2)); + } + } + + private static string Hash(byte[] content) => Convert.ToHexStringLower(SHA256.HashData(content)); + + private static byte[] Read(ZipArchive zip, string name) + { + using var entry = zip.GetEntry(name)!.Open(); + + using var target = new MemoryStream(); + + entry.CopyTo(target); + + return target.ToArray(); + } + + private static async Task UploadAsync(LambdaFixture fixture, string privateKey, byte[] archive) + { + using var request = fixture.Host.GetRequest($"/api/v1/lambdas/{privateKey}/versions/zip", HttpMethod.Post); + + request.Content = new ByteArrayContent(archive); + request.Content.Headers.ContentType = new("application/zip"); + + return await fixture.Host.GetResponseAsync(request); + } + + private static byte[] Zip(params (string Name, byte[] Content)[] files) + { + using var buffer = new MemoryStream(); + + using (var zip = new ZipArchive(buffer, ZipArchiveMode.Create, true)) + { + foreach (var (name, content) in files) + { + using var target = zip.CreateEntry(name).Open(); + + target.Write(content); + } + } + + return buffer.ToArray(); + } + + #endregion + +} diff --git a/tests/GenHTTP.Lambda.Tests/Lambdas/SourceTests.cs b/tests/GenHTTP.Lambda.Tests/Lambdas/SourceTests.cs index 9fbcedd..b7b6887 100644 --- a/tests/GenHTTP.Lambda.Tests/Lambdas/SourceTests.cs +++ b/tests/GenHTTP.Lambda.Tests/Lambdas/SourceTests.cs @@ -122,7 +122,7 @@ public void AVersionOfTheFirstLayoutIsReadInTodays() var written = LambdaSource.Serialize(files); - StringAssert.StartsWith(written, "{\"version\":2,", "written again, it is written in today's layout"); + StringAssert.StartsWith(written, "{\"version\":3,", "written again, it is written in today's layout"); Assert.IsTrue(LambdaSource.Same(stored.ReplaceLineEndings(string.Empty), written), "and is the same lambda"); }