diff --git a/docs-builder.slnx b/docs-builder.slnx index 365026c4e4..96d3e46027 100644 --- a/docs-builder.slnx +++ b/docs-builder.slnx @@ -70,6 +70,7 @@ + diff --git a/docs/cli-schema.json b/docs/cli-schema.json index 44e750515d..e81b688f2b 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -4754,7 +4754,225 @@ ] } ], - "namespaces": [] + "namespaces": [ + { + "segment": "sync", + "summary": "Sync built codex output to S3 using a two-step plan/apply workflow.", + "options": [], + "commands": [ + { + "path": [ + "codex", + "sync" + ], + "name": "apply", + "summary": "Upload the changes described in a plan file to S3.", + "notes": "Run after codex sync plan. Applies the pre-computed diff to the S3 bucket.", + "usage": "docs-builder codex sync apply \u003Cconfig\u003E --s3-bucket-name \u003Cstring\u003E --plan-file \u003Cfile\u003E", + "examples": [], + "parameters": [ + { + "role": "positional", + "name": "config", + "type": "string", + "required": true, + "summary": "Path to the codex.yml configuration file.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + }, + { + "kind": "existing" + }, + { + "kind": "fileExtensions", + "values": [ + "yml", + "yaml" + ] + } + ] + }, + { + "role": "flag", + "name": "s3-bucket-name", + "type": "string", + "required": true, + "summary": "S3 bucket to deploy to." + }, + { + "role": "flag", + "name": "plan-file", + "type": "string", + "required": true, + "summary": "Path to the plan file produced by codex sync plan.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + }, + { + "kind": "existing" + }, + { + "kind": "fileExtensions", + "values": [ + "json", + "plan" + ] + } + ] + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ], + "intent": { + "destructive": true, + "scope": "global", + "requiresAuth": true + } + }, + { + "path": [ + "codex", + "sync" + ], + "name": "plan", + "summary": "Compute a diff of what would change when deploying to S3 and write it to a plan file.", + "notes": "Two-step deployment: plan computes the diff and writes a plan file; apply executes it.\nReview the plan before applying to avoid accidental mass deletions.", + "usage": "docs-builder codex sync plan \u003Cconfig\u003E --s3-bucket-name \u003Cstring\u003E [options]", + "examples": [], + "parameters": [ + { + "role": "positional", + "name": "config", + "type": "string", + "required": true, + "summary": "Path to the codex.yml configuration file.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + }, + { + "kind": "existing" + }, + { + "kind": "fileExtensions", + "values": [ + "yml", + "yaml" + ] + } + ] + }, + { + "role": "flag", + "name": "s3-bucket-name", + "type": "string", + "required": true, + "summary": "S3 bucket to deploy to." + }, + { + "role": "flag", + "name": "out", + "type": "string", + "required": false, + "summary": "Path to write the plan file. Defaults to stdout.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + } + ] + }, + { + "role": "flag", + "name": "delete-threshold", + "type": "number", + "required": false, + "summary": "Abort if the plan would delete more than this percentage of objects (0\u2013100).", + "defaultValue": "default" + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ], + "intent": { + "scope": "global", + "requiresAuth": true + } + } + ], + "namespaces": [] + } + ] }, { "segment": "inbound-links", diff --git a/src/Elastic.Codex/CodexContext.cs b/src/Elastic.Codex/CodexContext.cs index 0f2f544d5c..327a9743af 100644 --- a/src/Elastic.Codex/CodexContext.cs +++ b/src/Elastic.Codex/CodexContext.cs @@ -5,6 +5,7 @@ using System.IO.Abstractions; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Codex; +using Elastic.Documentation.Deploying.Synchronization; using Elastic.Documentation.Diagnostics; using Nullean.ScopedFileSystem; @@ -13,7 +14,7 @@ namespace Elastic.Codex; /// /// Context for codex operations containing configuration, file systems, and directories. /// -public class CodexContext +public class CodexContext : IDocsSyncContext { public ScopedFileSystem ReadFileSystem { get; } public ScopedFileSystem WriteFileSystem { get; } @@ -27,9 +28,10 @@ public class CodexContext /// The Elasticsearch index namespace for this codex, derived from the environment name. /// Falls back to "codex" when no environment is specified. /// - public string IndexNamespace => string.IsNullOrEmpty(Configuration.Environment) - ? "codex" - : $"codex-{Configuration.Environment}"; + public string IndexNamespace => string.IsNullOrEmpty(Configuration.Environment) ? "codex" : $"codex-{EnvironmentName}"; + + /// + public string EnvironmentName { get; } public CodexContext( CodexConfiguration configuration, @@ -46,6 +48,8 @@ public CodexContext( ReadFileSystem = readFileSystem; WriteFileSystem = writeFileSystem; + EnvironmentName = string.IsNullOrEmpty(configuration.Environment) ? "codex" : configuration.Environment; + var defaultCheckoutDirectory = Path.Join(Paths.ApplicationData.FullName, "codex", "clone"); CheckoutDirectory = checkoutDirectory is null ? FileSystemFactory.AppData.DirectoryInfo.New(defaultCheckoutDirectory) diff --git a/src/Elastic.Codex/Elastic.Codex.csproj b/src/Elastic.Codex/Elastic.Codex.csproj index f6fe523b07..50a3971a5f 100644 --- a/src/Elastic.Codex/Elastic.Codex.csproj +++ b/src/Elastic.Codex/Elastic.Codex.csproj @@ -31,6 +31,7 @@ + diff --git a/src/services/Elastic.Documentation.Assembler/AssembleContext.cs b/src/services/Elastic.Documentation.Assembler/AssembleContext.cs index 29ec966254..2970240090 100644 --- a/src/services/Elastic.Documentation.Assembler/AssembleContext.cs +++ b/src/services/Elastic.Documentation.Assembler/AssembleContext.cs @@ -9,12 +9,13 @@ using Elastic.Documentation.Configuration.Products; using Elastic.Documentation.Configuration.Search; using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Deploying.Synchronization; using Elastic.Documentation.Diagnostics; using Nullean.ScopedFileSystem; namespace Elastic.Documentation.Assembler; -public class AssembleContext : IDocumentationConfigurationContext +public class AssembleContext : IDocumentationConfigurationContext, IDocsSyncContext { public ScopedFileSystem ReadFileSystem { get; } public ScopedFileSystem WriteFileSystem { get; } @@ -57,6 +58,9 @@ public class AssembleContext : IDocumentationConfigurationContext public PublishEnvironment Environment { get; } + /// + public string EnvironmentName => Environment.Name; + public AssembleContext( AssemblyConfiguration configuration, IConfigurationContext configurationContext, diff --git a/src/services/Elastic.Documentation.Assembler/Elastic.Documentation.Assembler.csproj b/src/services/Elastic.Documentation.Assembler/Elastic.Documentation.Assembler.csproj index cdfa8983db..6b1318548f 100644 --- a/src/services/Elastic.Documentation.Assembler/Elastic.Documentation.Assembler.csproj +++ b/src/services/Elastic.Documentation.Assembler/Elastic.Documentation.Assembler.csproj @@ -31,6 +31,7 @@ + diff --git a/src/services/Elastic.Documentation.Deploying/Elastic.Documentation.Deploying.csproj b/src/services/Elastic.Documentation.Deploying/Elastic.Documentation.Deploying.csproj new file mode 100644 index 0000000000..c3f6e6bd36 --- /dev/null +++ b/src/services/Elastic.Documentation.Deploying/Elastic.Documentation.Deploying.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + + + diff --git a/src/services/Elastic.Documentation.Assembler/Deploying/IncrementalDeployService.cs b/src/services/Elastic.Documentation.Deploying/IncrementalDeployService.cs similarity index 57% rename from src/services/Elastic.Documentation.Assembler/Deploying/IncrementalDeployService.cs rename to src/services/Elastic.Documentation.Deploying/IncrementalDeployService.cs index 6d3a97d0e8..a47b46f11d 100644 --- a/src/services/Elastic.Documentation.Assembler/Deploying/IncrementalDeployService.cs +++ b/src/services/Elastic.Documentation.Deploying/IncrementalDeployService.cs @@ -5,41 +5,30 @@ using Actions.Core.Services; using Amazon.S3; using Amazon.S3.Transfer; -using Elastic.Documentation.Assembler.Deploying.Synchronization; -using Elastic.Documentation.Configuration; -using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Deploying.Synchronization; using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Integrations.S3; using Elastic.Documentation.Services; using Microsoft.Extensions.Logging; -using Nullean.ScopedFileSystem; -namespace Elastic.Documentation.Assembler.Deploying; +namespace Elastic.Documentation.Deploying; public class IncrementalDeployService( ILoggerFactory logFactory, - AssemblyConfiguration assemblyConfiguration, - IConfigurationContext configurationContext, ICoreService githubActionsService, - ScopedFileSystem readFileSystem, - ScopedFileSystem writeFileSystem + IAmazonS3? s3Client = null, + ITransferUtility? transferUtility = null, + IS3EtagCalculator? etagCalculator = null ) : IService { private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly IAmazonS3 _s3 = s3Client ?? new AmazonS3Client(); - public async Task Plan(IDiagnosticsCollector collector, string environment, string s3BucketName, string @out, float? deleteThreshold, Cancel ctx) + public async Task Plan(IDiagnosticsCollector collector, IDocsSyncContext context, string s3BucketName, string @out, float? deleteThreshold, Cancel ctx) { - var assembleContext = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, readFileSystem, writeFileSystem, null, null); - var s3Client = new AmazonS3Client(); - var planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext); + var planner = new AwsS3SyncPlanStrategy(logFactory, _s3, s3BucketName, context, etagCalculator); var plan = await planner.Plan(deleteThreshold, ctx); - _logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted); - _logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count); - _logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count); - _logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count); - _logger.LogInformation("Total files to skip: {SkipCount}", plan.SkipRequests.Count); - _logger.LogInformation("Total local source files: {TotalSourceFiles}", plan.TotalSourceFiles); - _logger.LogInformation("Total remote source files: {TotalSourceFiles}", plan.TotalRemoteFiles); + LogPlanSummary(plan); var validator = new DocsSyncPlanValidator(logFactory); var validationResult = validator.Validate(plan); if (!validationResult.Valid) @@ -52,7 +41,7 @@ public async Task Plan(IDiagnosticsCollector collector, string environment if (!string.IsNullOrEmpty(@out)) { var output = SyncPlan.Serialize(plan); - await using var fileStream = writeFileSystem.File.Create(@out); + await using var fileStream = context.WriteFileSystem.File.Create(@out); await using var writer = new StreamWriter(fileStream); await writer.WriteAsync(output); _logger.LogInformation("Plan written to {OutputFile}", @out); @@ -61,34 +50,26 @@ public async Task Plan(IDiagnosticsCollector collector, string environment return collector.Errors == 0; } - public async Task Apply(IDiagnosticsCollector collector, string environment, string s3BucketName, string planFile, Cancel ctx) + public async Task Apply(IDiagnosticsCollector collector, IDocsSyncContext context, string s3BucketName, string planFile, Cancel ctx) { - var assembleContext = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, readFileSystem, writeFileSystem, null, null); - var s3Client = new AmazonS3Client(); - var transferUtility = new TransferUtility(s3Client, new TransferUtilityConfig + var xfer = transferUtility ?? new TransferUtility(_s3, new TransferUtilityConfig { ConcurrentServiceRequests = Environment.ProcessorCount * 2, MinSizeBeforePartUpload = S3EtagCalculator.PartSize }); - if (!readFileSystem.File.Exists(planFile)) + if (!context.ReadFileSystem.File.Exists(planFile)) { collector.EmitError(planFile, "Plan file does not exist."); return false; } - var planJson = await readFileSystem.File.ReadAllTextAsync(planFile, ctx); + var planJson = await context.ReadFileSystem.File.ReadAllTextAsync(planFile, ctx); var plan = SyncPlan.Deserialize(planJson); - _logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted); _logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests); - _logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count); - _logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count); - _logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count); - _logger.LogInformation("Total files to skip: {SkipCount}", plan.SkipRequests.Count); - _logger.LogInformation("Total local source files: {TotalSourceFiles}", plan.TotalSourceFiles); - _logger.LogInformation("Total remote source files: {TotalSourceFiles}", plan.TotalRemoteFiles); + LogPlanSummary(plan); if (plan.TotalSyncRequests == 0) { _logger.LogInformation("Plan has no files to sync, skipping incremental synchronization"); - return false; + return true; } var validator = new DocsSyncPlanValidator(logFactory); var validationResult = validator.Validate(plan); @@ -97,8 +78,19 @@ public async Task Apply(IDiagnosticsCollector collector, string environmen collector.EmitError(planFile, $"Plan is invalid, {validationResult}, delete ratio: {validationResult.DeleteRatio}, remote listing completed: {plan.RemoteListingCompleted}"); return false; } - var applier = new AwsS3SyncApplyStrategy(logFactory, s3Client, transferUtility, s3BucketName, assembleContext, collector); + var applier = new AwsS3SyncApplyStrategy(logFactory, _s3, xfer, s3BucketName, context, collector); await applier.Apply(plan, ctx); return true; } + + private void LogPlanSummary(SyncPlan plan) + { + _logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted); + _logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count); + _logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count); + _logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count); + _logger.LogInformation("Total files to skip: {SkipCount}", plan.SkipRequests.Count); + _logger.LogInformation("Total local source files: {TotalSourceFiles}", plan.TotalSourceFiles); + _logger.LogInformation("Total remote source files: {TotalRemoteFiles}", plan.TotalRemoteFiles); + } } diff --git a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs b/src/services/Elastic.Documentation.Deploying/Synchronization/AwsS3SyncApplyStrategy.cs similarity index 90% rename from src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs rename to src/services/Elastic.Documentation.Deploying/Synchronization/AwsS3SyncApplyStrategy.cs index d51d16edf2..54e083d452 100644 --- a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs +++ b/src/services/Elastic.Documentation.Deploying/Synchronization/AwsS3SyncApplyStrategy.cs @@ -11,14 +11,14 @@ using Elastic.Documentation.ServiceDefaults.Telemetry; using Microsoft.Extensions.Logging; -namespace Elastic.Documentation.Assembler.Deploying.Synchronization; +namespace Elastic.Documentation.Deploying.Synchronization; public partial class AwsS3SyncApplyStrategy( ILoggerFactory logFactory, IAmazonS3 s3Client, ITransferUtility transferUtility, string bucketName, - AssembleContext context, + IDocsSyncContext context, IDiagnosticsCollector collector ) : IDocsSyncApplyStrategy { @@ -137,7 +137,7 @@ public async Task Apply(SyncPlan plan, Cancel ctx = default) _logger.LogInformation( "Deployment sync: {TotalFiles} files ({AddCount} added, {UpdateCount} updated, {DeleteCount} deleted, {SkipCount} skipped) in {Environment}", - totalFiles, addCount, updateCount, deleteCount, skipCount, context.Environment.Name); + totalFiles, addCount, updateCount, deleteCount, skipCount, context.EnvironmentName); await Upload(plan, ctx); await Delete(plan, ctx); @@ -161,28 +161,7 @@ private async Task Upload(SyncPlan plan, Cancel ctx) _logger.LogInformation("Starting to process {AddCount} new files and {UpdateCount} updated files", addCount, updateCount); - // Emit file-level metrics (low cardinality) and logs for each file - foreach (var upload in uploadRequests) - { - var operation = plan.AddRequests.Contains(upload) ? "add" : "update"; - var fileSize = context.WriteFileSystem.FileInfo.New(upload.LocalPath).Length; - var extension = Path.GetExtension(upload.DestinationPath).ToLowerInvariant(); - - // Record file size distribution (histogram for p50, p95, p99 analysis) - FileSizeHistogram.Record(fileSize); - - // Record by extension (low cardinality) - if (!string.IsNullOrEmpty(extension)) - { - FilesByExtensionCounter.Add(1, - new("operation", operation), - new("extension", extension)); - } - - // Log individual file operations for detailed analysis - LogFileOperation(_logger, operation, upload.DestinationPath, fileSize); - } - + var addPaths = plan.AddRequests.Select(r => r.LocalPath).ToHashSet(); var tempDir = Path.Join(context.WriteFileSystem.Path.GetTempPath(), context.WriteFileSystem.Path.GetRandomFileName()); _ = context.WriteFileSystem.Directory.CreateDirectory(tempDir); try @@ -190,6 +169,15 @@ private async Task Upload(SyncPlan plan, Cancel ctx) _logger.LogInformation("Copying {Count} files to temp directory", uploadRequests.Count); foreach (var upload in uploadRequests) { + var operation = addPaths.Contains(upload.LocalPath) ? "add" : "update"; + var fileSize = context.WriteFileSystem.FileInfo.New(upload.LocalPath).Length; + var extension = Path.GetExtension(upload.DestinationPath).ToLowerInvariant(); + + FileSizeHistogram.Record(fileSize); + if (!string.IsNullOrEmpty(extension)) + FilesByExtensionCounter.Add(1, new("operation", operation), new("extension", extension)); + LogFileOperation(_logger, operation, upload.DestinationPath, fileSize); + var destPath = context.WriteFileSystem.Path.Join(tempDir, upload.DestinationPath); var destDirPath = context.WriteFileSystem.Path.GetDirectoryName(destPath)!; _ = context.WriteFileSystem.Directory.CreateDirectory(destDirPath); @@ -222,7 +210,7 @@ private async Task Upload(SyncPlan plan, Cancel ctx) private async Task Delete(SyncPlan plan, Cancel ctx) { var deleteCount = 0; - var deleteRequests = plan.DeleteRequests.ToList(); + var deleteRequests = plan.DeleteRequests; // Always create activity span (even if 0 files) for consistent tracing using var deleteActivity = ApplyStrategyActivitySource.StartActivity("delete files", ActivityKind.Client); @@ -272,9 +260,9 @@ private async Task Delete(SyncPlan plan, Cancel ctx) } else { - var newCount = Interlocked.Add(ref deleteCount, batch.Length); + deleteCount += batch.Length; _logger.LogInformation("Deleted {BatchCount} files ({CurrentCount}/{TotalCount})", - batch.Length, newCount, deleteRequests.Count); + batch.Length, deleteCount, deleteRequests.Count); } } diff --git a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncPlanStrategy.cs b/src/services/Elastic.Documentation.Deploying/Synchronization/AwsS3SyncPlanStrategy.cs similarity index 94% rename from src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncPlanStrategy.cs rename to src/services/Elastic.Documentation.Deploying/Synchronization/AwsS3SyncPlanStrategy.cs index 534f4af150..31a6efae03 100644 --- a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncPlanStrategy.cs +++ b/src/services/Elastic.Documentation.Deploying/Synchronization/AwsS3SyncPlanStrategy.cs @@ -8,13 +8,13 @@ using Elastic.Documentation.Integrations.S3; using Microsoft.Extensions.Logging; -namespace Elastic.Documentation.Assembler.Deploying.Synchronization; +namespace Elastic.Documentation.Deploying.Synchronization; public class AwsS3SyncPlanStrategy( ILoggerFactory logFactory, IAmazonS3 s3Client, string bucketName, - AssembleContext context, + IDocsSyncContext context, IS3EtagCalculator? calculator = null ) : IDocsSyncPlanStrategy @@ -29,10 +29,12 @@ private bool IsSymlink(string path) public async Task Plan(float? deleteThreshold, Cancel ctx = default) { - var (readToCompletion, remoteObjects) = await ListObjects(ctx); + // Start S3 listing in background while scanning local files concurrently + var listTask = ListObjects(ctx); var localObjects = context.OutputDirectory.GetFiles("*", SearchOption.AllDirectories) .Where(f => !IsSymlink(f.FullName)) .ToArray(); + var (readToCompletion, remoteObjects) = await listTask; var deleteRequests = new ConcurrentBag(); var addRequests = new ConcurrentBag(); var updateRequests = new ConcurrentBag(); diff --git a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/DocsSync.cs b/src/services/Elastic.Documentation.Deploying/Synchronization/DocsSync.cs similarity index 97% rename from src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/DocsSync.cs rename to src/services/Elastic.Documentation.Deploying/Synchronization/DocsSync.cs index b0e87d597d..eb506cc13f 100644 --- a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/DocsSync.cs +++ b/src/services/Elastic.Documentation.Deploying/Synchronization/DocsSync.cs @@ -5,13 +5,13 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace Elastic.Documentation.Assembler.Deploying.Synchronization; +namespace Elastic.Documentation.Deploying.Synchronization; public interface IDocsSyncPlanStrategy { Task Plan(float? deleteThreshold, Cancel ctx = default); - } + public record PlanValidationResult(bool Valid, float DeleteRatio, float DeleteThreshold); public interface IDocsSyncApplyStrategy diff --git a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/DocsSyncPlanValidator.cs b/src/services/Elastic.Documentation.Deploying/Synchronization/DocsSyncPlanValidator.cs similarity index 92% rename from src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/DocsSyncPlanValidator.cs rename to src/services/Elastic.Documentation.Deploying/Synchronization/DocsSyncPlanValidator.cs index d2ebb2b3d1..bc030a58e8 100644 --- a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/DocsSyncPlanValidator.cs +++ b/src/services/Elastic.Documentation.Deploying/Synchronization/DocsSyncPlanValidator.cs @@ -4,11 +4,11 @@ using Microsoft.Extensions.Logging; -namespace Elastic.Documentation.Assembler.Deploying.Synchronization; +namespace Elastic.Documentation.Deploying.Synchronization; public class DocsSyncPlanValidator(ILoggerFactory logFactory) { - private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly ILogger _logger = logFactory.CreateLogger(); public PlanValidationResult Validate(SyncPlan plan) { @@ -58,6 +58,4 @@ public PlanValidationResult Validate(SyncPlan plan) return new(true, deleteRatio, deleteThreshold); } - - } diff --git a/src/services/Elastic.Documentation.Deploying/Synchronization/IDocsSyncContext.cs b/src/services/Elastic.Documentation.Deploying/Synchronization/IDocsSyncContext.cs new file mode 100644 index 0000000000..c93e39c568 --- /dev/null +++ b/src/services/Elastic.Documentation.Deploying/Synchronization/IDocsSyncContext.cs @@ -0,0 +1,24 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Elastic.Documentation.Diagnostics; +using Nullean.ScopedFileSystem; + +namespace Elastic.Documentation.Deploying.Synchronization; + +/// +/// Minimal context required by the S3 sync strategies. +/// Implemented by both AssembleContext and CodexContext. +/// +public interface IDocsSyncContext +{ + ScopedFileSystem ReadFileSystem { get; } + ScopedFileSystem WriteFileSystem { get; } + IDirectoryInfo OutputDirectory { get; } + IDiagnosticsCollector Collector { get; } + + /// Deployment environment name, used only for log messages. + string EnvironmentName { get; } +} diff --git a/src/tooling/docs-builder/Commands/Assembler/DeployCommands.cs b/src/tooling/docs-builder/Commands/Assembler/DeployCommands.cs index b6a18a787b..f5eb9a245a 100644 --- a/src/tooling/docs-builder/Commands/Assembler/DeployCommands.cs +++ b/src/tooling/docs-builder/Commands/Assembler/DeployCommands.cs @@ -5,9 +5,11 @@ using System.ComponentModel.DataAnnotations; using Actions.Core.Services; using Elastic.Documentation; +using Elastic.Documentation.Assembler; using Elastic.Documentation.Assembler.Deploying; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Deploying; using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Services; using Microsoft.Extensions.Logging; @@ -41,9 +43,10 @@ public async Task Plan(string environment, string s3BucketName, [ExpandUser { await using var serviceInvoker = new ServiceInvoker(collector); - var service = new IncrementalDeployService(logFactory, assemblyConfiguration, configurationContext, githubActionsService, FileSystemFactory.RealRead, FileSystemFactory.RealWrite); - serviceInvoker.AddCommand(service, (environment, s3BucketName, @out, deleteThreshold), - static async (s, collector, state, ctx) => await s.Plan(collector, state.environment, state.s3BucketName, state.@out?.FullName ?? "", state.deleteThreshold, ctx) + var context = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, FileSystemFactory.RealRead, FileSystemFactory.RealWrite, null, null); + var service = new IncrementalDeployService(logFactory, githubActionsService); + serviceInvoker.AddCommand(service, (context, s3BucketName, @out, deleteThreshold), + static async (s, collector, state, ctx) => await s.Plan(collector, state.context, state.s3BucketName, state.@out?.FullName ?? "", state.deleteThreshold, ctx) ); return await serviceInvoker.InvokeAsync(ct); } @@ -61,9 +64,10 @@ public async Task Apply(string environment, string s3BucketName, [Existing, { await using var serviceInvoker = new ServiceInvoker(collector); - var service = new IncrementalDeployService(logFactory, assemblyConfiguration, configurationContext, githubActionsService, FileSystemFactory.RealRead, FileSystemFactory.RealWrite); - serviceInvoker.AddCommand(service, (environment, s3BucketName, planFile), - static async (s, collector, state, ctx) => await s.Apply(collector, state.environment, state.s3BucketName, state.planFile.FullName, ctx) + var context = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, FileSystemFactory.RealRead, FileSystemFactory.RealWrite, null, null); + var service = new IncrementalDeployService(logFactory, githubActionsService); + serviceInvoker.AddCommand(service, (context, s3BucketName, planFile), + static async (s, collector, state, ctx) => await s.Apply(collector, state.context, state.s3BucketName, state.planFile.FullName, ctx) ); return await serviceInvoker.InvokeAsync(ct); } diff --git a/src/tooling/docs-builder/Commands/Codex/CodexSyncCommand.cs b/src/tooling/docs-builder/Commands/Codex/CodexSyncCommand.cs new file mode 100644 index 0000000000..1148dcf7d7 --- /dev/null +++ b/src/tooling/docs-builder/Commands/Codex/CodexSyncCommand.cs @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.ComponentModel.DataAnnotations; +using Actions.Core.Services; +using Elastic.Codex; +using Elastic.Documentation; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Codex; +using Elastic.Documentation.Deploying; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; +using Nullean.Argh; +using Nullean.Argh.Documentation; + +namespace Documentation.Builder.Commands.Codex; + +/// Sync built codex output to S3 using a two-step plan/apply workflow. +internal sealed class CodexSyncCommand( + IDiagnosticsCollector collector, + ILoggerFactory logFactory, + ICoreService githubActionsService +) +{ + /// Compute a diff of what would change when deploying to S3 and write it to a plan file. + /// + /// Two-step deployment: plan computes the diff and writes a plan file; apply executes it. + /// Review the plan before applying to avoid accidental mass deletions. + /// + /// Path to the codex.yml configuration file. + /// S3 bucket to deploy to. + /// Path to write the plan file. Defaults to stdout. + /// Abort if the plan would delete more than this percentage of objects (0–100). + [RequiresAuth] + [MutationScope(MutationScope.Global)] + [NoOptionsInjection] + public async Task Plan( + GlobalCliOptions _, + [Argument, Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "yml,yaml")] FileInfo config, + string s3BucketName, + [ExpandUserProfile, RejectSymbolicLinks] FileInfo? @out = null, + float? deleteThreshold = null, + CancellationToken ct = default) + { + await using var serviceInvoker = new ServiceInvoker(collector); + var (context, service) = LoadContext(config); + serviceInvoker.AddCommand(service, (context, s3BucketName, @out, deleteThreshold), + static async (s, collector, state, ctx) => await s.Plan(collector, state.context, state.s3BucketName, state.@out?.FullName ?? "", state.deleteThreshold, ctx) + ); + return await serviceInvoker.InvokeAsync(ct); + } + + /// Upload the changes described in a plan file to S3. + /// Run after codex sync plan. Applies the pre-computed diff to the S3 bucket. + /// Path to the codex.yml configuration file. + /// S3 bucket to deploy to. + /// Path to the plan file produced by codex sync plan. + [RequiresAuth] + [CommandIntent(Intent.Destructive)] + [MutationScope(MutationScope.Global)] + [NoOptionsInjection] + public async Task Apply( + GlobalCliOptions _, + [Argument, Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "yml,yaml")] FileInfo config, + string s3BucketName, + [Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "json,plan")] FileInfo planFile, + CancellationToken ct = default) + { + await using var serviceInvoker = new ServiceInvoker(collector); + var (context, service) = LoadContext(config); + serviceInvoker.AddCommand(service, (context, s3BucketName, planFile), + static async (s, collector, state, ctx) => await s.Apply(collector, state.context, state.s3BucketName, state.planFile.FullName, ctx) + ); + return await serviceInvoker.InvokeAsync(ct); + } + + private (CodexContext context, IncrementalDeployService service) LoadContext(FileInfo config) + { + var fs = FileSystemFactory.RealRead; + var configFile = fs.FileInfo.New(config.FullName); + var codexConfig = CodexConfiguration.Load(configFile); + return (new CodexContext(codexConfig, configFile, collector, fs, FileSystemFactory.RealWrite, null, null), + new IncrementalDeployService(logFactory, githubActionsService)); + } +} diff --git a/src/tooling/docs-builder/Program.cs b/src/tooling/docs-builder/Program.cs index 51f6d651e3..07ce3e74bd 100644 --- a/src/tooling/docs-builder/Program.cs +++ b/src/tooling/docs-builder/Program.cs @@ -71,6 +71,7 @@ { _ = g.Map(); _ = g.Map(); + _ = g.MapNamespace("sync"); }); }); diff --git a/tests-integration/Elastic.Documentation.IntegrationTests/DocsSyncTests.cs b/tests-integration/Elastic.Documentation.IntegrationTests/DocsSyncTests.cs index d88c7e6b53..5b59cc271a 100644 --- a/tests-integration/Elastic.Documentation.IntegrationTests/DocsSyncTests.cs +++ b/tests-integration/Elastic.Documentation.IntegrationTests/DocsSyncTests.cs @@ -9,9 +9,9 @@ using Amazon.S3.Transfer; using AwesomeAssertions; using Elastic.Documentation.Assembler; -using Elastic.Documentation.Assembler.Deploying.Synchronization; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Deploying.Synchronization; using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Integrations.S3; using Elastic.Documentation.ServiceDefaults.Telemetry; diff --git a/tests-integration/Elastic.Documentation.IntegrationTests/IncrementalDeployRoundTripTests.cs b/tests-integration/Elastic.Documentation.IntegrationTests/IncrementalDeployRoundTripTests.cs new file mode 100644 index 0000000000..a3cebde853 --- /dev/null +++ b/tests-integration/Elastic.Documentation.IntegrationTests/IncrementalDeployRoundTripTests.cs @@ -0,0 +1,178 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions.TestingHelpers; +using Actions.Core.Services; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using AwesomeAssertions; +using Elastic.Codex; +using Elastic.Documentation.Assembler; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Configuration.Codex; +using Elastic.Documentation.Deploying; +using Elastic.Documentation.Deploying.Synchronization; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Integrations.S3; +using FakeItEasy; +using Microsoft.Extensions.Logging; +using Nullean.ScopedFileSystem; + +namespace Elastic.Documentation.IntegrationTests; + +/// +/// End-to-end round-trip tests that drive — plan writes +/// a JSON plan file to a mock filesystem, apply reads it back and syncs against a mocked S3 — +/// for both the assembler and codex contexts. +/// +/// +/// File mix for both tests: 3 adds, 1 update (stale ETag), 1 skip (ETag matches the mocked +/// calculator), 1 remote-only delete. A mocked returns a fixed +/// ETag per file so the skip decision is deterministic regardless of mock-FS internals. +/// +public class IncrementalDeployRoundTripTests +{ + // Fixed ETags returned by the mocked calculator + private const string SkipETag = "aaaa0000skip0000etag0000aaaa0000"; + private const string AnyOtherETag = "bbbb1111other1111etag1111bbbb1111"; + + [Fact] + public async Task AssemblerRoundTrip() + { + var outputDir = Path.Join(Paths.WorkingDirectoryRoot.FullName, ".artifacts", "assembly"); + var (fs, s3, xfer, gh, svc) = Arrange(outputDir); + var configurationContext = TestHelpers.CreateConfigurationContext(fs); + var config = AssemblyConfiguration.Create(configurationContext.ConfigurationFileProvider); + var collector = new DiagnosticsCollector([]); + var scopedFs = FileSystemFactory.ScopeCurrentWorkingDirectory(fs); + var scopedWriteFs = FileSystemFactory.ScopeCurrentWorkingDirectoryForWrite(fs); + var context = new AssembleContext(config, configurationContext, "dev", collector, scopedFs, scopedWriteFs, null, outputDir); + + await RunRoundTrip(fs, s3, xfer, gh, svc, context, outputDir); + } + + [Fact] + public async Task CodexRoundTrip() + { + var outputDir = Path.Join(Paths.WorkingDirectoryRoot.FullName, ".artifacts", "codex", "docs"); + var (fs, s3, xfer, gh, svc) = Arrange(outputDir); + var collector = new DiagnosticsCollector([]); + var scopedFs = FileSystemFactory.ScopeCurrentWorkingDirectory(fs); + var scopedWriteFs = FileSystemFactory.ScopeCurrentWorkingDirectoryForWrite(fs); + // CodexContext only stores configurationPath — it never reads from it — + // so we can point to any path without adding it to the mock FS. + var codexConfig = new CodexConfiguration { Environment = "dev" }; + var configFile = fs.FileInfo.New(Path.Join(Paths.WorkingDirectoryRoot.FullName, "codex.yml")); + var context = new CodexContext(codexConfig, configFile, collector, scopedFs, scopedWriteFs, null, outputDir); + + await RunRoundTrip(fs, s3, xfer, gh, svc, context, outputDir); + } + + /// + /// Shared arrange: a mock filesystem with local docs, a mocked ETag calculator, stubbed S3, + /// and the deploy service. + /// + /// + /// The mocked returns for + /// skip.md and for everything else. The S3 listing mirrors + /// this so that skip.md matches (→ skip), update.md does not (→ update), and + /// delete.md has no local counterpart (→ delete). The three add*.md files + /// have no remote entry (→ add). This exercises every sync category in a single round-trip. + /// + /// The delete ratio (1/6 ≈ 17 %) is below the enforced 0.8 floor for small sync sets, so + /// deleteThreshold: 1.0f is passed to allow any deletion ratio. + /// + private static (MockFileSystem fs, IAmazonS3 s3, ITransferUtility xfer, ICoreService gh, IncrementalDeployService svc) + Arrange(string outputDir) + { + var fs = new MockFileSystem(new Dictionary + { + { Path.Join(outputDir, "docs/add1.md"), new MockFileData("# New Document 1") }, + { Path.Join(outputDir, "docs/add2.md"), new MockFileData("# New Document 2") }, + { Path.Join(outputDir, "docs/add3.md"), new MockFileData("# New Document 3") }, + { Path.Join(outputDir, "docs/skip.md"), new MockFileData("# Skipped Document") }, + { Path.Join(outputDir, "docs/update.md"), new MockFileData("# Existing Document") }, + }, new MockFileSystemOptions { CurrentDirectory = outputDir }); + + var s3 = A.Fake(); + var xfer = A.Fake(); + var gh = A.Fake(); + + // Mocked ETag calculator: skip.md returns SkipETag (matches remote → skip); + // all other files return AnyOtherETag (remote has "stale-etag" → update). + var etagCalculator = A.Fake(); + A.CallTo(() => etagCalculator.CalculateS3ETag(A.That.EndsWith("skip.md"), A._)) + .Returns(SkipETag); + A.CallTo(() => etagCalculator.CalculateS3ETag(A.That.Not.EndsWith("skip.md"), A._)) + .Returns(AnyOtherETag); + + A.CallTo(() => s3.ListObjectsV2Async(A._, A._)) + .Returns(new ListObjectsV2Response + { + S3Objects = + [ + new S3Object { Key = "docs/delete.md" }, + new S3Object { Key = "docs/skip.md", ETag = $"\"{SkipETag}\"" }, + new S3Object { Key = "docs/update.md", ETag = "\"stale-etag\"" }, + ] + }); + + A.CallTo(() => s3.DeleteObjectsAsync(A._, A._)) + .Returns(new DeleteObjectsResponse { HttpStatusCode = System.Net.HttpStatusCode.OK }); + + var svc = new IncrementalDeployService(new LoggerFactory(), gh, s3, xfer, etagCalculator); + return (fs, s3, xfer, gh, svc); + } + + private static async Task RunRoundTrip( + MockFileSystem fs, + IAmazonS3 s3, + ITransferUtility xfer, + ICoreService gh, + IncrementalDeployService svc, + IDocsSyncContext context, + string outputDir) + { + // Capture the files passed to the upload call + var transferredFiles = Array.Empty(); + A.CallTo(() => xfer.UploadDirectoryAsync(A._, A._)) + .Invokes((TransferUtilityUploadDirectoryRequest request, Cancel _) => + { + transferredFiles = fs.Directory.GetFiles(request.Directory, request.SearchPattern, request.SearchOption); + }); + + var planPath = Path.Join(outputDir, "sync-plan.json"); + + // Act — Plan + // deleteThreshold: 1.0 permits any delete ratio (needed because the validator + // enforces a 0.8 floor for small sync sets where TotalSyncRequests < 100) + var planOk = await svc.Plan(context.Collector, context, "fake-bucket", planPath, deleteThreshold: 1.0f, Cancel.None); + planOk.Should().BeTrue("plan should succeed with valid file mix"); + fs.File.Exists(planPath).Should().BeTrue("plan JSON must be written to the mock filesystem"); + + // Act — Apply (reads plan.json from the same mock filesystem) + var applyOk = await svc.Apply(context.Collector, context, "fake-bucket", planPath, Cancel.None); + applyOk.Should().BeTrue("apply should succeed"); + + // Assert — GitHub Actions output + A.CallTo(() => gh.SetOutputAsync("plan-valid", "true")).MustHaveHappenedOnceExactly(); + + // Assert — uploads: 3 adds + 1 update; skip.md and remote-only delete.md not uploaded + transferredFiles.Select(Path.GetFileName).Should() + .BeEquivalentTo(["add1.md", "add2.md", "add3.md", "update.md"], + "skip.md is unchanged (ETag matches) so it is not re-uploaded"); + + // Assert — deletes: exactly one S3 delete call for docs/delete.md + A.CallTo(() => s3.DeleteObjectsAsync( + A.That.Matches(r => r.Objects.Any(o => o.Key == "docs/delete.md")), + A._)) + .MustHaveHappenedOnceExactly(); + + // Assert — uploads called once + A.CallTo(() => xfer.UploadDirectoryAsync(A._, A._)) + .MustHaveHappenedOnceExactly(); + } +}