diff --git a/src/Netclaw.Actors.Tests/Skills/SkillInventoryRefresherTests.cs b/src/Netclaw.Actors.Tests/Skills/SkillInventoryRefresherTests.cs index 70581c32d..64e58744a 100644 --- a/src/Netclaw.Actors.Tests/Skills/SkillInventoryRefresherTests.cs +++ b/src/Netclaw.Actors.Tests/Skills/SkillInventoryRefresherTests.cs @@ -72,6 +72,32 @@ public void Refresh_preserves_all_sources_and_applies_canonical_precedence() Assert.Contains(result.AcceptedSkills, skill => skill.Name == "new-native"); } + [Fact] + public void Managed_git_plugins_stay_between_server_feeds_and_external_sources() + { + var feedRoot = _paths.ServerFeedDirectory("managed"); + var gitRoot = Path.Join(_home, "git-plugin"); + var externalRoot = Path.Join(_home, "external"); + WriteSkill(_paths.SkillsDirectory, "native-only", "native"); + WriteSkill(feedRoot, "shared", "feed wins"); + WriteSkill(gitRoot, "shared", "git loses"); + WriteSkill(gitRoot, "git-only", "managed git"); + WriteSkill(externalRoot, "shared", "external loses"); + WriteSkill(externalRoot, "external-only", "external"); + + var refresher = CreateRefresher( + new SkillFeedsConfig { Feeds = [new SkillFeedSource { Name = "managed" }] }, + [new ResolvedExternalSource("external", [externalRoot], AllowSymlinks: false)]); + + refresher.ReplaceManagedGitPluginSourcesAndRefresh( + [new ResolvedExternalSource("managed-git:fixture", [gitRoot], AllowSymlinks: false)]); + var result = refresher.Refresh(); + + Assert.Equal("feed wins", _registry.GetByName("shared")!.Description); + Assert.Contains(result.AcceptedSkills, skill => skill.Name == "git-only"); + Assert.Contains(result.AcceptedSkills, skill => skill.Name == "external-only"); + } + [Fact] public void ReplaceAll_never_exposes_a_partially_replaced_inventory() { diff --git a/src/Netclaw.Actors/Skills/SkillInventoryRefresher.cs b/src/Netclaw.Actors/Skills/SkillInventoryRefresher.cs index eba30b808..cb27a7de0 100644 --- a/src/Netclaw.Actors/Skills/SkillInventoryRefresher.cs +++ b/src/Netclaw.Actors/Skills/SkillInventoryRefresher.cs @@ -19,6 +19,7 @@ public sealed class SkillInventoryRefresher private readonly IReadOnlyList _externalSources; private readonly SkillRegistry _registry; private readonly SkillIndexPublisher _indexPublisher; + private IReadOnlyList _managedGitPluginSources = []; public SkillInventoryRefresher( NetclawPaths paths, @@ -38,17 +39,38 @@ public MergedSkillScanResult Refresh() { lock (_refreshLock) { - var result = SkillScanner.ScanAndMerge( - _paths.SkillsDirectory, - ResolveServerFeedSources(), - _externalSources); + return RefreshCore(); + } + } - _registry.ReplaceAll(result.AcceptedSkills, result.Issues); - _indexPublisher.Publish(); - return result; + /// + /// Replaces the managed Git plugin paths and publishes one inventory snapshot. + /// + public MergedSkillScanResult ReplaceManagedGitPluginSourcesAndRefresh( + IReadOnlyList managedGitPluginSources) + { + ArgumentNullException.ThrowIfNull(managedGitPluginSources); + + lock (_refreshLock) + { + _managedGitPluginSources = managedGitPluginSources; + return RefreshCore(); } } + private MergedSkillScanResult RefreshCore() + { + var result = SkillScanner.ScanAndMerge( + _paths.SkillsDirectory, + ResolveServerFeedSources(), + _managedGitPluginSources, + _externalSources); + + _registry.ReplaceAll(result.AcceptedSkills, result.Issues); + _indexPublisher.Publish(); + return result; + } + private IReadOnlyList ResolveServerFeedSources() { var sources = new List(); diff --git a/src/Netclaw.Actors/Skills/SkillScanner.cs b/src/Netclaw.Actors/Skills/SkillScanner.cs index 3fa5fc3fd..e17492ef1 100644 --- a/src/Netclaw.Actors/Skills/SkillScanner.cs +++ b/src/Netclaw.Actors/Skills/SkillScanner.cs @@ -208,6 +208,17 @@ public static MergedSkillScanResult ScanAndMerge( string nativeSkillsDirectory, IReadOnlyList serverFeedSources, IReadOnlyList externalSources) + => ScanAndMerge(nativeSkillsDirectory, serverFeedSources, Array.Empty(), externalSources); + + /// + /// Scans all configured skill tiers. Managed Git plugins are lower priority than + /// organization feeds and higher priority than local external directories. + /// + public static MergedSkillScanResult ScanAndMerge( + string nativeSkillsDirectory, + IReadOnlyList serverFeedSources, + IReadOnlyList managedGitPluginSources, + IReadOnlyList externalSources) { var nativeScan = Scan(nativeSkillsDirectory, allowSymlinks: false, strictNameMatch: true); var allAccepted = new List(nativeScan.AcceptedSkills); @@ -219,7 +230,11 @@ public static MergedSkillScanResult ScanAndMerge( // Server feed sources (second tier — org-managed private skill servers) MergeSources(serverFeedSources, allAccepted, allIssues, knownNames, allowFrontmatterlessFlatFiles: false); - // External filesystem sources (third tier — Claude Code, Open Code, custom paths) + // Managed Git plugins are verified, immutable third-party content. They yield + // to organization feeds but take priority over locally discovered sources. + MergeSources(managedGitPluginSources, allAccepted, allIssues, knownNames, allowFrontmatterlessFlatFiles: false); + + // External filesystem sources are the lowest-precedence tier. MergeSources(externalSources, allAccepted, allIssues, knownNames, allowFrontmatterlessFlatFiles: true); return new MergedSkillScanResult(allAccepted, allIssues); diff --git a/src/Netclaw.Actors/Tools/SkillManageTool.cs b/src/Netclaw.Actors/Tools/SkillManageTool.cs index f45a88373..04f46e7c7 100644 --- a/src/Netclaw.Actors/Tools/SkillManageTool.cs +++ b/src/Netclaw.Actors/Tools/SkillManageTool.cs @@ -449,6 +449,8 @@ private bool IsSystemSkill(string name) return $"Cannot {verb} system skills. System skills are read-only."; if (IsServerFeedSkill(skill)) return $"Cannot {verb} server feed skills. Server feed skill directories are read-only."; + if (IsManagedGitPluginSkill(skill)) + return $"Cannot {verb} managed Git plugin skills. Managed Git plugin directories are read-only."; if (IsExternalSkill(skill)) return $"Cannot {verb} external skills. External skill directories are read-only."; return null; @@ -471,6 +473,13 @@ private bool IsExternalSkill(SkillEntry skill) return !PathUtility.IsWithinRoot(skillPath, nativeRoot); } + private bool IsManagedGitPluginSkill(SkillEntry skill) + { + var managedRoot = PathUtility.Normalize(_paths.ManagedGitSkillsDirectory); + var skillPath = PathUtility.Normalize(Path.GetDirectoryName(skill.FilePath)!); + return PathUtility.IsWithinRoot(skillPath, managedRoot); + } + private static void AtomicWrite(string path, string content) { var tempPath = path + ".tmp"; diff --git a/src/Netclaw.Configuration/SkillSyncResult.cs b/src/Netclaw.Configuration/SkillSyncResult.cs index 144dabdd6..bef371f1a 100644 --- a/src/Netclaw.Configuration/SkillSyncResult.cs +++ b/src/Netclaw.Configuration/SkillSyncResult.cs @@ -11,6 +11,9 @@ namespace Netclaw.Configuration; /// public static class SkillSyncResult { + public const string ServerFeedSourceKind = "server-feed"; + public const string GitPluginSourceKind = "git-plugin"; + /// Result data for one pass. public sealed class Response : IWireType { @@ -26,6 +29,9 @@ public sealed class SourceRow : IWireType { public required string Name { get; init; } + /// The source type that owns this result. + public string SourceKind { get; init; } = ServerFeedSourceKind; + public int ChangedCount { get; init; } public int UnchangedCount { get; init; } @@ -45,6 +51,9 @@ public sealed class SourceRow : IWireType /// The declared plugin version, when available. public string? Version { get; init; } + + /// Safe package diagnostics that do not stop this source. + public IReadOnlyList Notices { get; init; } = []; } /// Result data for the inventory publication owned by this pass. diff --git a/src/Netclaw.Daemon.IntegrationTests/SkillServerNativeSidecarIntegrationTests.cs b/src/Netclaw.Daemon.IntegrationTests/SkillServerNativeSidecarIntegrationTests.cs index 6ba1b377b..49b26b034 100644 --- a/src/Netclaw.Daemon.IntegrationTests/SkillServerNativeSidecarIntegrationTests.cs +++ b/src/Netclaw.Daemon.IntegrationTests/SkillServerNativeSidecarIntegrationTests.cs @@ -116,7 +116,15 @@ public async Task Syncs_skill_and_subagent_from_real_skillserver_container() TimeProvider.System, new NoOpSkillContentScanner(), NullLogger.Instance, - []); + [], + feed => new SkillServerClient(new HttpClient(new HttpClientHandler()) + { + BaseAddress = new Uri(feed.Url), + }), + CreatePluginStateStore(paths), + new GitSkillPluginAcquirer( + new HttpClient(new HttpClientHandler()), paths, TimeProvider.System, new NoOpSkillContentScanner()), + NullNotificationSink.Instance); await service.SyncAsync(CancellationToken.None); @@ -139,6 +147,13 @@ public async Task Syncs_skill_and_subagent_from_real_skillserver_container() Assert.NotNull(registry.TryGetByName("code-reviewer")); } + private static ManagedPluginStateStore CreatePluginStateStore(NetclawPaths paths) + { + new SchemaMigrator(paths, NullLogger.Instance) + .MigrateAsync(paths.SqliteDbPath, CancellationToken.None).GetAwaiter().GetResult(); + return new ManagedPluginStateStore(paths, TimeProvider.System); + } + private static async Task SeedSkillServerAsync() { using var client = new SkillServerClient(ServerUrl, ApiKey); diff --git a/src/Netclaw.Daemon.Tests/Configuration/DaemonToolPathPolicyFactoryTests.cs b/src/Netclaw.Daemon.Tests/Configuration/DaemonToolPathPolicyFactoryTests.cs index 37f2ef329..f39d31b90 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/DaemonToolPathPolicyFactoryTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/DaemonToolPathPolicyFactoryTests.cs @@ -70,6 +70,20 @@ public void System_skills_are_readable_but_not_writable(ShellPlatform platform) Assert.True(policy.IsDenied(skillPath)); } + [Fact] + public void Managed_git_plugin_skills_are_readable_but_not_writable_or_shell_accessible() + { + var paths = new NetclawPaths(Path.Combine(Path.GetTempPath(), "netclaw-policy-contract")); + var policy = DaemonToolPathPolicyFactory.Create( + paths, + ShellExecutionEnvironment.CreateBash(ShellPlatform.Linux)); + var skillPath = Path.Combine(paths.ManagedGitSkillsDirectory, "fixture", "commit", "SKILL.md"); + + Assert.False(policy.IsReadDenied(skillPath)); + Assert.True(policy.IsDenied(skillPath)); + Assert.True(policy.CommandReferencesDeniedPath($"cat '{skillPath}'")); + } + [Theory] [InlineData("tool-index.md")] [InlineData("mcp/synthetic-server.md")] diff --git a/src/Netclaw.Daemon.Tests/Services/GitSkillPluginSyncServiceTests.cs b/src/Netclaw.Daemon.Tests/Services/GitSkillPluginSyncServiceTests.cs new file mode 100644 index 000000000..82697fea8 --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Services/GitSkillPluginSyncServiceTests.cs @@ -0,0 +1,710 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Netclaw.Actors.Skills; +using Netclaw.Configuration; +using Netclaw.Daemon.Services; +using Netclaw.Security.Skills; +using Netclaw.Tests.Utilities; +using Xunit; + +namespace Netclaw.Daemon.Tests.Services; + +public sealed class GitSkillPluginSyncServiceTests : IDisposable +{ + private const string FirstCommit = "13e26d39ed01d97ea592235d041304d289f4ba07"; + private const string SecondCommit = "23e26d39ed01d97ea592235d041304d289f4ba08"; + private readonly DisposableTempDir _temp = new(); + private readonly NetclawPaths _paths; + private readonly FakeTimeProvider _time = new(); + + public GitSkillPluginSyncServiceTests() + { + _paths = new NetclawPaths(_temp.Path); + _paths.EnsureDirectoriesExist(); + } + + public void Dispose() + { + SqliteTestPools.Clear(_paths); + _temp.Dispose(); + } + + [Fact] + public async Task First_install_publishes_all_skills_in_one_inventory_snapshot() + { + var source = Source(); + var acquirer = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill"); + var registry = new SkillRegistry(); + var publicationCount = 0; + var refresher = CreateRefresher(registry, () => publicationCount++); + var service = await CreateServiceAsync(source, refresher, acquirer, new RecordingSink()); + + var result = await service.SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(2, publicationCount); + var plugin = Assert.Single(result.Sources); + Assert.Equal(SkillSyncResult.GitPluginSourceKind, plugin.SourceKind); + Assert.Equal(FirstCommit, plugin.Commit); + Assert.NotNull(registry.GetByName("plugin-skill")); + } + + [Fact] + public async Task Branch_update_keeps_prior_directory_and_publishes_new_files() + { + var source = Source(); + var registry = new SkillRegistry(); + var refresher = CreateRefresher(registry, static () => { }); + var first = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill", "old text"); + var service = await CreateServiceAsync(source, refresher, first, new RecordingSink()); + await service.SyncAsync(TestContext.Current.CancellationToken); + var oldPath = registry.GetByName("plugin-skill")!.FilePath; + + var second = new FakeAcquirer(_paths, source, SecondCommit, "2.0.0", "plugin-skill", "new text"); + var nextService = await CreateServiceAsync(source, refresher, second, new RecordingSink()); + await nextService.SyncAsync(TestContext.Current.CancellationToken); + + Assert.True(File.Exists(oldPath)); + Assert.Contains("old text", await File.ReadAllTextAsync(oldPath, TestContext.Current.CancellationToken)); + Assert.Contains("new text", await File.ReadAllTextAsync( + registry.GetByName("plugin-skill")!.FilePath, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task Interleaved_refresh_keeps_the_prior_snapshot_until_receipt_publication() + { + var source = Source(); + var registry = new SkillRegistry(); + var refresher = CreateRefresher(registry, static () => { }); + var first = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill", "old text"); + await (await CreateServiceAsync(source, refresher, first, new RecordingSink())) + .SyncAsync(TestContext.Current.CancellationToken); + var oldPath = registry.GetByName("plugin-skill")!.FilePath; + string? pathDuringInterleave = null; + string? contentDuringInterleave = null; + + var second = new FakeAcquirer(_paths, source, SecondCommit, "2.0.0", "plugin-skill", "new text") + { + BeforeReturn = () => + { + refresher.Refresh(); + pathDuringInterleave = registry.GetByName("plugin-skill")!.FilePath; + contentDuringInterleave = File.ReadAllText(pathDuringInterleave); + }, + }; + + await (await CreateServiceAsync(source, refresher, second, new RecordingSink())) + .SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(oldPath, pathDuringInterleave); + Assert.Contains("old text", contentDuringInterleave); + Assert.Contains("new text", await File.ReadAllTextAsync( + registry.GetByName("plugin-skill")!.FilePath, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task Receipt_persistence_failure_keeps_the_prior_receipt_and_registry() + { + var source = Source(); + var registry = new SkillRegistry(); + var refresher = CreateRefresher(registry, static () => { }); + var first = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill", "old text"); + await (await CreateServiceAsync(source, refresher, first, new RecordingSink())) + .SyncAsync(TestContext.Current.CancellationToken); + var oldPath = registry.GetByName("plugin-skill")!.FilePath; + + await AddReceiptFailureTriggerAsync(); + var second = new FakeAcquirer(_paths, source, SecondCommit, "2.0.0", "plugin-skill", "new text"); + var result = await (await CreateServiceAsync(source, refresher, second, new RecordingSink())) + .SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, Assert.Single(result.Sources).FailedCount); + var receipt = await new ManagedPluginStateStore(_paths, _time).GetReceiptAsync( + source.Id, TestContext.Current.CancellationToken); + Assert.NotNull(receipt); + Assert.Equal(FirstCommit, receipt.InstalledCommit); + Assert.Equal(oldPath, registry.GetByName("plugin-skill")!.FilePath); + Assert.Contains("old text", await File.ReadAllTextAsync(oldPath, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task Equal_version_records_last_observed_commit_without_a_second_fetch() + { + var source = Source(); + var registry = new SkillRegistry(); + var refresher = CreateRefresher(registry, static () => { }); + var first = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill"); + var service = await CreateServiceAsync(source, refresher, first, new RecordingSink()); + await service.SyncAsync(TestContext.Current.CancellationToken); + + var next = new FakeAcquirer(_paths, source, SecondCommit, "1.0.0", "plugin-skill"); + var nextService = await CreateServiceAsync(source, refresher, next, new RecordingSink()); + await nextService.SyncAsync(TestContext.Current.CancellationToken); + await nextService.SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, next.AcquireCount); + var receipt = await new ManagedPluginStateStore(_paths, _time).GetReceiptAsync( + source.Id, TestContext.Current.CancellationToken); + Assert.NotNull(receipt); + Assert.Equal(FirstCommit, receipt.InstalledCommit); + Assert.Equal(SecondCommit, receipt.LastObservedCommit); + Assert.False(Directory.Exists(_paths.ManagedGitSkillCommitDirectory( + source.Id, ManagedPluginSourceValidator.Fingerprint(source), SecondCommit))); + } + + [Fact] + public async Task Commit_pin_does_not_resolve_and_known_rejection_survives_a_new_service() + { + var source = Source(); + source.ReferenceKind = ManagedPluginReferenceKind.Commit; + source.Reference = FirstCommit.ToUpperInvariant(); + var registry = new SkillRegistry(); + var refresher = CreateRefresher(registry, static () => { }); + var store = await CreateStoreAsync(); + await store.SaveRejectionAsync( + source.Id, + ManagedPluginSourceValidator.Fingerprint(source), + FirstCommit, + "unsafe plugin", + false, + TestContext.Current.CancellationToken); + var acquirer = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill") + { + FailOnResolve = true, + }; + var service = new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [source] }, + _paths, + refresher, + _time, + new NoOpSkillContentScanner(), + NullLogger.Instance, + store, + acquirer, + new RecordingSink()); + + var result = await service.SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(0, acquirer.ResolveCount); + Assert.Equal(0, acquirer.AcquireCount); + Assert.Equal(1, Assert.Single(result.Sources).RejectedCount); + } + + [Fact] + public async Task Security_rejection_persists_and_emits_one_claimed_alert() + { + var source = Source(); + var registry = new SkillRegistry(); + var refresher = CreateRefresher(registry, static () => { }); + var acquirer = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill") + { + Rejection = new GitSkillPluginRejectedException(FirstCommit, "scanner result", true), + }; + var alerts = new RecordingSink(); + var service = await CreateServiceAsync(source, refresher, acquirer, alerts); + + await service.SyncAsync(TestContext.Current.CancellationToken); + await service.SyncAsync(TestContext.Current.CancellationToken); + + var alert = Assert.Single(alerts.Alerts); + Assert.Equal("skill.plugin.security_rejected", alert.Type); + Assert.Equal(AlertType.SkillPluginSecurityRejected, alert.Category); + Assert.DoesNotContain("scanner result", alert.Summary); + } + + [Fact] + public async Task Restart_claims_and_emits_a_persisted_security_rejection_alert() + { + var source = Source(); + var store = await CreateStoreAsync(); + var fingerprint = ManagedPluginSourceValidator.Fingerprint(source); + await store.SaveRejectionAsync( + source.Id, fingerprint, FirstCommit, "scanner result", true, TestContext.Current.CancellationToken); + var acquirer = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill"); + var alerts = new RecordingSink(); + var service = new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [source] }, _paths, CreateRefresher(new SkillRegistry(), static () => { }), _time, + new NoOpSkillContentScanner(), NullLogger.Instance, store, acquirer, alerts); + + await service.SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(0, acquirer.AcquireCount); + Assert.Equal("skill.plugin.security_rejected", Assert.Single(alerts.Alerts).Type); + Assert.True((await store.GetRejectionAsync( + source.Id, fingerprint, FirstCommit, TestContext.Current.CancellationToken))!.AlertEmitted); + } + + [Fact] + public async Task Source_timeout_covers_branch_resolution_before_archive_acquisition() + { + var source = Source(); + source.TimeoutSeconds = 1; + var acquirer = new BlockingResolveAcquirer(); + var service = await CreateServiceAsync( + source, CreateRefresher(new SkillRegistry(), static () => { }), acquirer, new RecordingSink()); + + var sync = service.SyncAsync(TestContext.Current.CancellationToken); + await acquirer.ResolutionStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + _time.Advance(TimeSpan.FromSeconds(1)); + var result = await sync; + + Assert.Equal(1, Assert.Single(result.Sources).FailedCount); + Assert.Equal(0, acquirer.AcquireCount); + } + + [Fact] + public async Task Startup_cleanup_removes_staging_and_orphan_commits_but_keeps_receipt_directory() + { + var source = Source(); + var store = await CreateStoreAsync(); + await store.SaveReceiptAsync(source, ReceiptCandidate(FirstCommit, "1.0.0"), TestContext.Current.CancellationToken); + var fingerprint = ManagedPluginSourceValidator.Fingerprint(source); + var selected = _paths.ManagedGitSkillCommitDirectory(source.Id, fingerprint, FirstCommit); + var orphan = _paths.ManagedGitSkillCommitDirectory(source.Id, fingerprint, SecondCommit); + var staging = Path.Combine(_paths.ManagedGitSkillDirectory(source.Id), ".staging", "candidate"); + var publishedStagingResource = Path.Combine(selected, "plugin-skill", "resources", ".staging", "guide.md"); + var publishedCommitsResource = Path.Combine(selected, "plugin-skill", "resources", "commits", "guide.md"); + Directory.CreateDirectory(selected); + Directory.CreateDirectory(orphan); + Directory.CreateDirectory(staging); + Directory.CreateDirectory(Path.GetDirectoryName(publishedStagingResource)!); + Directory.CreateDirectory(Path.GetDirectoryName(publishedCommitsResource)!); + File.WriteAllText(publishedStagingResource, "published staging resource"); + File.WriteAllText(publishedCommitsResource, "published commits resource"); + var registry = new SkillRegistry(); + var service = new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [source] }, + _paths, + CreateRefresher(registry, static () => { }), + _time, + new NoOpSkillContentScanner(), + NullLogger.Instance, + store, + new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill"), + new RecordingSink()); + + await service.SyncAsync(TestContext.Current.CancellationToken); + + Assert.True(Directory.Exists(selected)); + Assert.False(Directory.Exists(orphan)); + Assert.False(Directory.Exists(staging)); + Assert.True(File.Exists(publishedStagingResource)); + Assert.True(File.Exists(publishedCommitsResource)); + } + + [Fact] + public async Task Disabled_source_does_not_publish_a_prior_receipt() + { + var source = Source(); + var store = await CreateStoreAsync(); + await store.SaveReceiptAsync(source, ReceiptCandidate(FirstCommit, "1.0.0"), TestContext.Current.CancellationToken); + var directory = _paths.ManagedGitSkillCommitDirectory( + source.Id, ManagedPluginSourceValidator.Fingerprint(source), FirstCommit); + Directory.CreateDirectory(Path.Combine(directory, "plugin-skill")); + File.WriteAllText(Path.Combine(directory, "plugin-skill", "SKILL.md"), SkillMarkdown("plugin-skill", "hidden")); + source.Enabled = false; + var acquirer = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill"); + var registry = new SkillRegistry(); + var service = new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [source] }, _paths, CreateRefresher(registry, static () => { }), _time, + new NoOpSkillContentScanner(), NullLogger.Instance, store, acquirer, + new RecordingSink()); + + await service.SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(0, acquirer.ResolveCount); + Assert.Null(registry.GetByName("plugin-skill")); + Assert.True(Directory.Exists(directory)); + } + + [Fact] + public async Task Changed_source_acquisition_failure_keeps_the_prior_receipt_and_directory() + { + var original = Source(); + var store = await CreateStoreAsync(); + await store.SaveReceiptAsync(original, ReceiptCandidate(FirstCommit, "1.0.0"), TestContext.Current.CancellationToken); + var oldDirectory = _paths.ManagedGitSkillCommitDirectory( + original.Id, ManagedPluginSourceValidator.Fingerprint(original), FirstCommit); + var oldSkillPath = Path.Combine(oldDirectory, "plugin-skill", "SKILL.md"); + Directory.CreateDirectory(Path.GetDirectoryName(oldSkillPath)!); + File.WriteAllText(oldSkillPath, SkillMarkdown("plugin-skill", "prior package")); + var changed = Source(); + changed.Reference = "release"; + var acquirer = new FakeAcquirer(_paths, changed, SecondCommit, "2.0.0", "plugin-skill") + { + Failure = new IOException("network unavailable"), + }; + var registry = new SkillRegistry(); + var service = new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [changed] }, _paths, CreateRefresher(registry, static () => { }), _time, + new NoOpSkillContentScanner(), NullLogger.Instance, store, acquirer, + new RecordingSink()); + + var result = await service.SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, Assert.Single(result.Sources).FailedCount); + Assert.Equal(FirstCommit, (await store.GetReceiptAsync(original.Id, TestContext.Current.CancellationToken))!.InstalledCommit); + Assert.True(Directory.Exists(oldDirectory)); + Assert.Contains("prior package", await File.ReadAllTextAsync( + registry.GetByName("plugin-skill")!.FilePath, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task Restart_publishes_the_installed_plugin_before_remote_work_finishes() + { + var source = Source(); + var store = await CreateStoreAsync(); + await store.SaveReceiptAsync(source, ReceiptCandidate(FirstCommit, "1.0.0"), TestContext.Current.CancellationToken); + var directory = _paths.ManagedGitSkillCommitDirectory( + source.Id, ManagedPluginSourceValidator.Fingerprint(source), FirstCommit); + var skillPath = Path.Combine(directory, "plugin-skill", "SKILL.md"); + Directory.CreateDirectory(Path.GetDirectoryName(skillPath)!); + File.WriteAllText(skillPath, SkillMarkdown("plugin-skill", "installed package")); + var acquirer = new BlockingResolveAcquirer(); + var registry = new SkillRegistry(); + var service = new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [source] }, _paths, CreateRefresher(registry, static () => { }), _time, + new NoOpSkillContentScanner(), NullLogger.Instance, store, acquirer, + new RecordingSink()); + + var sync = service.SyncAsync(TestContext.Current.CancellationToken); + await acquirer.ResolutionStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + + Assert.NotNull(registry.GetByName("plugin-skill")); + + _time.Advance(TimeSpan.FromSeconds(source.TimeoutSeconds)); + await sync; + } + + [Fact] + public async Task Invalid_plugin_configuration_reports_failure_and_keeps_installed_content() + { + var source = Source(); + var store = await CreateStoreAsync(); + await store.SaveReceiptAsync(source, ReceiptCandidate(FirstCommit, "1.0.0"), TestContext.Current.CancellationToken); + var directory = _paths.ManagedGitSkillCommitDirectory( + source.Id, ManagedPluginSourceValidator.Fingerprint(source), FirstCommit); + var skillPath = Path.Combine(directory, "plugin-skill", "SKILL.md"); + Directory.CreateDirectory(Path.GetDirectoryName(skillPath)!); + File.WriteAllText(skillPath, SkillMarkdown("plugin-skill", "installed package")); + var duplicate = Source(); + var registry = new SkillRegistry(); + var service = new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [source, duplicate] }, _paths, + CreateRefresher(registry, static () => { }), _time, new NoOpSkillContentScanner(), + NullLogger.Instance, store, + new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill"), new RecordingSink()); + + var result = await service.SyncAsync(TestContext.Current.CancellationToken); + + var failure = Assert.Single(result.Sources); + Assert.Equal("managed-git-plugins", failure.Name); + Assert.Equal(SkillSyncResult.GitPluginSourceKind, failure.SourceKind); + Assert.Equal(1, failure.FailedCount); + Assert.NotNull(registry.GetByName("plugin-skill")); + } + + [Fact] + public async Task Scanner_service_failure_creates_no_rejection_or_alert() + { + var source = Source(); + var acquirer = new FakeAcquirer(_paths, source, FirstCommit, "1.0.0", "plugin-skill") + { + Failure = new GitSkillPluginScannerUnavailableException("scanner unavailable"), + }; + var alerts = new RecordingSink(); + var service = await CreateServiceAsync(source, CreateRefresher(new SkillRegistry(), static () => { }), acquirer, alerts); + + var result = await service.SyncAsync(TestContext.Current.CancellationToken); + var store = new ManagedPluginStateStore(_paths, _time); + + Assert.Equal(1, Assert.Single(result.Sources).FailedCount); + Assert.Null(await store.GetRejectionAsync( + source.Id, ManagedPluginSourceValidator.Fingerprint(source), FirstCommit, + TestContext.Current.CancellationToken)); + Assert.Empty(alerts.Alerts); + } + + [Fact] + public async Task Versionless_commit_change_publishes_the_new_commit() + { + var source = Source(); + var registry = new SkillRegistry(); + var refresher = CreateRefresher(registry, static () => { }); + await (await CreateServiceAsync( + source, refresher, new FakeAcquirer(_paths, source, FirstCommit, null, "plugin-skill"), new RecordingSink())) + .SyncAsync(TestContext.Current.CancellationToken); + var updated = new FakeAcquirer(_paths, source, SecondCommit, null, "plugin-skill", "new versionless content"); + await (await CreateServiceAsync(source, refresher, updated, new RecordingSink())) + .SyncAsync(TestContext.Current.CancellationToken); + + var receipt = await new ManagedPluginStateStore(_paths, _time).GetReceiptAsync( + source.Id, TestContext.Current.CancellationToken); + Assert.NotNull(receipt); + Assert.Equal(SecondCommit, receipt.InstalledCommit); + Assert.Equal(1, updated.AcquireCount); + Assert.Contains("new versionless content", await File.ReadAllTextAsync( + registry.GetByName("plugin-skill")!.FilePath, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task One_failed_plugin_does_not_block_a_successful_plugin() + { + var failed = Source(); + failed.Id = "failed"; + var healthy = Source(); + healthy.Id = "healthy"; + var store = await CreateStoreAsync(); + var registry = new SkillRegistry(); + var service = new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [failed, healthy] }, + _paths, + CreateRefresher(registry, static () => { }), + _time, + new NoOpSkillContentScanner(), + NullLogger.Instance, + store, + new TwoPluginAcquirer(_paths, failed, healthy), + new RecordingSink()); + + var result = await service.SyncAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, Assert.Single(result.Sources, row => row.Name == "failed").FailedCount); + Assert.Equal(1, Assert.Single(result.Sources, row => row.Name == "healthy").ChangedCount); + Assert.NotNull(registry.GetByName("healthy-skill")); + } + + private async Task CreateStoreAsync() + { + await new SchemaMigrator(_paths, NullLogger.Instance) + .MigrateAsync(_paths.SqliteDbPath, TestContext.Current.CancellationToken); + return new ManagedPluginStateStore(_paths, _time); + } + + private async Task AddReceiptFailureTriggerAsync() + { + await using var connection = new SqliteConnection($"Data Source={_paths.SqliteDbPath}"); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var command = connection.CreateCommand(); + command.CommandText = + """ + CREATE TRIGGER reject_managed_plugin_receipt_update + BEFORE UPDATE ON managed_plugin_receipts + BEGIN + SELECT RAISE(ABORT, 'receipt persistence failed'); + END; + """; + await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + private async Task CreateServiceAsync( + ManagedPluginSource source, + SkillInventoryRefresher refresher, + IGitSkillPluginAcquirer acquirer, + RecordingSink sink) + { + var store = await CreateStoreAsync(); + return new ServerFeedSkillSyncService( + new SkillFeedsConfig { Plugins = [source] }, + _paths, + refresher, + _time, + new NoOpSkillContentScanner(), + NullLogger.Instance, + store, + acquirer, + sink); + } + + private SkillInventoryRefresher CreateRefresher(SkillRegistry registry, Action publish) + => new( + _paths, + new SkillFeedsConfig(), + [], + registry, + new SkillIndexPublisher(registry, new SkillIndexContextLayer(), (_, _) => + { + publish(); + return true; + })); + + private static ManagedPluginSource Source() => new() + { + Id = "fixture", + Repository = "owner/repository", + Format = "codex", + ReferenceKind = ManagedPluginReferenceKind.Branch, + Reference = "main", + }; + + private static ManagedPluginCandidate ReceiptCandidate(string commit, string? version) => new( + commit, + ManagedPluginSourceValidator.CodexFormat, + "fixture-package", + version, + "unused", + [], + []); + + private sealed class FakeAcquirer( + NetclawPaths paths, + ManagedPluginSource source, + string commit, + string? version, + string skillName, + string description = "plugin guidance") : IGitSkillPluginAcquirer + { + public bool FailOnResolve { get; init; } + public GitSkillPluginRejectedException? Rejection { get; init; } + public Exception? Failure { get; init; } + public Action? BeforeReturn { get; init; } + public int ResolveCount { get; private set; } + public int AcquireCount { get; private set; } + + public Task ResolveDefaultBranchAsync( + string repository, + CancellationToken cancellationToken) => Task.FromResult("main"); + + public Task AcquireAsync( + ManagedPluginSource ignored, + CancellationToken cancellationToken) + => AcquireAsync(ignored, commit, cancellationToken); + + public Task ResolveCommitAsync( + ManagedPluginSource ignored, + CancellationToken cancellationToken) + { + ResolveCount++; + if (FailOnResolve) + throw new InvalidOperationException("The commit pin must not resolve."); + return Task.FromResult(commit); + } + + public Task AcquireAsync( + ManagedPluginSource ignored, + string resolvedCommit, + CancellationToken cancellationToken) + { + AcquireCount++; + if (Failure is not null) + throw Failure; + if (Rejection is not null) + throw Rejection; + + var directory = paths.ManagedGitSkillCommitDirectory( + source.Id, + ManagedPluginSourceValidator.Fingerprint(source), + resolvedCommit); + var skillDirectory = Path.Combine(directory, skillName); + Directory.CreateDirectory(skillDirectory); + File.WriteAllText(Path.Combine(skillDirectory, "SKILL.md"), $$""" + --- + name: {{skillName}} + description: {{description}} + metadata: + version: {{version ?? ""}} + --- + + # {{skillName}} + """); + BeforeReturn?.Invoke(); + return Task.FromResult(new ManagedPluginCandidate( + resolvedCommit, + ManagedPluginSourceValidator.CodexFormat, + skillName, + version, + directory, + SkillScanner.Scan(directory).AcceptedSkills, + [])); + } + } + + private sealed class RecordingSink : IOperationalNotificationSink + { + public List Alerts { get; } = []; + public void Emit(OperationalAlert alert) => Alerts.Add(alert); + } + + private sealed class TwoPluginAcquirer(NetclawPaths paths, ManagedPluginSource failed, ManagedPluginSource healthy) + : IGitSkillPluginAcquirer + { + public Task ResolveDefaultBranchAsync( + string repository, + CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task AcquireAsync(ManagedPluginSource source, CancellationToken cancellationToken) + => AcquireAsync(source, source.Id == failed.Id ? FirstCommit : SecondCommit, cancellationToken); + + public Task ResolveCommitAsync(ManagedPluginSource source, CancellationToken cancellationToken) + => Task.FromResult(source.Id == failed.Id ? FirstCommit : SecondCommit); + + public Task AcquireAsync( + ManagedPluginSource source, + string commit, + CancellationToken cancellationToken) + { + if (source.Id == failed.Id) + throw new IOException("network unavailable"); + + var directory = paths.ManagedGitSkillCommitDirectory( + healthy.Id, ManagedPluginSourceValidator.Fingerprint(healthy), commit); + var skillDirectory = Path.Combine(directory, "healthy-skill"); + Directory.CreateDirectory(skillDirectory); + File.WriteAllText(Path.Combine(skillDirectory, "SKILL.md"), SkillMarkdown("healthy-skill", "healthy")); + return Task.FromResult(new ManagedPluginCandidate( + commit, + ManagedPluginSourceValidator.CodexFormat, + "healthy-plugin", + null, + directory, + SkillScanner.Scan(directory).AcceptedSkills, + [])); + } + } + + private sealed class BlockingResolveAcquirer : IGitSkillPluginAcquirer + { + private readonly TaskCompletionSource _never = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ResolutionStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int AcquireCount { get; private set; } + + public Task ResolveDefaultBranchAsync( + string repository, + CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task AcquireAsync(ManagedPluginSource source, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public async Task ResolveCommitAsync(ManagedPluginSource source, CancellationToken cancellationToken) + { + ResolutionStarted.TrySetResult(); + await _never.Task.WaitAsync(cancellationToken); + return ""; + } + + public Task AcquireAsync( + ManagedPluginSource source, + string commit, + CancellationToken cancellationToken) + { + AcquireCount++; + throw new NotSupportedException(); + } + } + + private static string SkillMarkdown(string name, string description) => $$""" + --- + name: {{name}} + description: {{description}} + --- + + # {{name}} + """; +} diff --git a/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncResultTests.cs b/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncResultTests.cs index 43fb42867..419f4c720 100644 --- a/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncResultTests.cs +++ b/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncResultTests.cs @@ -29,9 +29,15 @@ public ServerFeedSkillSyncResultTests() { _paths = new NetclawPaths(_directory.Path); _paths.EnsureDirectoriesExist(); + new SchemaMigrator(_paths, NullLogger.Instance) + .MigrateAsync(_paths.SqliteDbPath, CancellationToken.None).GetAwaiter().GetResult(); } - public void Dispose() => _directory.Dispose(); + public void Dispose() + { + SqliteTestPools.Clear(_paths); + _directory.Dispose(); + } [Fact] public async Task Rejected_skill_preserves_old_bytes_and_reports_partial_source_failure() @@ -185,7 +191,11 @@ private ServerFeedSkillSyncService CreateService( scanner, NullLogger.Instance, [], - feed => new SkillServerClient(new HttpClient(handler) { BaseAddress = new Uri(feed.Url) })); + feed => new SkillServerClient(new HttpClient(handler) { BaseAddress = new Uri(feed.Url) }), + new ManagedPluginStateStore(_paths, TimeProvider.System), + new GitSkillPluginAcquirer( + new HttpClient(new HttpClientHandler()), _paths, TimeProvider.System, new NoOpSkillContentScanner()), + NullNotificationSink.Instance); private static Task RunAsync(ServerFeedSkillSyncService service) => service.SyncAsync(TestContext.Current.CancellationToken); diff --git a/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncServiceTests.cs b/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncServiceTests.cs index 2efb4ceb8..dc8bad132 100644 --- a/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncServiceTests.cs +++ b/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncServiceTests.cs @@ -32,6 +32,8 @@ public sealed class ServerFeedSkillSyncServiceTests : IDisposable private readonly SkillRegistry _skillRegistry = new(); private readonly SkillIndexContextLayer _skillIndexLayer = new(); private readonly SkillIndexPublisher _skillIndexPublisher; + private readonly ManagedPluginStateStore _pluginStateStore; + private readonly IGitSkillPluginAcquirer _pluginAcquirer; public ServerFeedSkillSyncServiceTests() { @@ -41,9 +43,18 @@ public ServerFeedSkillSyncServiceTests() _skillRegistry, _skillIndexLayer, static (_, _) => true); + new SchemaMigrator(_paths, NullLogger.Instance) + .MigrateAsync(_paths.SqliteDbPath, CancellationToken.None).GetAwaiter().GetResult(); + _pluginStateStore = new ManagedPluginStateStore(_paths, TimeProvider.System); + _pluginAcquirer = new GitSkillPluginAcquirer( + new HttpClient(new HttpClientHandler()), _paths, TimeProvider.System, new NoOpSkillContentScanner()); } - public void Dispose() => _dir.Dispose(); + public void Dispose() + { + SqliteTestPools.Clear(_paths); + _dir.Dispose(); + } [Fact] public async Task SyncAsync_with_no_enabled_sources_returns_an_empty_successful_result() @@ -409,7 +420,11 @@ private ServerFeedSkillSyncService CreateService(ISkillContentScanner? scanner = TimeProvider.System, scanner ?? new NoOpSkillContentScanner(), NullLogger.Instance, - []); + [], + _ => new SkillServerClient(new HttpClient(new HttpClientHandler())), + _pluginStateStore, + _pluginAcquirer, + NullNotificationSink.Instance); private static Task RunSyncAsync(ServerFeedSkillSyncService service) => service.SyncAsync(TestContext.Current.CancellationToken); @@ -440,7 +455,10 @@ private ServerFeedSkillSyncService CreateService(FakeHttpMessageHandler handler) if (feed.ApiKey is { Value: { Length: > 0 } apiKey }) client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); return new SkillServerClient(client); - }); + }, + _pluginStateStore, + _pluginAcquirer, + NullNotificationSink.Instance); } private ServerFeedSkillSyncService CreateControlledService( @@ -467,7 +485,10 @@ private ServerFeedSkillSyncService CreateControlledService( feed => new SkillServerClient(new HttpClient(handler) { BaseAddress = new Uri(feed.Url), - })); + }), + _pluginStateStore, + _pluginAcquirer, + NullNotificationSink.Instance); } private ServerFeedSkillSyncService CreateService( @@ -486,7 +507,10 @@ private ServerFeedSkillSyncService CreateService( feed => new SkillServerClient(new HttpClient(handler) { BaseAddress = new Uri(feed.Url), - })); + }), + _pluginStateStore, + _pluginAcquirer, + NullNotificationSink.Instance); private SkillSyncState ReadAgentSyncState() { diff --git a/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncToolIntegrationTests.cs b/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncToolIntegrationTests.cs index efcb7d8be..ec9d910f0 100644 --- a/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncToolIntegrationTests.cs +++ b/src/Netclaw.Daemon.Tests/Services/ServerFeedSkillSyncToolIntegrationTests.cs @@ -35,7 +35,11 @@ public ServerFeedSkillSyncToolIntegrationTests() _paths.EnsureDirectoriesExist(); } - public void Dispose() => _directory.Dispose(); + public void Dispose() + { + SqliteTestPools.Clear(_paths); + _directory.Dispose(); + } [Fact] public async Task SyncAsync_updates_the_live_registry_for_SkillReadResourceTool() @@ -95,7 +99,18 @@ private ServerFeedSkillSyncService CreateService(RevisionFeedHandler handler) feed => new SkillServerClient(new HttpClient(handler, disposeHandler: false) { BaseAddress = new Uri(feed.Url), - })); + }), + CreatePluginStateStore(), + new GitSkillPluginAcquirer( + new HttpClient(new HttpClientHandler()), _paths, TimeProvider.System, new NoOpSkillContentScanner()), + NullNotificationSink.Instance); + } + + private ManagedPluginStateStore CreatePluginStateStore() + { + new SchemaMigrator(_paths, NullLogger.Instance) + .MigrateAsync(_paths.SqliteDbPath, CancellationToken.None).GetAwaiter().GetResult(); + return new ManagedPluginStateStore(_paths, TimeProvider.System); } private sealed class RevisionFeedHandler : HttpMessageHandler diff --git a/src/Netclaw.Daemon.Tests/Skills/SkillEndpointRouteBuilderExtensionsTests.cs b/src/Netclaw.Daemon.Tests/Skills/SkillEndpointRouteBuilderExtensionsTests.cs index 9f1e9b219..e277f7aba 100644 --- a/src/Netclaw.Daemon.Tests/Skills/SkillEndpointRouteBuilderExtensionsTests.cs +++ b/src/Netclaw.Daemon.Tests/Skills/SkillEndpointRouteBuilderExtensionsTests.cs @@ -37,7 +37,11 @@ public sealed class SkillEndpointRouteBuilderExtensionsTests : IDisposable private readonly DisposableTempDir _dir = new(); - public void Dispose() => _dir.Dispose(); + public void Dispose() + { + SqliteTestPools.Clear(new NetclawPaths(_dir.Path)); + _dir.Dispose(); + } private async Task CreateAppAsync( bool spoofLoopback, @@ -261,7 +265,10 @@ private static ServerFeedSkillSyncService CreateSyncService(SkillRegistry regist new SkillInventoryRefresher(paths, new SkillFeedsConfig(), [], registry, publisher), TimeProvider.System, new NoOpSkillContentScanner(), - Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + CreatePluginStateStore(paths), + CreatePluginAcquirer(paths), + NullNotificationSink.Instance); } private static ServerFeedSkillSyncService CreateBlockingSyncService( @@ -287,9 +294,26 @@ private static ServerFeedSkillSyncService CreateBlockingSyncService( feed => new Netclaw.SkillClient.SkillServerClient(new HttpClient(handler) { BaseAddress = new Uri(feed.Url), - })); + }), + CreatePluginStateStore(paths), + CreatePluginAcquirer(paths), + NullNotificationSink.Instance); + } + + private static ManagedPluginStateStore CreatePluginStateStore(NetclawPaths paths) + { + new SchemaMigrator(paths, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance) + .MigrateAsync(paths.SqliteDbPath, CancellationToken.None).GetAwaiter().GetResult(); + return new ManagedPluginStateStore(paths, TimeProvider.System); } + private static IGitSkillPluginAcquirer CreatePluginAcquirer(NetclawPaths paths) + => new GitSkillPluginAcquirer( + new HttpClient(new HttpClientHandler()), + paths, + TimeProvider.System, + new NoOpSkillContentScanner()); + private sealed class BlockingFeedHandler : HttpMessageHandler { private const string SkillContent = "---\nname: route-proof\ndescription: Route proof.\n---\n\nRoute proof body.\n"; diff --git a/src/Netclaw.Daemon/Configuration/DaemonToolPathPolicyFactory.cs b/src/Netclaw.Daemon/Configuration/DaemonToolPathPolicyFactory.cs index 2ffede70b..877eb078b 100644 --- a/src/Netclaw.Daemon/Configuration/DaemonToolPathPolicyFactory.cs +++ b/src/Netclaw.Daemon/Configuration/DaemonToolPathPolicyFactory.cs @@ -38,6 +38,7 @@ public static ToolPathPolicy Create( ..processControlPaths, paths.SystemSkillsDirectory, paths.ServerFeedsDirectory, + paths.ManagedGitSkillsDirectory, paths.ToolingShadowDirectory, ]; string[] readDenyList = @@ -65,6 +66,7 @@ public static ToolPathPolicy Create( ..sqliteSidecars, ..processControlPaths, paths.ToolingShadowDirectory, + paths.ManagedGitSkillsDirectory, ]; return new ToolPathPolicy( diff --git a/src/Netclaw.Daemon/Program.cs b/src/Netclaw.Daemon/Program.cs index 961c4166b..242b0c442 100644 --- a/src/Netclaw.Daemon/Program.cs +++ b/src/Netclaw.Daemon/Program.cs @@ -53,6 +53,7 @@ using Netclaw.Search; using Netclaw.Tools; using Netclaw.Security; +using Netclaw.Security.Skills; // Handled first, before any directory creation, lock-file acquisition, or host startup: // `netclawd --version`/`-v` must print the version and exit rather than booting a real @@ -947,6 +948,23 @@ static void ConfigureDaemonServices( services.AddHostedService(); // The runner owns one pass. The actor owns startup, timers, and shared requests. + services.AddSingleton(); + services.AddHttpClient("GitSkillPlugin", client => client.Timeout = Timeout.InfiniteTimeSpan) + .ConfigurePrimaryHttpMessageHandler(GitSkillPluginAcquirer.CreateHttpHandler) + .AddNetclawHeaders("git-skill-plugin"); + services.AddSingleton(sp => new GitSkillPluginAcquirer( + sp.GetRequiredService().CreateClient("GitSkillPlugin"), + paths, + sp.GetRequiredService(), + sp.GetRequiredService())); + services.AddSingleton(sp => new ManagedPluginSyncParticipant( + skillFeedsConfig, + paths, + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>())); services.AddSingleton(); services.AddSingleton( sp => sp.GetRequiredService()); diff --git a/src/Netclaw.Daemon/Services/ManagedPluginSyncParticipant.cs b/src/Netclaw.Daemon/Services/ManagedPluginSyncParticipant.cs new file mode 100644 index 000000000..296b255a1 --- /dev/null +++ b/src/Netclaw.Daemon/Services/ManagedPluginSyncParticipant.cs @@ -0,0 +1,377 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Microsoft.Extensions.Logging; +using Netclaw.Actors.Skills; +using Netclaw.Configuration; +using Netclaw.Security.Skills; + +namespace Netclaw.Daemon.Services; + +internal sealed record ManagedPluginSyncResult( + IReadOnlyList Rows, + IReadOnlyList Sources); + +/// Owns the managed plugin state machine within one shared skill sync pass. +internal sealed class ManagedPluginSyncParticipant( + SkillFeedsConfig feedsConfig, + NetclawPaths paths, + TimeProvider timeProvider, + ManagedPluginStateStore stateStore, + IGitSkillPluginAcquirer acquirer, + IOperationalNotificationSink notificationSink, + ILogger logger) +{ + private bool _startupPublicationComplete; + private bool _startupCleanupComplete; + + public async Task?> LoadStartupSourcesAsync( + CancellationToken cancellationToken) + { + if (_startupPublicationComplete) + return null; + + var receipts = await stateStore.LoadReceiptsAsync(cancellationToken); + _startupPublicationComplete = true; + return ResolveSources(receipts); + } + + public async Task SyncAsync( + bool retryRejected, + CancellationToken cancellationToken) + { + var rows = new List(); + var configuredSources = feedsConfig.Plugins; + if (!ManagedPluginSourceValidator.TryValidateSources(configuredSources, out var validationError)) + { + logger.LogWarning("Managed plugin configuration is invalid: {Error}", validationError); + rows.Add(ConfigurationFailure()); + return new ManagedPluginSyncResult( + rows, + ResolveSources(await stateStore.LoadReceiptsAsync(cancellationToken))); + } + + await stateStore.RemoveSourcesExceptAsync( + configuredSources.Select(static source => source.Id).ToArray(), + cancellationToken); + var receipts = await stateStore.LoadReceiptsAsync(cancellationToken); + if (!_startupCleanupComplete) + { + CleanupDirectories(receipts); + _startupCleanupComplete = true; + } + + foreach (var source in configuredSources.Where(static source => source.Enabled)) + { + try + { + rows.Add(await SyncSourceAsync(source, retryRejected, cancellationToken)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "Managed plugin sync failed for '{PluginId}' and kept the prior publication", + source.Id); + rows.Add(PluginFailure(source.Id)); + } + } + + receipts = await stateStore.LoadReceiptsAsync(cancellationToken); + return new ManagedPluginSyncResult(rows, ResolveSources(receipts)); + } + + private async Task SyncSourceAsync( + ManagedPluginSource source, + bool retryRejected, + CancellationToken cancellationToken) + { + using var timeout = new CancellationTokenSource( + TimeSpan.FromSeconds(source.TimeoutSeconds), + timeProvider); + using var sourceOperation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeout.Token); + var sourceToken = sourceOperation.Token; + var sourceFingerprint = ManagedPluginSourceValidator.Fingerprint(source); + var receipt = await stateStore.GetReceiptAsync(source.Id, sourceToken); + var commit = source.ReferenceKind == ManagedPluginReferenceKind.Commit + ? source.Reference.ToLowerInvariant() + : await acquirer.ResolveCommitAsync(source, sourceToken); + var installedDirectory = receipt is null + ? null + : paths.ManagedGitSkillCommitDirectory( + receipt.SourceId, + receipt.SourceFingerprint, + receipt.InstalledCommit); + + if (receipt is not null + && string.Equals(receipt.SourceFingerprint, sourceFingerprint, StringComparison.Ordinal) + && Directory.Exists(installedDirectory) + && (string.Equals(receipt.InstalledCommit, commit, StringComparison.Ordinal) + || string.Equals(receipt.LastObservedCommit, commit, StringComparison.Ordinal))) + { + return PluginUnchanged(source.Id, receipt.InstalledCommit, receipt.InstalledVersion); + } + + var rejection = await stateStore.GetRejectionAsync( + source.Id, + sourceFingerprint, + commit, + sourceToken); + if (rejection is not null && !retryRejected) + { + if (rejection.SecurityRejection + && !rejection.AlertEmitted + && await stateStore.TryClaimSecurityAlertAsync( + source.Id, + sourceFingerprint, + commit, + sourceToken)) + { + EmitSecurityRejectionAlert(sourceFingerprint, commit); + } + + logger.LogInformation( + "Managed plugin '{PluginId}' commit {Commit} remains rejected", + source.Id, + commit); + return PluginRejected(source.Id, commit); + } + + try + { + var candidate = await acquirer.AcquireAsync(source, commit, sourceToken); + if (!Directory.Exists(candidate.Directory)) + throw new IOException("The immutable managed plugin candidate is missing."); + + if (receipt is not null + && string.Equals(receipt.SourceFingerprint, sourceFingerprint, StringComparison.Ordinal) + && Directory.Exists(installedDirectory) + && candidate.Version is not null + && string.Equals(candidate.Version, receipt.InstalledVersion, StringComparison.Ordinal)) + { + if (rejection is null) + { + await stateStore.UpdateLastObservedCommitAsync( + source.Id, + candidate.Commit, + sourceToken); + } + else + { + await stateStore.UpdateLastObservedCommitAfterRetryAsync( + source.Id, + sourceFingerprint, + candidate.Commit, + sourceToken); + } + DeleteUnpublishedCandidate(candidate.Directory, installedDirectory); + return PluginUnchanged(source.Id, receipt.InstalledCommit, receipt.InstalledVersion); + } + + if (rejection is null) + await stateStore.SaveReceiptAsync(source, candidate, sourceToken); + else + await stateStore.SaveReceiptAfterRetryAsync(source, candidate, sourceToken); + + return new SkillSyncResult.SourceRow + { + Name = source.Id, + SourceKind = SkillSyncResult.GitPluginSourceKind, + ChangedCount = 1, + Sidecar = "not-applicable", + Commit = candidate.Commit, + Version = candidate.Version, + Notices = candidate.Notices, + }; + } + catch (GitSkillPluginRejectedException exception) + { + await stateStore.SaveRejectionAsync( + source.Id, + sourceFingerprint, + exception.Commit, + exception.Message, + exception.SecurityRejection, + sourceToken); + + if (exception.SecurityRejection + && await stateStore.TryClaimSecurityAlertAsync( + source.Id, + sourceFingerprint, + exception.Commit, + sourceToken)) + { + EmitSecurityRejectionAlert(sourceFingerprint, exception.Commit); + } + + logger.LogWarning( + "Managed plugin '{PluginId}' commit {Commit} was rejected", + source.Id, + exception.Commit); + return PluginRejected(source.Id, exception.Commit); + } + } + + private IReadOnlyList ResolveSources( + IReadOnlyList receipts) + => receipts + .Where(receipt => feedsConfig.Plugins.Any(source => source.Enabled + && string.Equals(source.Id, receipt.SourceId, StringComparison.Ordinal))) + .Select(receipt => new + { + Receipt = receipt, + Directory = paths.ManagedGitSkillCommitDirectory( + receipt.SourceId, + receipt.SourceFingerprint, + receipt.InstalledCommit), + }) + .Where(static candidate => Directory.Exists(candidate.Directory)) + .OrderBy(static candidate => candidate.Receipt.SourceId, StringComparer.Ordinal) + .Select(static candidate => new ResolvedExternalSource( + $"managed-git:{candidate.Receipt.SourceId}", + [candidate.Directory], + AllowSymlinks: false)) + .ToArray(); + + private void EmitSecurityRejectionAlert(string sourceFingerprint, string commit) + { + notificationSink.Emit(OperationalAlert.Create( + timeProvider, + "skill.plugin.security_rejected", + AlertType.SkillPluginSecurityRejected, + "A managed Git plugin failed a security check.", + AlertSeverity.Warning, + $"{sourceFingerprint}:{commit}", + new Dictionary + { + ["source_fingerprint"] = sourceFingerprint, + ["commit"] = commit, + })); + } + + private void CleanupDirectories(IReadOnlyList receipts) + { + var selectedDirectories = receipts + .Select(receipt => Path.GetFullPath(paths.ManagedGitSkillCommitDirectory( + receipt.SourceId, + receipt.SourceFingerprint, + receipt.InstalledCommit))) + .ToHashSet(StringComparer.Ordinal); + var root = paths.ManagedGitSkillsDirectory; + if (!Directory.Exists(root)) + return; + + try + { + foreach (var sourceDirectory in Directory.EnumerateDirectories(root)) + { + try + { + CleanupSourceDirectory(sourceDirectory, selectedDirectories); + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "Managed plugin cleanup failed for source directory {Directory}", + sourceDirectory); + } + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Managed plugin cleanup could not enumerate {Directory}", root); + } + } + + private void CleanupSourceDirectory(string sourceDirectory, HashSet selectedDirectories) + { + DeleteDirectory(Path.Combine(sourceDirectory, ".staging")); + foreach (var fingerprintDirectory in Directory.EnumerateDirectories(sourceDirectory)) + { + if (string.Equals(Path.GetFileName(fingerprintDirectory), ".staging", StringComparison.Ordinal)) + continue; + var commitsDirectory = Path.Combine(fingerprintDirectory, "commits"); + if (!Directory.Exists(commitsDirectory)) + continue; + foreach (var commitDirectory in Directory.EnumerateDirectories(commitsDirectory)) + { + if (!selectedDirectories.Contains(Path.GetFullPath(commitDirectory))) + DeleteDirectory(commitDirectory); + } + } + } + + private void DeleteDirectory(string directory) + { + try + { + GitSkillPluginAcquirer.DeleteDirectory(directory); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Managed plugin cleanup failed for directory {Directory}", directory); + } + } + + private static void DeleteUnpublishedCandidate(string candidateDirectory, string installedDirectory) + { + if (!string.Equals( + Path.GetFullPath(candidateDirectory), + Path.GetFullPath(installedDirectory), + StringComparison.Ordinal) + && Directory.Exists(candidateDirectory)) + { + GitSkillPluginAcquirer.DeleteDirectory(candidateDirectory); + } + } + + private static SkillSyncResult.SourceRow ConfigurationFailure() => new() + { + Name = "managed-git-plugins", + SourceKind = SkillSyncResult.GitPluginSourceKind, + FailedCount = 1, + Sidecar = "not-applicable", + Error = "The managed plugin configuration is invalid.", + }; + + private static SkillSyncResult.SourceRow PluginUnchanged( + string sourceId, + string commit, + string? version) => new() + { + Name = sourceId, + SourceKind = SkillSyncResult.GitPluginSourceKind, + UnchangedCount = 1, + Sidecar = "not-applicable", + Commit = commit, + Version = version, + }; + + private static SkillSyncResult.SourceRow PluginRejected(string sourceId, string commit) => new() + { + Name = sourceId, + SourceKind = SkillSyncResult.GitPluginSourceKind, + RejectedCount = 1, + Sidecar = "not-applicable", + Commit = commit, + Error = "The plugin commit is rejected.", + }; + + private static SkillSyncResult.SourceRow PluginFailure(string sourceId) => new() + { + Name = sourceId, + SourceKind = SkillSyncResult.GitPluginSourceKind, + FailedCount = 1, + Sidecar = "not-applicable", + Error = "The source sync failed. Existing files remain in use.", + }; +} diff --git a/src/Netclaw.Daemon/Services/ServerFeedSkillSyncService.cs b/src/Netclaw.Daemon/Services/ServerFeedSkillSyncService.cs index a60dc30a6..0a50f6451 100644 --- a/src/Netclaw.Daemon/Services/ServerFeedSkillSyncService.cs +++ b/src/Netclaw.Daemon/Services/ServerFeedSkillSyncService.cs @@ -37,6 +37,7 @@ internal sealed class ServerFeedSkillSyncService : IServerFeedSkillSyncRunner private readonly ISkillContentScanner _scanner; private readonly ILogger _logger; private readonly Func _clientFactory; + private readonly ManagedPluginSyncParticipant _pluginSyncParticipant; public ServerFeedSkillSyncService( SkillFeedsConfig feedsConfig, @@ -44,7 +45,8 @@ public ServerFeedSkillSyncService( SkillInventoryRefresher inventoryRefresher, TimeProvider timeProvider, ISkillContentScanner scanner, - ILogger logger) + ILogger logger, + ManagedPluginSyncParticipant pluginSyncParticipant) : this( feedsConfig, paths, @@ -52,29 +54,37 @@ public ServerFeedSkillSyncService( timeProvider, scanner, logger, - CreateSkillServerClient) + CreateSkillServerClient, + pluginSyncParticipant) { } internal ServerFeedSkillSyncService( SkillFeedsConfig feedsConfig, NetclawPaths paths, - SkillRegistry skillRegistry, - SkillIndexPublisher skillIndexPublisher, + SkillInventoryRefresher inventoryRefresher, TimeProvider timeProvider, ISkillContentScanner scanner, ILogger logger, - IReadOnlyList externalSources) + ManagedPluginStateStore pluginStateStore, + IGitSkillPluginAcquirer pluginAcquirer, + IOperationalNotificationSink notificationSink) : this( feedsConfig, paths, - skillRegistry, - skillIndexPublisher, + inventoryRefresher, timeProvider, scanner, logger, - externalSources, - CreateSkillServerClient) + CreateSkillServerClient, + new ManagedPluginSyncParticipant( + feedsConfig, + paths, + timeProvider, + pluginStateStore, + pluginAcquirer, + notificationSink, + logger)) { } @@ -87,7 +97,10 @@ internal ServerFeedSkillSyncService( ISkillContentScanner scanner, ILogger logger, IReadOnlyList externalSources, - Func clientFactory) + Func clientFactory, + ManagedPluginStateStore pluginStateStore, + IGitSkillPluginAcquirer pluginAcquirer, + IOperationalNotificationSink notificationSink) : this( feedsConfig, paths, @@ -100,7 +113,15 @@ internal ServerFeedSkillSyncService( timeProvider, scanner, logger, - clientFactory) + clientFactory, + new ManagedPluginSyncParticipant( + feedsConfig, + paths, + timeProvider, + pluginStateStore, + pluginAcquirer, + notificationSink, + logger)) { } @@ -111,7 +132,8 @@ private ServerFeedSkillSyncService( TimeProvider timeProvider, ISkillContentScanner scanner, ILogger logger, - Func clientFactory) + Func clientFactory, + ManagedPluginSyncParticipant pluginSyncParticipant) { _feedsConfig = feedsConfig; _paths = paths; @@ -120,6 +142,7 @@ private ServerFeedSkillSyncService( _scanner = scanner; _logger = logger; _clientFactory = clientFactory; + _pluginSyncParticipant = pluginSyncParticipant; } /// @@ -133,6 +156,10 @@ private ServerFeedSkillSyncService( try { var sources = new List(); + var startupPluginSources = await _pluginSyncParticipant.LoadStartupSourcesAsync(cancellationToken); + if (startupPluginSources is not null) + RescanAndUpdateIndex(startupPluginSources); + foreach (var feed in _feedsConfig.Feeds.Where(static feed => feed.Enabled)) { try @@ -158,9 +185,13 @@ private ServerFeedSkillSyncService( } } + var pluginResult = await _pluginSyncParticipant.SyncAsync(retryRejected: false, cancellationToken); + sources.AddRange(pluginResult.Rows); + var managedPluginSources = pluginResult.Sources; + try { - var scan = RescanAndUpdateIndex(); + var scan = RescanAndUpdateIndex(managedPluginSources); var result = new SkillSyncResult.Response { PassId = passId, @@ -208,6 +239,7 @@ private ServerFeedSkillSyncService( } } + private async Task SyncFeedAsync(SkillFeedSource feed, CancellationToken cancellationToken) { var feedDir = _paths.ServerFeedDirectory(feed.Name); @@ -889,9 +921,11 @@ private static bool IsZipSymlink(ZipArchiveEntry entry) return mode == 0 ? null : mode; } - private MergedSkillScanResult RescanAndUpdateIndex() + private MergedSkillScanResult RescanAndUpdateIndex( + IReadOnlyList managedGitPluginSources) { - var mergedResult = _inventoryRefresher.Refresh(); + var mergedResult = _inventoryRefresher.ReplaceManagedGitPluginSourcesAndRefresh( + managedGitPluginSources); if (mergedResult.Issues.Count > 0) { @@ -919,9 +953,11 @@ private MergedSkillScanResult RescanAndUpdateIndex() private static SkillSyncResult.SourceRow SourceFailure( string name, string sidecar, - string error) => new() + string error, + string sourceKind = SkillSyncResult.ServerFeedSourceKind) => new() { Name = name, + SourceKind = sourceKind, FailedCount = 1, Sidecar = sidecar, Error = error, diff --git a/src/Netclaw.Daemon/Services/SkillDirectoryWatcherService.cs b/src/Netclaw.Daemon/Services/SkillDirectoryWatcherService.cs index f1f4c6e17..51f35e2ac 100644 --- a/src/Netclaw.Daemon/Services/SkillDirectoryWatcherService.cs +++ b/src/Netclaw.Daemon/Services/SkillDirectoryWatcherService.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using Netclaw.Actors.Skills; using Netclaw.Configuration; +using Netclaw.Security; namespace Netclaw.Daemon.Services; @@ -184,8 +185,11 @@ private void OnDebounceTimerFired() } } - private static bool ShouldIgnore(string fullPath) + private bool ShouldIgnore(string fullPath) { + if (PathUtility.IsWithinRoot(fullPath, _paths.ManagedGitSkillsDirectory)) + return true; + // Ignore staging directories and temp files used by atomic write operations return fullPath.Contains($"{Path.DirectorySeparatorChar}.staging{Path.DirectorySeparatorChar}", StringComparison.Ordinal) || fullPath.Contains($"{Path.AltDirectorySeparatorChar}.staging{Path.AltDirectorySeparatorChar}", StringComparison.Ordinal)