Keep the history of existing clones when old commits are deleted

Directory.Build.props

@@ -11,9 +11,9 @@
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
         <ImplicitUsings>enable</ImplicitUsings>
 
-        <AssemblyVersion>0.1.0.0</AssemblyVersion>
-        <FileVersion>0.1.0.0</FileVersion>
-        <Version>0.1.0</Version>
+        <AssemblyVersion>0.1.1.0</AssemblyVersion>
+        <FileVersion>0.1.1.0</FileVersion>
+        <Version>0.1.1</Version>
 
         <Authors>Andreas Nägeli</Authors>
         <Copyright>Copyright © Andreas Nägeli</Copyright>

Module/Protocol/FetchPlanner.cs

@@ -89,7 +89,15 @@
 
         foreach (var have in haves)
         {
-            if (seen.Add(have) && await context.GetCommitAsync(have) != null)
+            if (!seen.Add(have))
+            {
+                continue;
+            }
+
+            // also the commits the history no longer has, which a client that
+            // cloned before it was truncated still has - acknowledged, it
+            // keeps saying so for the rest of the negotiation
+            if (await context.GetCommitAsync(have) != null || (await context.GetTruncatedAsync()).Contains(have))
             {
                 result.Add(have);
             }
@@ -115,6 +123,8 @@
 
         var owned = await GetOwnedAsync(context, request, common);
 
+        var haves = request.Haves.ToHashSet();
+
         var update = await GetShallowUpdateAsync(context, request);
 
         var shallow = new List<GitObjectId>(update.Shallow);
@@ -172,8 +182,10 @@
 
             if (parents == null)
             {
-                // the history of the repository ends here
-                if (!request.ClientShallows.Contains(id) && !shallow.Contains(id))
+                // the history of the repository ends here - unless the client
+                // still has what came before (e.g. a clone made before old
+                // versions were deleted), which it would refuse to cut off
+                if (!request.ClientShallows.Contains(id) && !shallow.Contains(id) && !HasHistory(commit, haves, owned))
                 {
                     shallow.Add(id);
                 }
@@ -246,6 +258,20 @@
         return new FetchPlan(shallow, update.Unshallow, objects);
     }
 
+    /// <summary>
+    /// Whether the client has the history before a commit whose parents the
+    /// repository does not have (anymore): it said it has every one of them,
+    /// or they are part of what it has with the common commits.
+    /// </summary>
+    /// <remarks>
+    /// Announcing such a commit as shallow would cut off history the client
+    /// already has, which git refuses ("shallow roots are not allowed to be
+    /// updated") - so a clone made before old commits were deleted could not
+    /// fetch anything newer.
+    /// </remarks>
+    private static bool HasHistory(GitCommit commit, HashSet<GitObjectId> haves, HashSet<GitObjectId> owned)
+        => commit.Parents.Count > 0 && commit.Parents.All(p => haves.Contains(p) || owned.Contains(p));
+
     /// <summary>
     /// Collects the commits the client has: everything reachable from the
     /// common commits, cut at the shallow boundaries of the client.
@@ -262,6 +288,18 @@
             }
         }
 
+        // what the history no longer has but the client does: the commits
+        // based on it are connected to what the client has
+        var truncated = await context.GetTruncatedAsync();
+
+        foreach (var commit in common)
+        {
+            if (truncated.Contains(commit))
+            {
+                owned.Add(commit);
+            }
+        }
+
         return owned;
     }
 
@@ -290,7 +328,7 @@
 
             pending.Push(want);
 
             while (!found && pending.TryPop(out var id))
             {
                 if (owned.Contains(id) || connected.Contains(id))
                 {

Module/Protocol/RepositoryContext.cs

@@ -6,7 +6,7 @@
 /// Thrown if a repository returns inconsistent data, such as files
 /// that do not match the tree of their commit.
 /// </summary>
 internal sealed class RepositoryException(string message) : Exception(message);
 
 /// <summary>
 /// Wraps a repository for the duration of a single request, caching
@@ -20,6 +20,14 @@
 
     private readonly Dictionary<GitObjectId, MaterializedTree> _trees = new();
 
+    private HashSet<GitObjectId>? _truncated;
+
+    /// <summary>
+    /// How many commits are searched for the ones the history refers to but
+    /// does not have.
+    /// </summary>
+    private const int SearchLimit = 10_000;
+
     #region Get-/Setters
 
     public IGitRepository Repository => repository;
@@ -147,6 +155,55 @@
         return result;
     }
 
+    /// <summary>
+    /// The commits the history of the repository refers to but does not have
+    /// (anymore): the parents of the commits it starts with, if it has been
+    /// truncated.
+    /// </summary>
+    /// <remarks>
+    /// A client that cloned before the history was truncated still has them.
+    /// Knowing them lets the server tell this client that it has the history
+    /// before a truncation, instead of cutting it off as shallow - which git
+    /// refuses for a clone that is not shallow. Only the newest commits of
+    /// the repository are searched, as for the objects a client did not send.
+    /// </remarks>
+    public async ValueTask<IReadOnlySet<GitObjectId>> GetTruncatedAsync()
+    {
+        if (_truncated != null)
+        {
+            return _truncated;
+        }
+
+        var result = new HashSet<GitObjectId>();
+
+        var visited = new HashSet<GitObjectId>();
+
+        var pending = new Stack<GitObjectId>((await GetReferencesAsync()).Select(r => r.Target));
+
+        while (pending.TryPop(out var id) && visited.Count < SearchLimit)
+        {
+            if (!visited.Add(id))
+            {
+                continue;
+            }
+
+            var commit = await GetCommitAsync(id);
+
+            if (commit == null)
+            {
+                result.Add(id);
+                continue;
+            }
+
+            foreach (var parent in commit.Parents)
+            {
+                pending.Push(parent);
+            }
+        }
+
+        return _truncated = result;
+    }
+
     /// <summary>
     /// Finds the commits reachable from the given ones whose history has
     /// been truncated, which need to be announced as shallow to clients.

Testing/Integration/VersionedRepositoryTests.cs

@@ -299,4 +299,37 @@ public async Task TestRetention(int protocol)
         await git.RunAsync("clone", "fsck");
     }
 
+    [TestMethod]
+    [DataRow(0)]
+    [DataRow(2)]
+    public async Task TestRetentionKeepsTheHistoryOfExistingClones(int protocol)
+    {
+        var repository = await CreateAsync(3);
+
+        repository.Retention = 2;
+
+        await using var host = await TestHost.RunAsync(GitServer.Create().Repository(repository));
+
+        using var git = new GitClient(protocol);
+
+        await git.RunAsync(null, "clone", host.GetUrl("/"), "clone");
+
+        // the versions the clone has are deleted while newer ones are saved
+        await repository.SaveVersionAsync(Files(("lambda.cs", "return 4;")), "Version 4");
+        await repository.SaveVersionAsync(Files(("lambda.cs", "return 5;")), "Version 5");
+
+        await git.RunAsync("clone", "pull", "--ff-only");
+
+        Assert.AreEqual("Version 5\nVersion 4\nVersion 3\nVersion 2\nVersion 1\n", await git.RunAsync("clone", "log", "--format=%s"));
+        Assert.AreEqual("false\n", await git.RunAsync("clone", "rev-parse", "--is-shallow-repository"), "the history the clone had is kept");
+
+        await git.RunAsync("clone", "fsck", "--strict");
+
+        // a new clone starts where the repository does
+        await git.RunAsync(null, "clone", host.GetUrl("/"), "fresh");
+
+        Assert.AreEqual("Version 5\nVersion 4\n", await git.RunAsync("fresh", "log", "--format=%s"));
+        Assert.AreEqual("true\n", await git.RunAsync("fresh", "rev-parse", "--is-shallow-repository"));
+    }
+
 }

docs/concepts.md

@@ -110,4 +110,6 @@ A typical incremental fetch therefore only reads the files of the new commits an
 If your data model deletes old content (e.g. only the newest 50 versions are kept), the oldest remaining commit refers
 to a parent that no longer exists. Return `null` from `GetCommitAsync` for such commits: the server treats the
 repository as shallow and tells clients that the history ends there, exactly like a shallow clone created with
-`git clone --depth`. Clients that already have older commits keep them.
+`git clone --depth`. Clients that already have older commits keep them: a client that says it has the parent of
+the oldest remaining commit is told it has the history before it, so it goes on fetching newer commits as a
+complete clone rather than being cut off as a shallow one (which git refuses).

docs/repositories.md

@@ -154,8 +154,10 @@ against its id.
 ## Truncated histories
 
 If old commits are deleted, return `null` from `GetCommitAsync` for them. The oldest remaining commit then acts as
-the shallow boundary of the repository, and clients receive a shallow clone. References must only point to commits
-that still exist.
+the shallow boundary of the repository, and clients receive a shallow clone. Clients that cloned before keep their
+history and keep fetching: the server recognizes the deleted parents of the remaining commits by their ids (as the
+remaining commits name them), and does not cut off a client that still has them. References must only point to
+commits that still exist.
 
 ## Read-only access