diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index c99a602056..c887baccc8 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -29,6 +29,14 @@ public class WindowsFileSystemVirtualizer : FileSystemVirtualizer, IRequiredCall private const int MaxBlobStreamBufferSize = 64 * 1024; private const int MinPrjLibThreads = 5; + // Paired telemetry event names for the read-time corrupt-placeholder self-heal (see + // GetFileStreamHandlerAsyncHandler). Defined once and shared by every emit site so the two + // events stay greppable and cannot drift; _MalformedBlobShaRepaired + _MalformedBlobShaRepairFailed + // together account for every non-aborted repair attempt (a read cancelled mid-repair, or aborted + // because the app closed the handle, emits neither - it is retried, not a repair outcome). + private const string MalformedShaRepairedEventName = nameof(GetFileStreamHandlerAsyncHandler) + "_MalformedBlobShaRepaired"; + private const string MalformedShaRepairFailedEventName = nameof(GetFileStreamHandlerAsyncHandler) + "_MalformedBlobShaRepairFailed"; + private IVirtualizationInstance virtualizationInstance; private ConcurrentDictionary activeEnumerations; private ConcurrentDictionary activeCommands; @@ -763,6 +771,7 @@ public HResult GetFileDataCallback( length, streamGuid, sha, + virtualPath, metadata, triggeringProcessImageFileName), () => @@ -1221,6 +1230,7 @@ private void GetFileStreamHandlerAsyncHandler( uint length, Guid streamGuid, string sha, + string virtualPath, EventMetadata requestMetadata, string triggeringProcessImageFileName) { @@ -1229,10 +1239,62 @@ private void GetFileStreamHandlerAsyncHandler( return; } + // Hoisted above the try so the catch handlers can tell whether a failure occurred while + // repairing a malformed placeholder SHA and still emit the paired repair telemetry. The + // _MalformedBlobShaRepaired and _MalformedBlobShaRepairFailed events must together account + // for every non-aborted repair attempt, so a post-recovery hydration failure that THROWS + // (size mismatch, local IO, ProjFS write failure, or any other exception) must be funneled + // too - not just the TryCopyBlobContentStream-returned-false miss. A read cancelled + // mid-repair, or aborted because the app closed the handle (HResult.Handle), emits neither. + string hydrationSha = sha; + bool repairingMalformedSha = false; + try { + // Read-time self-heal for a corrupt placeholder whose stored content-id is not a valid + // SHA (for example the all-NUL content-id of a corrupt placeholder). #2074 fails such a + // read cleanly to stop the crash and retry storm; here we go one step further and repair + // the underlying data. The authoritative SHA for this path still exists in the git index + // projection, so recover it and hydrate from that instead of the corrupt content-id. A + // successful hydration writes the whole file, which converts the placeholder into a full + // file on disk - the corrupt content-id is superseded and future reads never call back. + // + // We deliberately do NOT rewrite the placeholder's content-id in place (via + // UpdateFileIfNeeded): the reader holds the file open for this GetFileData, so an in-place + // update fails with ERROR_SHARING_VIOLATION. Confirmed empirically against real inbox + // ProjFS: (1) serving the full content converts the placeholder to a full file so a second + // read issues no GetFileData callback, and (2) UpdateFileIfNeeded on the file mid-read + // returns 0x80070020 (ERROR_SHARING_VIOLATION). So hydrating from the recovered SHA is + // both sufficient (the file is repaired for good) and the only option that works here. + if (!SHA1Util.IsValidShaFormat(sha)) + { + if (!this.TryRecoverShaForMalformedPlaceholder( + virtualPath, + cancellationToken, + requestMetadata, + out hydrationSha)) + { + // Could not recover a valid SHA (path deleted/renamed, or the projection lookup + // threw). Fall back to the same clean, non-crashing failure as #2074. The repair + // miss is already funneled by TryRecoverShaForMalformedPlaceholder. + requestMetadata.Add(nameof(GVFSGitObjects.BlobHydrationFailureCategory), GVFSGitObjects.BlobHydrationFailureCategory.Unexpected.ToString()); + this.TryCompleteCommand(commandId, (HResult)HResultExtensions.HResultFromNtStatus.FileNotAvailable); + return; + } + + repairingMalformedSha = true; + + // Note: hydrating the recovered SHA goes through TryCopyBlobContentStream's normal + // download + retry budget. For a projected file whose blob is genuinely unavailable + // this costs a projection lookup plus that budget on every read - the same cost any + // valid-but-unavailable placeholder already pays - rather than #2074's cheap + // fast-fail. Accepted: the corruption is rare and this narrow sub-case (corrupt + // content-id AND path projected AND blob missing) is bounded per read and never + // re-crashes or retry-storms the way the original unhandled ArgumentException did. + } + if (!this.GitObjects.TryCopyBlobContentStream( - sha, + hydrationSha, cancellationToken, GVFSGitObjects.RequestSource.FileStreamCallback, (stream, blobLength) => @@ -1302,11 +1364,28 @@ private void GetFileStreamHandlerAsyncHandler( out GVFSGitObjects.BlobHydrationFailureCategory failureCategory)) { requestMetadata.Add(nameof(GVFSGitObjects.BlobHydrationFailureCategory), failureCategory.ToString()); + + if (repairingMalformedSha) + { + // We recovered a valid SHA from the projection but TryCopyBlobContentStream + // reported a clean miss (the blob itself is unavailable). Funnel it distinctly + // from a projection miss. + this.EmitMalformedShaRepairFailed(requestMetadata, MalformedShaRepairFailureReason.HydrateFailed, hydrationSha); + } + this.Context.Tracer.RelatedError(requestMetadata, $"{nameof(this.GetFileStreamHandlerAsyncHandler)}: TryCopyBlobContentStream failed"); this.TryCompleteCommand(commandId, (HResult)HResultExtensions.HResultFromNtStatus.FileNotAvailable); return; } + + if (repairingMalformedSha) + { + // The corrupt placeholder was hydrated from the recovered SHA, so it is now a full + // file on disk. Emit a distinct event (paired with #2074's *_MalformedBlobSha + // detection) so we can watch the corrupt-placeholder population drain in telemetry. + this.EmitMalformedShaRepaired(requestMetadata, hydrationSha); + } } catch (OperationCanceledException) { @@ -1321,6 +1400,16 @@ private void GetFileStreamHandlerAsyncHandler( catch (GetFileStreamException e) { requestMetadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(this.GetFileStreamHandlerAsyncHandler)}: GetFileStreamException HResult 0x{e.HResult:X8}"); + + // Funnel a post-recovery hydration failure that threw (size mismatch, local IO, ProjFS + // write failure). Exclude HResult.Handle: that means the application closed the file + // handle before hydration finished, so the repair attempt was aborted by the reader, not + // failed - counting it would inflate the repair-failure rate. + if (repairingMalformedSha && (HResult)e.HResult != HResult.Handle) + { + this.EmitMalformedShaRepairFailed(requestMetadata, MalformedShaRepairFailureReason.HydrateException, hydrationSha); + } + this.Context.Tracer.RelatedWarning(requestMetadata, nameof(this.GetFileStreamHandlerAsyncHandler) + "_GetFileStreamException"); this.TryCompleteCommand(commandId, (HResult)e.HResult); @@ -1330,6 +1419,12 @@ private void GetFileStreamHandlerAsyncHandler( { requestMetadata.Add("Exception", e.ToString()); requestMetadata.Add(nameof(GVFSGitObjects.BlobHydrationFailureCategory), GVFSGitObjects.BlobHydrationFailureCategory.Unexpected.ToString()); + + if (repairingMalformedSha) + { + this.EmitMalformedShaRepairFailed(requestMetadata, MalformedShaRepairFailureReason.HydrateException, hydrationSha); + } + this.Context.Tracer.RelatedError(requestMetadata, $"{nameof(this.GetFileStreamHandlerAsyncHandler)}: TryCopyBlobContentStream failed"); this.TryCompleteCommand(commandId, (HResult)HResultExtensions.HResultFromNtStatus.FileNotAvailable); @@ -1340,6 +1435,110 @@ private void GetFileStreamHandlerAsyncHandler( this.TryCompleteCommand(commandId, HResult.Ok); } + /// + /// The reason a corrupt-placeholder repair attempt failed, tagged on the + /// _MalformedBlobShaRepairFailed telemetry event so the failure population is diagnosable. + /// + private enum MalformedShaRepairFailureReason + { + ProjectionMiss, // The path is no longer projected as a file, so no authoritative SHA exists. + ProjectionException, // The projection lookup threw while recovering the SHA. + HydrateFailed, // A SHA was recovered but TryCopyBlobContentStream reported a clean miss. + HydrateException, // A SHA was recovered but hydration threw (see BlobHydrationFailureCategory). + } + + /// + /// Recovers the authoritative blob SHA for a corrupt placeholder from the git index projection. + /// A corrupt placeholder presents a malformed content-id (for example 40 NUL characters), but the + /// path is still projected, so the projection can supply the correct SHA. Returns true and sets + /// when a SHA is found; otherwise emits a distinct repair-failed + /// telemetry event (with the reason) and returns false so the caller can fail the read cleanly + /// instead of crashing. Never swallows . + /// + private bool TryRecoverShaForMalformedPlaceholder( + string virtualPath, + CancellationToken cancellationToken, + EventMetadata requestMetadata, + out string recoveredSha) + { + recoveredSha = null; + MalformedShaRepairFailureReason repairFailedReason; + + try + { + // Recover the authoritative SHA for this path from the git index projection. Pass a null + // BlobSizesConnection: repair needs only the SHA, not the blob size, so size resolution + // (which can throw SizesUnavailableException) is skipped - a size-lookup fault must not + // deny a SHA-only self-heal. + // + // The projection is read live, so a concurrent checkout / projection update can change + // the SHA projected for this path between when ProjFS opened the placeholder and now. + // Serving the currently-projected SHA is the best available answer for an already-corrupt + // placeholder and matches the normal placeholder-creation path, which reads the same live + // projection; the corrupt content-id carries no trustworthy prior revision to prefer. + ProjectedFileInfo fileInfo = this.FileSystemCallbacks.GitIndexProjection.GetProjectedFileInfo( + cancellationToken, + blobSizesConnection: null, + virtualPath, + out string _); + + if (fileInfo != null && !fileInfo.IsFolder) + { + // ProjectedFileInfo.Sha is a strongly-typed Sha1Id, which always renders as 40 hex + // characters, so the recovered SHA is well-formed by construction. + recoveredSha = fileInfo.Sha.ToString(); + return true; + } + + // The path is no longer projected as a file (for example it was deleted or renamed), so + // there is no authoritative SHA to repair it with. + repairFailedReason = MalformedShaRepairFailureReason.ProjectionMiss; + } + catch (OperationCanceledException) + { + // Let the caller's handler treat cancellation exactly as it does for a normal read. + throw; + } + catch (Exception e) + { + // Any other projection failure is treated as an unrecoverable repair miss rather than a + // crash: we are already on the rare corrupt-placeholder branch, so failing cleanly is + // safer than propagating and exiting the mount. + repairFailedReason = MalformedShaRepairFailureReason.ProjectionException; + requestMetadata.Add("RepairException", e.ToString()); + } + + this.EmitMalformedShaRepairFailed(requestMetadata, repairFailedReason, recoveredSha: null); + return false; + } + + /// + /// Emits the telemetry event marking a corrupt placeholder as repaired (hydrated from the + /// recovered SHA). Paired with so the two events + /// together account for every repair attempt. + /// + private void EmitMalformedShaRepaired(EventMetadata requestMetadata, string recoveredSha) + { + requestMetadata["recoveredSha"] = SHA1Util.ToLoggableShaString(recoveredSha); + this.Context.Tracer.RelatedEvent(EventLevel.Warning, MalformedShaRepairedEventName, requestMetadata, Keywords.Telemetry); + } + + /// + /// Emits the telemetry event marking a corrupt-placeholder repair attempt as failed, tagged with + /// the . Emitted on every repair-failure exit so that + /// (repaired + repair-failed) accounts for every non-aborted repair attempt. + /// + private void EmitMalformedShaRepairFailed(EventMetadata requestMetadata, MalformedShaRepairFailureReason reason, string recoveredSha) + { + requestMetadata["RepairFailedReason"] = reason.ToString(); + if (recoveredSha != null) + { + requestMetadata["recoveredSha"] = SHA1Util.ToLoggableShaString(recoveredSha); + } + + this.Context.Tracer.RelatedEvent(EventLevel.Warning, MalformedShaRepairFailedEventName, requestMetadata, Keywords.Telemetry); + } + private void NotifyNewFileCreatedHandler( string virtualPath, bool isDirectory, diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs index d933584e94..90900cf55e 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs @@ -10,6 +10,10 @@ public class MockTracer : ITracer { private AutoResetEvent waitEvent; + // Guards the parallel RelatedEventNames / RelatedEventMetadata lists so their appends stay + // index-aligned even if two callbacks report events concurrently. + private readonly object relatedEventLock = new object(); + public MockTracer() { this.waitEvent = new AutoResetEvent(false); @@ -17,6 +21,7 @@ public MockTracer() this.RelatedWarningEvents = new List(); this.RelatedErrorEvents = new List(); this.RelatedEventNames = new List(); + this.RelatedEventMetadata = new List(); } public MockTracer StartActivityTracer { get; private set; } @@ -30,6 +35,10 @@ public MockTracer() // do not otherwise get recorded). Lets tests assert a specific diagnostic event fired. public List RelatedEventNames { get; } + // Serialized metadata for each RelatedEvent call, parallel to RelatedEventNames by index. + // Lets tests assert on a diagnostic event's payload (for example a RepairFailedReason value). + public List RelatedEventMetadata { get; } + public void WaitForRelatedEvent() { this.waitEvent.WaitOne(); @@ -37,7 +46,12 @@ public void WaitForRelatedEvent() public void RelatedEvent(EventLevel error, string eventName, EventMetadata metadata) { - this.RelatedEventNames.Add(eventName); + lock (this.relatedEventLock) + { + this.RelatedEventNames.Add(eventName); + this.RelatedEventMetadata.Add(metadata != null ? GVFSJsonOptions.Serialize(metadata) : string.Empty); + } + if (eventName == this.WaitRelatedEventName) { this.waitEvent.Set(); @@ -46,7 +60,12 @@ public void RelatedEvent(EventLevel error, string eventName, EventMetadata metad public void RelatedEvent(EventLevel error, string eventName, EventMetadata metadata, Keywords keyword) { - this.RelatedEventNames.Add(eventName); + lock (this.relatedEventLock) + { + this.RelatedEventNames.Add(eventName); + this.RelatedEventMetadata.Add(metadata != null ? GVFSJsonOptions.Serialize(metadata) : string.Empty); + } + if (eventName == this.WaitRelatedEventName) { this.waitEvent.Set(); diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGVFSGitObjects.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGVFSGitObjects.cs index bb34332875..f253521d4d 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockGVFSGitObjects.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGVFSGitObjects.cs @@ -21,9 +21,14 @@ public MockGVFSGitObjects(GVFSContext context, GitObjectsHttpRequestor httpGitOb public bool CancelTryCopyBlobContentStream { get; set; } public bool ThrowOnTryCopyBlobContentStream { get; set; } + public bool ReturnFalseFromTryCopyBlobContentStream { get; set; } public bool ThrowIOExceptionDuringCopy { get; set; } public uint FileLength { get; set; } = DefaultFileLength; + // Records the sha most recently passed to TryCopyBlobContentStream so a test can assert that a + // repaired corrupt placeholder hydrated from the RECOVERED sha, not the malformed content-id. + public string LastShaPassedToTryCopyBlobContentStream { get; private set; } + public override bool TryDownloadCommit(string objectSha) { RetryWrapper.InvocationResult result = this.GitObjectRequestor.TryDownloadObjects( @@ -48,6 +53,7 @@ public override bool TryCopyBlobContentStream( Action writeAction, out GVFSGitObjects.BlobHydrationFailureCategory failureCategory) { + this.LastShaPassedToTryCopyBlobContentStream = sha; failureCategory = GVFSGitObjects.BlobHydrationFailureCategory.None; if (this.CancelTryCopyBlobContentStream) @@ -62,6 +68,15 @@ public override bool TryCopyBlobContentStream( throw new InvalidOperationException("Simulated unexpected hydration failure"); } + if (this.ReturnFalseFromTryCopyBlobContentStream) + { + // A clean (no-exception) hydration miss: the blob is unavailable. Exercises the + // TryCopyBlobContentStream-returned-false path (used to test a repair that recovered a + // valid SHA but still could not hydrate it). + failureCategory = GVFSGitObjects.BlobHydrationFailureCategory.DownloadFailed; + return false; + } + if (this.ThrowIOExceptionDuringCopy) { // The served length matches the requested length (so no size mismatch), but reading diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index 143cad6129..d53f59ef5a 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -63,6 +63,12 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } + public bool ThrowExceptionOnProjectionRequest { get; set; } + + // The SHA that GetProjectedFileInfo returns for a projected file. Exposed so a test can assert + // that the corrupt-placeholder repair hydrates from THIS recovered SHA (and not the corrupt one). + public Sha1Id ProjectedFileSha { get; set; } = new Sha1Id(1, 1, 1); + public bool ProjectionParseComplete { get; set; } public PathSparseState GetFolderPathSparseStateValue { get; set; } = PathSparseState.Included; @@ -243,6 +249,11 @@ public override ProjectedFileInfo GetProjectedFileInfo( throw new OperationCanceledException(); } + if (this.ThrowExceptionOnProjectionRequest) + { + throw new InvalidOperationException("Simulated projection failure"); + } + this.unblockGetProjectedFileInfo.WaitOne(); if (this.projectedFiles.Contains(virtualPath)) @@ -251,7 +262,7 @@ public override ProjectedFileInfo GetProjectedFileInfo( string parentKey; this.GetChildNameAndParentKey(virtualPath, out childName, out parentKey); parentFolderPath = parentKey; - return new ProjectedFileInfo(childName, size: 0, isFolder: false, sha: new Sha1Id(1, 1, 1)); + return new ProjectedFileInfo(childName, size: 0, isFolder: false, sha: this.ProjectedFileSha); } parentFolderPath = null; diff --git a/GVFS/GVFS.UnitTests/Windows/Mock/WindowsFileSystemVirtualizerTester.cs b/GVFS/GVFS.UnitTests/Windows/Mock/WindowsFileSystemVirtualizerTester.cs index d25b1131b8..20ea92ac47 100644 --- a/GVFS/GVFS.UnitTests/Windows/Mock/WindowsFileSystemVirtualizerTester.cs +++ b/GVFS/GVFS.UnitTests/Windows/Mock/WindowsFileSystemVirtualizerTester.cs @@ -23,20 +23,25 @@ public WindowsFileSystemVirtualizerTester(CommonRepoSetup repo, string[] project public MockVirtualizationInstance MockVirtualization { get; private set; } public WindowsFileSystemVirtualizer WindowsVirtualizer { get; private set; } - public void InvokeGetFileDataCallback(HResult expectedResult = HResult.Pending, byte[] providerId = null, ulong byteOffset = 0) + public void InvokeGetFileDataCallback(HResult expectedResult = HResult.Pending, byte[] providerId = null, ulong byteOffset = 0, byte[] contentId = null, string relativePath = "test.txt") { if (providerId == null) { providerId = WindowsFileSystemVirtualizer.PlaceholderVersionId; } + if (contentId == null) + { + contentId = CommonRepoSetup.DefaultContentId; + } + this.MockVirtualization.RequiredCallbacks.GetFileDataCallback( commandId: 1, - relativePath: "test.txt", + relativePath: relativePath, byteOffset: byteOffset, length: MockGVFSGitObjects.DefaultFileLength, dataStreamId: Guid.NewGuid(), - contentId: CommonRepoSetup.DefaultContentId, + contentId: contentId, providerId: providerId, triggeringProcessId: 2, triggeringProcessImageFileName: "UnitTest").ShouldEqual(expectedResult); diff --git a/GVFS/GVFS.UnitTests/Windows/Virtualization/WindowsFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Windows/Virtualization/WindowsFileSystemVirtualizerTests.cs index bfd4a0e094..ff4da05069 100644 --- a/GVFS/GVFS.UnitTests/Windows/Virtualization/WindowsFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Windows/Virtualization/WindowsFileSystemVirtualizerTests.cs @@ -15,6 +15,8 @@ using System.Threading.Tasks; using System.Linq; using GVFS.Common.Tracing; +using GVFS.Common; +using GVFS.Common.Git; namespace GVFS.UnitTests.Windows.Virtualization { @@ -722,6 +724,220 @@ public void OnGetFileStreamHandlesHResultHandleResult() } } + [TestCase] + public void OnGetFileStreamRepairsMalformedPlaceholderShaFromProjection() + { + // A corrupt placeholder presents an all-NUL content-id (decodes to a malformed SHA), but the + // path is still projected, so the read-time self-heal recovers the authoritative SHA from the + // projection and hydrates from that. The hydration writes the whole file, so the read succeeds + // and a distinct *_MalformedBlobShaRepaired event lets telemetry watch the corrupt population drain. + using (WindowsFileSystemVirtualizerTester tester = new WindowsFileSystemVirtualizerTester(this.Repo)) + { + tester.MockVirtualization.WriteFileReturnResult = HResult.Ok; + + // Give the projection a distinctive recovered SHA so we can prove hydration used IT and + // not the corrupt content-id. + Sha1Id recoveredShaId = new Sha1Id(0x1122334455667788, 0x99AABBCCDDEEFF00, 0x12345678); + tester.GitIndexProjection.ProjectedFileSha = recoveredShaId; + + // "test.txt" is the default projected file, so the projection returns the recovered SHA. + tester.InvokeGetFileDataCallback(expectedResult: HResult.Pending, contentId: CorruptContentId(), relativePath: "test.txt"); + + tester.MockVirtualization.WaitForCompletionStatus().ShouldEqual(HResult.Ok); + + // The crux of the feature: hydration must use the RECOVERED sha, not the malformed one. + MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; + mockGVFSGitObjects.LastShaPassedToTryCopyBlobContentStream.ShouldEqual(recoveredShaId.ToString()); + mockGVFSGitObjects.LastShaPassedToTryCopyBlobContentStream.ShouldNotEqual(new string('\0', GVFSConstants.ShaStringLength)); + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + mockTracker.RelatedEventNames.ShouldContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepaired"); + mockTracker.RelatedErrorEvents.ShouldBeEmpty(); + } + } + + [TestCase] + public void OnGetFileStreamFailsRepairWhenPathNotProjected() + { + // The placeholder is corrupt AND the path is no longer projected (for example it was deleted or + // renamed), so there is no authoritative SHA to repair it with. The read must fail cleanly (as + // #2074 does) and emit a distinct *_MalformedBlobShaRepairFailed event rather than crashing. + using (WindowsFileSystemVirtualizerTester tester = new WindowsFileSystemVirtualizerTester(this.Repo)) + { + tester.InvokeGetFileDataCallback( + expectedResult: HResult.Pending, + contentId: CorruptContentId(), + relativePath: "not-projected.txt"); + + tester.MockVirtualization.WaitForCompletionStatus() + .ShouldEqual((HResult)HResultExtensions.HResultFromNtStatus.FileNotAvailable); + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + AssertRepairFailedWithReason(mockTracker, "ProjectionMiss"); + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepaired"); + } + } + + [TestCase] + public void OnGetFileStreamFailsRepairWhenProjectionThrows() + { + // The projection lookup itself throws (non-cancellation) while recovering the SHA. The repair + // helper must catch it, fail the read cleanly, and funnel it as a distinct repair failure with + // reason ProjectionException rather than propagating and exiting the mount. + using (WindowsFileSystemVirtualizerTester tester = new WindowsFileSystemVirtualizerTester(this.Repo)) + { + tester.GitIndexProjection.ThrowExceptionOnProjectionRequest = true; + + tester.InvokeGetFileDataCallback(expectedResult: HResult.Pending, contentId: CorruptContentId(), relativePath: "test.txt"); + + tester.MockVirtualization.WaitForCompletionStatus() + .ShouldEqual((HResult)HResultExtensions.HResultFromNtStatus.FileNotAvailable); + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + AssertRepairFailedWithReason(mockTracker, "ProjectionException"); + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepaired"); + } + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void OnGetFileStreamRepairCancelledDuringProjectionLookup() + { + // Cancellation while the repair is looking up the projection must be treated exactly like a + // normal-read cancellation: the helper rethrows OperationCanceledException, the outer handler + // logs _OperationCancelled, and NO repair success/failure event is emitted (the attempt was + // aborted, not decided). + using (WindowsFileSystemVirtualizerTester tester = new WindowsFileSystemVirtualizerTester(this.Repo)) + { + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + mockTracker.WaitRelatedEventName = "GetFileStreamHandlerAsyncHandler_OperationCancelled"; + tester.GitIndexProjection.ThrowOperationCanceledExceptionOnProjectionRequest = true; + + tester.InvokeGetFileDataCallback(expectedResult: HResult.Pending, contentId: CorruptContentId(), relativePath: "test.txt"); + + mockTracker.WaitForRelatedEvent(); + + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepaired"); + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepairFailed"); + } + } + + [TestCase] + public void OnGetFileStreamFailsRepairWhenRecoveredShaCannotHydrate() + { + // The path is projected so a valid SHA is recovered, but the blob for that SHA cannot be + // hydrated (for example the object is unavailable). TryCopyBlobContentStream returns false + // (no exception), so the read fails cleanly and emits the *_MalformedBlobShaRepairFailed + // event so telemetry can separate "could not recover the SHA" from "recovered the SHA but + // the blob itself is unavailable". + using (WindowsFileSystemVirtualizerTester tester = new WindowsFileSystemVirtualizerTester(this.Repo)) + { + MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; + mockGVFSGitObjects.ReturnFalseFromTryCopyBlobContentStream = true; + + tester.InvokeGetFileDataCallback(expectedResult: HResult.Pending, contentId: CorruptContentId(), relativePath: "test.txt"); + + tester.MockVirtualization.WaitForCompletionStatus() + .ShouldEqual((HResult)HResultExtensions.HResultFromNtStatus.FileNotAvailable); + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + AssertRepairFailedWithReason(mockTracker, "HydrateFailed"); + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepaired"); + } + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void OnGetFileStreamFailsRepairWhenRecoveredShaHydrationThrows() + { + // A valid SHA is recovered, but hydration THROWS mid-write (here a size mismatch). This failure + // reaches the outer catch handlers, which must still funnel it as a repair failure (reason + // HydrateException) so that repaired + repair-failed accounts for every repair attempt. + using (WindowsFileSystemVirtualizerTester tester = new WindowsFileSystemVirtualizerTester(this.Repo)) + { + MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; + mockGVFSGitObjects.FileLength = MockGVFSGitObjects.DefaultFileLength - 1; + + tester.InvokeGetFileDataCallback(expectedResult: HResult.Pending, contentId: CorruptContentId(), relativePath: "test.txt"); + + tester.MockVirtualization.WaitForCompletionStatus(); + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + AssertRepairFailedWithReason(mockTracker, "HydrateException"); + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepaired"); + } + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void OnGetFileStreamFailsRepairWhenRecoveredShaHydrationThrowsGenericException() + { + // A valid SHA is recovered, but hydration throws a non-GetFileStreamException (here an + // InvalidOperationException). This reaches the generic catch (Exception) handler, which - like + // the GetFileStreamException handler - must still funnel it as a repair failure (reason + // HydrateException) so that repaired + repair-failed accounts for every non-aborted attempt. + using (WindowsFileSystemVirtualizerTester tester = new WindowsFileSystemVirtualizerTester(this.Repo)) + { + MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; + mockGVFSGitObjects.ThrowOnTryCopyBlobContentStream = true; + + tester.InvokeGetFileDataCallback(expectedResult: HResult.Pending, contentId: CorruptContentId(), relativePath: "test.txt"); + + tester.MockVirtualization.WaitForCompletionStatus() + .ShouldEqual((HResult)HResultExtensions.HResultFromNtStatus.FileNotAvailable); + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + AssertRepairFailedWithReason(mockTracker, "HydrateException"); + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepaired"); + } + } + + [TestCase] + public void OnGetFileStreamDoesNotRepairValidPlaceholderSha() + { + // Regression: a placeholder with a valid content-id hydrates normally and must NOT take the + // repair path or emit any repair telemetry. + using (WindowsFileSystemVirtualizerTester tester = new WindowsFileSystemVirtualizerTester(this.Repo)) + { + tester.MockVirtualization.WriteFileReturnResult = HResult.Ok; + + tester.InvokeGetFileDataCallback(expectedResult: HResult.Pending); + + tester.MockVirtualization.WaitForCompletionStatus().ShouldEqual(HResult.Ok); + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepaired"); + mockTracker.RelatedEventNames.ShouldNotContain( + name => name == "GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepairFailed"); + mockTracker.RelatedErrorEvents.ShouldBeEmpty(); + } + } + + // A corrupt placeholder's content-id: 80 zero bytes, which GetShaFromContentId decodes to a + // 40-character all-NUL string (a malformed SHA), mirroring the field corruption. + private static byte[] CorruptContentId() + { + return new byte[GVFSConstants.ShaStringLength * sizeof(char)]; + } + + // Asserts the repair-failed event fired AND carried the expected RepairFailedReason in its metadata. + private static void AssertRepairFailedWithReason(MockTracer mockTracker, string expectedReason) + { + int index = mockTracker.RelatedEventNames.IndexOf("GetFileStreamHandlerAsyncHandler_MalformedBlobShaRepairFailed"); + index.ShouldNotEqual(-1, "Expected a _MalformedBlobShaRepairFailed event to be emitted"); + mockTracker.RelatedEventMetadata[index].Contains(expectedReason).ShouldBeTrue( + $"Expected RepairFailedReason '{expectedReason}' in the repair-failed event metadata, was: {mockTracker.RelatedEventMetadata[index]}"); + } + [TestCase] public void OnStartDirectoryEnumerationCleansUpEnumerationWhenNativeCompletionFails() {