Better polling

src/Mindee/V2/Client.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.DependencyInjection;
@@ -36,17 +37,14 @@ public sealed class Client
         /// <param name="loggerFactory">Factory for the logger.</param>
         public Client(string apiKey, ILoggerFactory loggerFactory = null)
         {
-            var loggerFactory1 = loggerFactory ?? LoggerFactory.Create(builder =>
-            {
-                builder.SetMinimumLevel(LogLevel.Debug);
-            });
-            _logger = loggerFactory1.CreateLogger<Client>();
+            var loggerFactoryInstance = loggerFactory ?? NullLoggerFactory.Instance;
+            _logger = loggerFactoryInstance.CreateLogger<Client>();
 
             var serviceCollection = new ServiceCollection();
             serviceCollection.AddMindeeApiV2(options =>
             {
                 options.ApiKey = apiKey;
-            }, loggerFactory1);
+            }, loggerFactoryInstance);
 
             var serviceProvider = serviceCollection.BuildServiceProvider();
 
@@ -225,7 +223,10 @@ InputSource inputSource
                 inputSource,
                 parameters,
                 ct);
-            return await PollForProductResultsAsync<TResponse>(enqueueResponse, pollingOptions, ct);
+            _logger?.LogInformation(
+                "Successfully enqueued document with job ID {JobID}", enqueueResponse.Job.Id);
+            return await PollForProductResultsAsync<TResponse>(
+                enqueueResponse, pollingOptions, false, ct);
         }
 
         /// <summary>
@@ -236,7 +237,6 @@ InputSource inputSource
         /// <param name="parameters"><see cref="RagDocumentUploadParameters"/></param>
         /// <param name="inputSource"><see cref="LocalInputSource"/></param>
         /// <param name="ct"></param>
-        /// <returns></returns>
         public async Task<TAnnotationResponse> UploadRagDocumentAsync<TAnnotationResponse>(
             LocalInputSource inputSource
             , BaseRagDocumentUploadParameters<TAnnotationResponse> parameters
@@ -251,10 +251,9 @@ LocalInputSource inputSource
         /// Add a document to the RAG database and return the initial annotation.
         /// </summary>
         /// <param name="parameters"><see cref="RagDocumentUploadParameters"/></param>
-        /// <param name="inputSource"><see cref="LocalInputSource"/></param>
+        /// <param name="inputSource"><see cref="LocalInputSource"/> The file to upload.</param>
         /// <param name="pollingOptions"><see cref="PollingOptions"/></param>
         /// <param name="ct"></param>
-        /// <returns></returns>
         public async Task<TAnnotationResponse> UploadAndGetRagDocumentPollAsync<TAnnotationResponse>(
             LocalInputSource inputSource
             , BaseRagDocumentUploadParameters<TAnnotationResponse> parameters
@@ -277,7 +276,6 @@ LocalInputSource inputSource
         /// </summary>
         /// <param name="documentId"></param>
         /// <param name="ct"></param>
-        /// <returns></returns>
         public async Task<TAnnotationResponse> GetRagDocumentAsync<TAnnotationResponse>(
             string documentId, CancellationToken ct = default)
             where TAnnotationResponse : BaseRagAnnotationResponse, new()
@@ -289,7 +287,7 @@ public async Task<TAnnotationResponse> GetRagDocumentAsync<TAnnotationResponse>(
         /// <summary>
         /// Get a document's info and annotations from the RAG database.
         /// </summary>
-        /// <param name="documentId"></param>
+        /// <param name="documentId">The document's ID.</param>
         /// <param name="pollingOptions"/>
         /// <param name="ct"></param>
         /// <returns></returns>
@@ -308,6 +306,8 @@ string documentId
         }
 
         /// <summary>
+        /// Not recommended for general use, prefer <see cref="UpdateAndGetRagAnnotationPollAsync{TAnnotationResponse}"/>.
+        /// You will need to poll until the document is ready for use.
         /// Update a document's annotations in the RAG database.
         /// </summary>
         /// <param name="parameters"></param>
@@ -334,8 +334,7 @@ BaseAnnotationParameters<TAnnotationResponse> parameters
             , CancellationToken ct = default)
             where TAnnotationResponse : ExtractionRagAnnotationResponse, new()
         {
-            _logger?.LogInformation("Updating RAG document ID: {DocumentId}", parameters.DocumentId);
-            var initialResponse = await _mindeeApi.ReqPatchRagAnnotationAsync(parameters, ct);
+            var initialResponse = await UpdateRagAnnotationAsync(parameters, ct);
             if (initialResponse.Status != "Processing")
                 return initialResponse;
 
@@ -348,7 +347,7 @@ BaseAnnotationParameters<TAnnotationResponse> parameters
         /// Delete a document from the RAG database.
         /// For extraction models only.
         /// </summary>
-        /// <param name="documentId"></param>
+        /// <param name="documentId">The document's ID.</param>
         /// <param name="ct"></param>
         /// <returns></returns>
         public async Task<bool> DeleteExtractionRagDocumentAsync(
@@ -412,7 +411,7 @@ BaseRagAnnotationResponse initialResponse
             {
                 var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(retryCount);
                 await Task.Delay(retryDelayMilliSec, cancellationToken);
-                _logger?.LogInformation(
+                _logger?.LogDebug(
                     "Poll attempt {RetryCount} of {MaxRetries}",
                     retryCount,
                     maxRetries);
@@ -425,7 +424,7 @@ BaseRagAnnotationResponse initialResponse
                     case "Processing":
                         continue;
                     case "Failed":
-                        throw new MindeeException("Job failed without an error payload.");
+                        throw new MindeeException("RAG failed without an error payload.");
                     default:
                         return response;
                 }
@@ -434,60 +433,106 @@ BaseRagAnnotationResponse initialResponse
         }
 
         /// <summary>
-        /// Poll until the inference results are retrieved or the max number of attempts is reached.
+        /// Checks if all webhooks associated with a job have finished processing.
         /// </summary>
-        /// <exception cref="MindeeException">Thrown when maxRetries is reached and the result isn't ready.</exception>
-        private async Task<TResponse> PollForProductResultsAsync<TResponse>(
-            JobResponse enqueueResponse,
+        private bool CheckWebhooksDone(JobResponse jobResponse)
+        {
+            var areWebhooksDone = jobResponse.Job.Webhooks == null ||
+                                  jobResponse.Job.Webhooks.All(w => w.Status != "Processing");
+
+            if (areWebhooksDone)
+            {
+                _logger?.LogDebug("All webhooks are completed.");
+                return true;
+            }
+
+            _logger?.LogDebug("Not all webhooks are completed.");
+            return false;
+        }
+
+        /// <summary>
+        /// Polls a job until it is processed or the maximum number of tries is reached.
+        /// </summary>
+        private async Task<JobResponse> PollOnJobAsync(
+            JobResponse initialResponse,
             PollingOptions pollingOptions,
+            bool waitForWebhooks = false,
             CancellationToken cancellationToken = default)
-            where TResponse : BaseResponse, new()
         {
-            _logger?.LogInformation("Polling for results on job ID: {JobID}", enqueueResponse.Job.Id);
-            var maxRetries = pollingOptions.MaxRetries + 1;
-            var pollingUrl = enqueueResponse.Job.PollingUrl;
             _logger?.LogDebug(
-                "Waiting {} seconds before attempting to retrieve the result...",
+                "Waiting {InitialDelaySec} seconds before attempting to retrieve the result...",
                 pollingOptions.InitialDelaySec);
+
             await Task.Delay(pollingOptions.InitialDelayMilliSec, cancellationToken);
-            var retryCount = 1;
-            var response = enqueueResponse; // First init is only for error handling purposes.
-            while (retryCount < maxRetries)
+            var tryCounter = 0;
+            var maxRetries = pollingOptions.MaxRetries;
+            var pollingUrl = initialResponse.Job.PollingUrl;
+
+            while (tryCounter < maxRetries)
             {
-                var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(retryCount);
-                await Task.Delay(retryDelayMilliSec, cancellationToken);
-                _logger?.LogInformation(
+                cancellationToken.ThrowIfCancellationRequested();
+
+                _logger?.LogDebug(
                     "Poll attempt {RetryCount} of {MaxRetries}",
-                    retryCount,
+                    tryCounter + 1,
                     maxRetries);
 
-                response = await GetJobFromUrlAsync(pollingUrl, cancellationToken);
-                if (response.Job.Error != null)
+                var jobResponse = await GetJobFromUrlAsync(pollingUrl, cancellationToken);
+
+                if (jobResponse.Job.Status == "Processed")
                 {
-                    break;
+                    _logger?.LogDebug(
+                        "Job ID {JobID} completed processing at: {CompletedAt}",
+                        jobResponse.Job.Id,
+                        jobResponse.Job.CompletedAt);
+
+                    if (!waitForWebhooks || CheckWebhooksDone(jobResponse))
+                    {
+                        return jobResponse;
+                    }
                 }
 
-                switch (response.Job.Status)
+                // normally the mindee_api will throw on error, this is a fallback
+                if (jobResponse.Job.Status == "Failed")
                 {
-                    case "Processed":
-                        {
-                            var resultUrl = response.Job.ResultUrl;
-                            return await GetResultFromUrlAsync<TResponse>(resultUrl, cancellationToken);
-                        }
-                    case "Failed":
-                        throw new MindeeException("Job failed without an error payload.");
+                    if (jobResponse.Job.Error != null)
+                    {
+                        throw new MindeeHttpExceptionV2(jobResponse.Job.Error);
+                    }
+                    throw new MindeeException($"Parsing failed for job {jobResponse.Job.Id}: No error detail available.");
                 }
 
-                retryCount++;
+                tryCounter++;
+                var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(tryCounter);
+                await Task.Delay(retryDelayMilliSec, cancellationToken);
             }
 
-            var error = response.Job.Error;
-            if (error != null)
+            throw new MindeeException($"Couldn't retrieve the result after {tryCounter} tries.");
+        }
+
+        /// <summary>
+        /// Poll until the inference results are retrieved or the max number of attempts is reached.
+        /// </summary>
+        private async Task<TResponse> PollForProductResultsAsync<TResponse>(
+            JobResponse enqueueResponse,
+            PollingOptions pollingOptions,
+            bool waitForWebhooks = false,
+            CancellationToken cancellationToken = default)
+            where TResponse : BaseResponse, new()
+        {
+            var jobResponse = await PollOnJobAsync(
+                enqueueResponse,
+                pollingOptions,
+                waitForWebhooks,
+                cancellationToken);
+
+            if (string.IsNullOrWhiteSpace(jobResponse.Job.ResultUrl))
             {
-                throw new MindeeHttpExceptionV2(error);
+                throw new MindeeException(
+                    "The result URL is undefined. This is a server error, try again later or contact support.");
             }
 
-            throw new MindeeException($"Result polling not complete after {retryCount} attempts.");
+            return await GetResultFromUrlAsync<TResponse>(jobResponse.Job.ResultUrl, cancellationToken);
         }
     }
 }

src/Mindee/V2/Http/HttpApiV2.cs

@@ -122,7 +122,7 @@ public abstract Task<TAnnotationResponse> ReqPatchRagAnnotationAsync<TAnnotation
         ///     Deletes a document from the RAG database.
         ///     For extraction models only.
         /// </summary>
-        /// <param name="documentId"></param>
+        /// <param name="documentId">The ID of the document to delete.</param>
         /// <param name="cancellationToken"></param>
         public abstract Task<bool> ReqDeleteExtractionRagDocumentAsync(
             string documentId, CancellationToken cancellationToken = default);

src/Mindee/V2/Http/MindeeApiV2.cs

@@ -14,7 +14,6 @@
 using Mindee.V2.Parsing;
 using Mindee.V2.Parsing.Search;
 using Mindee.V2.Product;
-using Mindee.V2.Product.Extraction.RagDocuments.Params;
 using Mindee.V2.Search.Models;
 using RestSharp;
 #if NET6_0_OR_GREATER
@@ -65,7 +64,7 @@ InputSource inputSource
 
             AddPredictRequestParameters(inputSource, parameters, request);
 
-            Logger?.LogInformation("HTTP POST to {RequestResource} ...", _baseUrl + request.Resource);
+            Logger?.LogDebug("HTTP POST to {RequestResource} ...", _baseUrl + request.Resource);
             var response = await _httpClient.ExecuteAsync(request, cancellationToken);
             cancellationToken.ThrowIfCancellationRequested();
             return HandleJobResponse(response);
@@ -80,7 +79,7 @@ public override async Task<TSearchResponse> ReqGetSearchAsync<TSearchResponse>(
                 throw new Exception($"ProductAttributes must be set for class: {searchType.Name}");
 
             var request = new RestRequest($"/v2/search/{productAttributes.Slug}");
-            Logger?.LogInformation("Searching {} ...", productAttributes.Slug);
+            Logger?.LogInformation("Searching {Slug} ...", productAttributes.Slug);
 
             foreach (KeyValuePair<string, string> entry in parameters.GetRequestParameters())
             {
@@ -109,13 +108,17 @@ BaseRagDocumentUploadParameters<TAnnotationResponse> parameters
             {
                 request.AddParameter(entry.Key, entry.Value);
             }
+
+            Logger?.LogDebug("HTTP POST to {RequestResource} ...", request.Resource);
             return await ExecuteRagAnnotationRequest<TAnnotationResponse>(request, cancellationToken);
         }
 
         public override async Task<TAnnotationResponse> ReqGetRagAnnotationAsync<TAnnotationResponse>(
             string documentId, CancellationToken cancellationToken = default)
         {
             var request = new RestRequest($"/v2/products/extraction/rag-documents/{documentId}");
+
+            Logger?.LogDebug("HTTP GET to {RequestResource} ...", request.Resource);
             return await ExecuteRagAnnotationRequest<TAnnotationResponse>(request, cancellationToken);
         }
 
@@ -126,6 +129,7 @@ public override async Task<TAnnotationResponse> ReqPatchRagAnnotationAsync<TAnno
                 $"/v2/products/extraction/rag-documents/{parameters.DocumentId}", Method.Patch);
             request.AddJsonBody(parameters.GetRequestParameters());
 
+            Logger?.LogDebug("HTTP PATCH to {RequestResource} ...", request.Resource);
             return await ExecuteRagAnnotationRequest<TAnnotationResponse>(request, cancellationToken);
         }
 
@@ -135,6 +139,7 @@ public override async Task<bool> ReqDeleteExtractionRagDocumentAsync(
             var request = new RestRequest(
                 $"/v2/products/extraction/rag-documents/{documentId}", Method.Delete);
 
+            Logger?.LogDebug("HTTP DELETE to {RequestResource} ...", request.Resource);
             var restResponse = await _httpClient.ExecuteAsync(request, cancellationToken);
             cancellationToken.ThrowIfCancellationRequested();
 
@@ -175,10 +180,9 @@ public override async Task<JobResponse> ReqGetJobByUrlAsync(
                 throw new ArgumentException("Job URL cannot be null or empty.", nameof(pollingUrl));
 
             var request = new RestRequest(new Uri(pollingUrl));
-            Logger?.LogInformation("HTTP GET to {RequestResource}...", request.Resource);
+            Logger?.LogDebug("HTTP GET to {RequestResource}...", request.Resource);
             var response = await _httpClient.ExecuteAsync(request, cancellationToken);
             cancellationToken.ThrowIfCancellationRequested();
-            Logger?.LogDebug("HTTP response: {ResponseContent}", response.Content);
             var handledResponse = HandleJobResponse(response);
             return handledResponse;
         }
@@ -190,7 +194,7 @@ public override async Task<TResponse> ReqGetResultByIdAsync<TResponse>(string in
                 throw new Exception($"ProductAttributes must be set for class: {typeof(TResponse).Name}");
             var request = new RestRequest(
                 $"/v2/products/{productAttributes.Slug}/results/{inferenceId}");
-            Logger?.LogInformation("HTTP GET to {RequestResource}...", request.Resource);
+            Logger?.LogDebug("HTTP GET to {RequestResource}...", request.Resource);
             var queueResponse = await _httpClient.ExecuteAsync(request, ct);
             ct.ThrowIfCancellationRequested();
             return HandleProductResponse<TResponse>(queueResponse);
@@ -203,7 +207,7 @@ public override async Task<TResponse> ReqGetResultByUrlAsync<TResponse>(
                 throw new ArgumentException("Result URL cannot be null or whitespace.", nameof(resultUrl));
 
             var request = new RestRequest(new Uri(resultUrl));
-            Logger?.LogInformation("HTTP GET to {RequestResource}...", resultUrl);
+            Logger?.LogDebug("HTTP GET to {RequestResource}...", resultUrl);
             var queueResponse = await _httpClient.ExecuteAsync(request, cancellationToken);
             cancellationToken.ThrowIfCancellationRequested();
             return HandleProductResponse<TResponse>(queueResponse);

tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs

@@ -19,7 +19,7 @@ public RagDocumentsTest()
             _extractionModelId = Environment.GetEnvironmentVariable("MindeeV2__Findoc__Model__Id");
         }
 
-        [Fact(Timeout = 180000)]
+        [Fact(Timeout = 180000, DisplayName = "should perform the entire lifecycle of a RAG document")]
         public async Task RagDocument_Lifecycle_MustSucceed()
         {
             var inputSource = new LocalInputSource(

tests/Mindee.UnitTests/V2/Parsing/FailedInferenceResponseTest.cs

@@ -7,7 +7,7 @@ namespace Mindee.UnitTests.V2.Parsing
     [Trait("Category", "FailedInferenceResponse")]
     public class FailedInferenceResponseTest
     {
-        [Fact]
+        [Fact(DisplayName = "should load from a JSON file")]
         public void WhenFailed_MustLoad()
         {
             var localResponse = new LocalResponse(