diff --git a/src/Mindee/V2/Client.cs b/src/Mindee/V2/Client.cs
index 32dc8eed..facf20c3 100644
--- a/src/Mindee/V2/Client.cs
+++ b/src/Mindee/V2/Client.cs
@@ -1,4 +1,5 @@
using System;
+using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
@@ -36,17 +37,14 @@ public sealed class Client
/// Factory for the logger.
public Client(string apiKey, ILoggerFactory loggerFactory = null)
{
- var loggerFactory1 = loggerFactory ?? LoggerFactory.Create(builder =>
- {
- builder.SetMinimumLevel(LogLevel.Debug);
- });
- _logger = loggerFactory1.CreateLogger();
+ var loggerFactoryInstance = loggerFactory ?? NullLoggerFactory.Instance;
+ _logger = loggerFactoryInstance.CreateLogger();
var serviceCollection = new ServiceCollection();
serviceCollection.AddMindeeApiV2(options =>
{
options.ApiKey = apiKey;
- }, loggerFactory1);
+ }, loggerFactoryInstance);
var serviceProvider = serviceCollection.BuildServiceProvider();
@@ -225,7 +223,10 @@ InputSource inputSource
inputSource,
parameters,
ct);
- return await PollForProductResultsAsync(enqueueResponse, pollingOptions, ct);
+ _logger?.LogInformation(
+ "Successfully enqueued document with job ID {JobID}", enqueueResponse.Job.Id);
+ return await PollForProductResultsAsync(
+ enqueueResponse, pollingOptions, false, ct);
}
///
@@ -236,7 +237,6 @@ InputSource inputSource
///
///
///
- ///
public async Task UploadRagDocumentAsync(
LocalInputSource inputSource
, BaseRagDocumentUploadParameters parameters
@@ -251,10 +251,9 @@ LocalInputSource inputSource
/// Add a document to the RAG database and return the initial annotation.
///
///
- ///
+ /// The file to upload.
///
///
- ///
public async Task UploadAndGetRagDocumentPollAsync(
LocalInputSource inputSource
, BaseRagDocumentUploadParameters parameters
@@ -277,7 +276,6 @@ LocalInputSource inputSource
///
///
///
- ///
public async Task GetRagDocumentAsync(
string documentId, CancellationToken ct = default)
where TAnnotationResponse : BaseRagAnnotationResponse, new()
@@ -289,7 +287,7 @@ public async Task GetRagDocumentAsync(
///
/// Get a document's info and annotations from the RAG database.
///
- ///
+ /// The document's ID.
///
///
///
@@ -308,6 +306,8 @@ string documentId
}
///
+ /// Not recommended for general use, prefer .
+ /// You will need to poll until the document is ready for use.
/// Update a document's annotations in the RAG database.
///
///
@@ -334,8 +334,7 @@ BaseAnnotationParameters parameters
, CancellationToken ct = default)
where TAnnotationResponse : ExtractionRagAnnotationResponse, new()
{
- _logger?.LogInformation("Updating RAG document ID: {DocumentId}", parameters.DocumentId);
- var initialResponse = await _mindeeApi.ReqPatchRagAnnotationAsync(parameters, ct);
+ var initialResponse = await UpdateRagAnnotationAsync(parameters, ct);
if (initialResponse.Status != "Processing")
return initialResponse;
@@ -348,7 +347,7 @@ BaseAnnotationParameters parameters
/// Delete a document from the RAG database.
/// For extraction models only.
///
- ///
+ /// The document's ID.
///
///
public async Task DeleteExtractionRagDocumentAsync(
@@ -412,7 +411,7 @@ BaseRagAnnotationResponse initialResponse
{
var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(retryCount);
await Task.Delay(retryDelayMilliSec, cancellationToken);
- _logger?.LogInformation(
+ _logger?.LogDebug(
"Poll attempt {RetryCount} of {MaxRetries}",
retryCount,
maxRetries);
@@ -425,7 +424,7 @@ BaseRagAnnotationResponse initialResponse
case "Processing":
continue;
case "Failed":
- throw new MindeeException("Job failed without an error payload.");
+ throw new MindeeException("RAG failed without an error payload.");
default:
return response;
}
@@ -434,60 +433,106 @@ BaseRagAnnotationResponse initialResponse
}
///
- /// Poll until the inference results are retrieved or the max number of attempts is reached.
+ /// Checks if all webhooks associated with a job have finished processing.
///
- /// Thrown when maxRetries is reached and the result isn't ready.
- private async Task PollForProductResultsAsync(
- JobResponse enqueueResponse,
+ private bool CheckWebhooksDone(JobResponse jobResponse)
+ {
+ var areWebhooksDone = jobResponse.Job.Webhooks == null ||
+ jobResponse.Job.Webhooks.All(w => w.Status != "Processing");
+
+ if (areWebhooksDone)
+ {
+ _logger?.LogDebug("All webhooks are completed.");
+ return true;
+ }
+
+ _logger?.LogDebug("Not all webhooks are completed.");
+ return false;
+ }
+
+ ///
+ /// Polls a job until it is processed or the maximum number of tries is reached.
+ ///
+ private async Task PollOnJobAsync(
+ JobResponse initialResponse,
PollingOptions pollingOptions,
+ bool waitForWebhooks = false,
CancellationToken cancellationToken = default)
- where TResponse : BaseResponse, new()
{
- _logger?.LogInformation("Polling for results on job ID: {JobID}", enqueueResponse.Job.Id);
- var maxRetries = pollingOptions.MaxRetries + 1;
- var pollingUrl = enqueueResponse.Job.PollingUrl;
_logger?.LogDebug(
- "Waiting {} seconds before attempting to retrieve the result...",
+ "Waiting {InitialDelaySec} seconds before attempting to retrieve the result...",
pollingOptions.InitialDelaySec);
+
await Task.Delay(pollingOptions.InitialDelayMilliSec, cancellationToken);
- var retryCount = 1;
- var response = enqueueResponse; // First init is only for error handling purposes.
- while (retryCount < maxRetries)
+ var tryCounter = 0;
+ var maxRetries = pollingOptions.MaxRetries;
+ var pollingUrl = initialResponse.Job.PollingUrl;
+
+ while (tryCounter < maxRetries)
{
- var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(retryCount);
- await Task.Delay(retryDelayMilliSec, cancellationToken);
- _logger?.LogInformation(
+ cancellationToken.ThrowIfCancellationRequested();
+
+ _logger?.LogDebug(
"Poll attempt {RetryCount} of {MaxRetries}",
- retryCount,
+ tryCounter + 1,
maxRetries);
- response = await GetJobFromUrlAsync(pollingUrl, cancellationToken);
- if (response.Job.Error != null)
+ var jobResponse = await GetJobFromUrlAsync(pollingUrl, cancellationToken);
+
+ if (jobResponse.Job.Status == "Processed")
{
- break;
+ _logger?.LogDebug(
+ "Job ID {JobID} completed processing at: {CompletedAt}",
+ jobResponse.Job.Id,
+ jobResponse.Job.CompletedAt);
+
+ if (!waitForWebhooks || CheckWebhooksDone(jobResponse))
+ {
+ return jobResponse;
+ }
}
- switch (response.Job.Status)
+ // normally the mindee_api will throw on error, this is a fallback
+ if (jobResponse.Job.Status == "Failed")
{
- case "Processed":
- {
- var resultUrl = response.Job.ResultUrl;
- return await GetResultFromUrlAsync(resultUrl, cancellationToken);
- }
- case "Failed":
- throw new MindeeException("Job failed without an error payload.");
+ if (jobResponse.Job.Error != null)
+ {
+ throw new MindeeHttpExceptionV2(jobResponse.Job.Error);
+ }
+ throw new MindeeException($"Parsing failed for job {jobResponse.Job.Id}: No error detail available.");
}
- retryCount++;
+ tryCounter++;
+ var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(tryCounter);
+ await Task.Delay(retryDelayMilliSec, cancellationToken);
}
- var error = response.Job.Error;
- if (error != null)
+ throw new MindeeException($"Couldn't retrieve the result after {tryCounter} tries.");
+ }
+
+ ///
+ /// Poll until the inference results are retrieved or the max number of attempts is reached.
+ ///
+ private async Task PollForProductResultsAsync(
+ JobResponse enqueueResponse,
+ PollingOptions pollingOptions,
+ bool waitForWebhooks = false,
+ CancellationToken cancellationToken = default)
+ where TResponse : BaseResponse, new()
+ {
+ var jobResponse = await PollOnJobAsync(
+ enqueueResponse,
+ pollingOptions,
+ waitForWebhooks,
+ cancellationToken);
+
+ if (string.IsNullOrWhiteSpace(jobResponse.Job.ResultUrl))
{
- throw new MindeeHttpExceptionV2(error);
+ throw new MindeeException(
+ "The result URL is undefined. This is a server error, try again later or contact support.");
}
- throw new MindeeException($"Result polling not complete after {retryCount} attempts.");
+ return await GetResultFromUrlAsync(jobResponse.Job.ResultUrl, cancellationToken);
}
}
}
diff --git a/src/Mindee/V2/Http/HttpApiV2.cs b/src/Mindee/V2/Http/HttpApiV2.cs
index 17735b1e..1825669d 100644
--- a/src/Mindee/V2/Http/HttpApiV2.cs
+++ b/src/Mindee/V2/Http/HttpApiV2.cs
@@ -122,7 +122,7 @@ public abstract Task ReqPatchRagAnnotationAsync
- ///
+ /// The ID of the document to delete.
///
public abstract Task ReqDeleteExtractionRagDocumentAsync(
string documentId, CancellationToken cancellationToken = default);
diff --git a/src/Mindee/V2/Http/MindeeApiV2.cs b/src/Mindee/V2/Http/MindeeApiV2.cs
index 1a520e82..c2887dd7 100644
--- a/src/Mindee/V2/Http/MindeeApiV2.cs
+++ b/src/Mindee/V2/Http/MindeeApiV2.cs
@@ -14,7 +14,6 @@
using Mindee.V2.Parsing;
using Mindee.V2.Parsing.Search;
using Mindee.V2.Product;
-using Mindee.V2.Product.Extraction.RagDocuments.Params;
using Mindee.V2.Search.Models;
using RestSharp;
#if NET6_0_OR_GREATER
@@ -65,7 +64,7 @@ InputSource inputSource
AddPredictRequestParameters(inputSource, parameters, request);
- Logger?.LogInformation("HTTP POST to {RequestResource} ...", _baseUrl + request.Resource);
+ Logger?.LogDebug("HTTP POST to {RequestResource} ...", _baseUrl + request.Resource);
var response = await _httpClient.ExecuteAsync(request, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
return HandleJobResponse(response);
@@ -80,7 +79,7 @@ public override async Task ReqGetSearchAsync(
throw new Exception($"ProductAttributes must be set for class: {searchType.Name}");
var request = new RestRequest($"/v2/search/{productAttributes.Slug}");
- Logger?.LogInformation("Searching {} ...", productAttributes.Slug);
+ Logger?.LogInformation("Searching {Slug} ...", productAttributes.Slug);
foreach (KeyValuePair entry in parameters.GetRequestParameters())
{
@@ -109,6 +108,8 @@ BaseRagDocumentUploadParameters parameters
{
request.AddParameter(entry.Key, entry.Value);
}
+
+ Logger?.LogDebug("HTTP POST to {RequestResource} ...", request.Resource);
return await ExecuteRagAnnotationRequest(request, cancellationToken);
}
@@ -116,6 +117,8 @@ public override async Task ReqGetRagAnnotationAsync(request, cancellationToken);
}
@@ -126,6 +129,7 @@ public override async Task ReqPatchRagAnnotationAsync(request, cancellationToken);
}
@@ -135,6 +139,7 @@ public override async Task ReqDeleteExtractionRagDocumentAsync(
var request = new RestRequest(
$"/v2/products/extraction/rag-documents/{documentId}", Method.Delete);
+ Logger?.LogDebug("HTTP DELETE to {RequestResource} ...", request.Resource);
var restResponse = await _httpClient.ExecuteAsync(request, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
@@ -175,10 +180,9 @@ public override async Task ReqGetJobByUrlAsync(
throw new ArgumentException("Job URL cannot be null or empty.", nameof(pollingUrl));
var request = new RestRequest(new Uri(pollingUrl));
- Logger?.LogInformation("HTTP GET to {RequestResource}...", request.Resource);
+ Logger?.LogDebug("HTTP GET to {RequestResource}...", request.Resource);
var response = await _httpClient.ExecuteAsync(request, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
- Logger?.LogDebug("HTTP response: {ResponseContent}", response.Content);
var handledResponse = HandleJobResponse(response);
return handledResponse;
}
@@ -190,7 +194,7 @@ public override async Task ReqGetResultByIdAsync(string in
throw new Exception($"ProductAttributes must be set for class: {typeof(TResponse).Name}");
var request = new RestRequest(
$"/v2/products/{productAttributes.Slug}/results/{inferenceId}");
- Logger?.LogInformation("HTTP GET to {RequestResource}...", request.Resource);
+ Logger?.LogDebug("HTTP GET to {RequestResource}...", request.Resource);
var queueResponse = await _httpClient.ExecuteAsync(request, ct);
ct.ThrowIfCancellationRequested();
return HandleProductResponse(queueResponse);
@@ -203,7 +207,7 @@ public override async Task ReqGetResultByUrlAsync(
throw new ArgumentException("Result URL cannot be null or whitespace.", nameof(resultUrl));
var request = new RestRequest(new Uri(resultUrl));
- Logger?.LogInformation("HTTP GET to {RequestResource}...", resultUrl);
+ Logger?.LogDebug("HTTP GET to {RequestResource}...", resultUrl);
var queueResponse = await _httpClient.ExecuteAsync(request, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
return HandleProductResponse(queueResponse);
diff --git a/tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs b/tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs
index d4b76411..50b22537 100644
--- a/tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs
+++ b/tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs
@@ -19,7 +19,7 @@ public RagDocumentsTest()
_extractionModelId = Environment.GetEnvironmentVariable("MindeeV2__Findoc__Model__Id");
}
- [Fact(Timeout = 180000)]
+ [Fact(Timeout = 180000, DisplayName = "should perform the entire lifecycle of a RAG document")]
public async Task RagDocument_Lifecycle_MustSucceed()
{
var inputSource = new LocalInputSource(
diff --git a/tests/Mindee.UnitTests/V2/Parsing/FailedInferenceResponseTest.cs b/tests/Mindee.UnitTests/V2/Parsing/FailedInferenceResponseTest.cs
index ff048244..9960d196 100644
--- a/tests/Mindee.UnitTests/V2/Parsing/FailedInferenceResponseTest.cs
+++ b/tests/Mindee.UnitTests/V2/Parsing/FailedInferenceResponseTest.cs
@@ -7,7 +7,7 @@ namespace Mindee.UnitTests.V2.Parsing
[Trait("Category", "FailedInferenceResponse")]
public class FailedInferenceResponseTest
{
- [Fact]
+ [Fact(DisplayName = "should load from a JSON file")]
public void WhenFailed_MustLoad()
{
var localResponse = new LocalResponse(
diff --git a/tests/Mindee.UnitTests/V2/Product/Extraction/RagDocumentsTest.cs b/tests/Mindee.UnitTests/V2/Product/Extraction/RagDocumentsTest.cs
index 0a2e23e9..24873932 100644
--- a/tests/Mindee.UnitTests/V2/Product/Extraction/RagDocumentsTest.cs
+++ b/tests/Mindee.UnitTests/V2/Product/Extraction/RagDocumentsTest.cs
@@ -8,7 +8,7 @@ namespace Mindee.UnitTests.V2.Product.Extraction
[Trait("Category", "ExtractionRagDocuments")]
public class RagDocumentsTest
{
- [Fact]
+ [Fact(DisplayName = "should init POST parameters")]
public void PostParameters_MustInit()
{
var parameters = new RagDocumentUploadParameters(modelId: "invalid-model-id");
@@ -16,7 +16,7 @@ public void PostParameters_MustInit()
Assert.Equal("invalid-model-id", reqParams["model_id"]);
}
- [Fact]
+ [Fact(DisplayName = "should init PATCH parameters")]
public void PatchParameters_MustInit()
{
var annotation = new RagAnnotation();
@@ -30,7 +30,7 @@ public void PatchParameters_MustInit()
Assert.Equal(annotation, reqParams["annotation"]);
}
- [Fact]
+ [Fact(DisplayName = "should load a POST response from a JSON string")]
public void RagDocumentsPost_MustHaveValidProperties()
{
var response = GetResponse("extraction/rag_documents/post_response.json");
@@ -40,7 +40,7 @@ public void RagDocumentsPost_MustHaveValidProperties()
Assert.Null(response.Annotation);
}
- [Fact]
+ [Fact(DisplayName = "should load a GET response from a JSON string")]
public void RagDocumentsGetDraft_MustHaveValidProperties()
{
var response = GetResponse("extraction/rag_documents/get_response_draft.json");
diff --git a/tests/resources b/tests/resources
index 4b7f3376..8b8423d2 160000
--- a/tests/resources
+++ b/tests/resources
@@ -1 +1 @@
-Subproject commit 4b7f33766fab0e67804b84447b73c80a902d886a
+Subproject commit 8b8423d239a360f7a8ac72d77e53fb644a76e345