diff --git a/.editorconfig b/.editorconfig index fa8765e2..4a056ee6 100644 --- a/.editorconfig +++ b/.editorconfig @@ -145,4 +145,3 @@ csharp_style_expression_bodied_local_functions = false:silent [src/Mindee/Product/**/*.cs] generated_code = true - diff --git a/src/Mindee/ClientOptions/BaseClient.cs b/src/Mindee/ClientOptions/BaseClient.cs new file mode 100644 index 00000000..3f08130e --- /dev/null +++ b/src/Mindee/ClientOptions/BaseClient.cs @@ -0,0 +1,51 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Mindee.ClientOptions +{ + /// + /// Base class for all clients. + /// + public abstract class BaseClient + { + /// + /// The logger instance. + /// + protected readonly ILogger Logger; + /// + /// The factory used to create the . + /// + protected readonly ILoggerFactory LoggerFactory; + + /// + /// Default constructor. + /// + /// Factory for the logger. + protected BaseClient(ILoggerFactory loggerFactory = null) + { + LoggerFactory = loggerFactory ?? NullLoggerFactory.Instance; + Logger = LoggerFactory.CreateLogger(GetType()); + } + + /// + /// Wait and retry logic for polling operations. + /// + protected async Task ThrottlePollingAttemptAsync( + int tryCounter, + int maxRetries, + BasePollingOptions pollingOptions, + CancellationToken cancellationToken) + { + if (tryCounter >= maxRetries) + { + Logger?.LogDebug("Not waiting, max retries reached."); + return; + } + + Logger?.LogDebug($"Waiting {pollingOptions.IntervalMilliSec} before next attempt..."); + await Task.Delay(pollingOptions.IntervalMilliSec, cancellationToken); + } + } +} diff --git a/src/Mindee/ClientOptions/BasePollingOptions.cs b/src/Mindee/ClientOptions/BasePollingOptions.cs index a908a19d..eca3b642 100644 --- a/src/Mindee/ClientOptions/BasePollingOptions.cs +++ b/src/Mindee/ClientOptions/BasePollingOptions.cs @@ -79,20 +79,5 @@ protected void ValidateSettings() throw new MindeeException($"Cannot set async retry to less than {MinRetries} attempts."); } } - - /// - /// Compute delay in milliseconds before the given polling attempt. - /// - /// 1-based polling attempt number. - /// Delay in milliseconds. - public int GetRetryDelayMilliSec(int attemptNumber) - { - if (attemptNumber <= 1) - { - return IntervalMilliSec; - } - - return (int)Math.Floor(IntervalSec * 1000); - } } } diff --git a/src/Mindee/Exceptions/MindeeInputException.cs b/src/Mindee/Exceptions/MindeeInputException.cs index e6d1b8aa..302a94ee 100644 --- a/src/Mindee/Exceptions/MindeeInputException.cs +++ b/src/Mindee/Exceptions/MindeeInputException.cs @@ -3,7 +3,7 @@ namespace Mindee.Exceptions { /// - /// Represent a 400 error, bad requests or malformed, from Mindee API. + /// Represent an invalid or malformed input. /// public class MindeeInputException : MindeeException { diff --git a/src/Mindee/MindeeLogger.cs b/src/Mindee/MindeeLogger.cs index 14ff5e0c..ab5986e9 100644 --- a/src/Mindee/MindeeLogger.cs +++ b/src/Mindee/MindeeLogger.cs @@ -3,7 +3,8 @@ namespace Mindee { /// - /// Global Mindee logger. + /// Legacy Static Mindee logger, only used in V1. + /// TODO: refactor to delete this class. /// public static class MindeeLogger { diff --git a/src/Mindee/V1/Client.cs b/src/Mindee/V1/Client.cs index ebfba87d..3288f237 100644 --- a/src/Mindee/V1/Client.cs +++ b/src/Mindee/V1/Client.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Mindee.ClientOptions; using Mindee.Exceptions; using Mindee.Extensions.DependencyInjection; using Mindee.Input; @@ -20,9 +21,8 @@ namespace Mindee.V1 /// /// The entry point to use the Mindee API legacy features. /// - public sealed class Client + public sealed class Client : BaseClient { - private readonly ILogger _logger; private readonly IHttpApi _mindeeApi; private readonly IPdfOperation _pdfOperation; @@ -30,7 +30,7 @@ public sealed class Client /// /// The required API key to use Mindee. /// - public Client(string apiKey, ILoggerFactory logger = null) + public Client(string apiKey, ILoggerFactory logger = null) : base(logger) { var serviceCollection = new ServiceCollection(); _pdfOperation = new DocNetApi(); @@ -43,8 +43,8 @@ public Client(string apiKey, ILoggerFactory logger = null) if (logger != null) { - serviceCollection.AddSingleton(logger); - _logger = logger.CreateLogger(); + // legacy static logging only used in v1 + MindeeLogger.Assign(logger); } _mindeeApi = serviceProvider.GetRequiredService(); @@ -56,7 +56,7 @@ public Client(string apiKey, ILoggerFactory logger = null) /// /// /// - public Client(Settings settings, ILoggerFactory logger = null) + public Client(Settings settings, ILoggerFactory logger = null) : base(logger) { var serviceCollection = new ServiceCollection(); _pdfOperation = new DocNetApi(); @@ -71,8 +71,8 @@ public Client(Settings settings, ILoggerFactory logger = null) if (logger != null) { + // legacy static logging only used in v1 MindeeLogger.Assign(logger); - _logger = MindeeLogger.GetLogger(); } _mindeeApi = serviceProvider.GetRequiredService(); @@ -87,14 +87,14 @@ public Client(Settings settings, ILoggerFactory logger = null) /// /// /// - public Client(IPdfOperation pdfOperation, IHttpApi httpApi, ILoggerFactory logger = null) + public Client(IPdfOperation pdfOperation, IHttpApi httpApi, ILoggerFactory logger = null) : base(logger) { _pdfOperation = pdfOperation; _mindeeApi = httpApi; if (logger != null) { + // legacy static logging only used in v1 MindeeLogger.Assign(logger); - _logger = MindeeLogger.GetLogger(); } } @@ -124,7 +124,7 @@ LocalInputSource inputSource , PageOptions pageOptions = null) where TInferenceModel : GeneratedV1, new() { - _logger?.LogInformation("Synchronous parsing of {} ...", nameof(TInferenceModel)); + Logger?.LogInformation("Synchronous parsing of {} ...", nameof(TInferenceModel)); if (predictOptions == null) { @@ -175,7 +175,7 @@ UrlInputSource inputSource , PredictOptions predictOptions = null) where TInferenceModel : GeneratedV1, new() { - _logger?.LogInformation("Synchronous parsing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Synchronous parsing of {} ...", typeof(TInferenceModel).Name); if (predictOptions == null) { @@ -224,7 +224,7 @@ LocalInputSource inputSource , PageOptions pageOptions = null) where TInferenceModel : GeneratedV1, new() { - _logger?.LogInformation("Enqueuing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Enqueuing of {} ...", typeof(TInferenceModel).Name); if (predictOptions == null) { @@ -276,7 +276,7 @@ UrlInputSource inputSource , PredictOptions predictOptions = null) where TInferenceModel : GeneratedV1, new() { - _logger?.LogInformation("Enqueuing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Enqueuing of {} ...", typeof(TInferenceModel).Name); if (predictOptions == null) { @@ -316,7 +316,7 @@ CustomEndpoint endpoint , CancellationToken ct = default) where TInferenceModel : GeneratedV1, new() { - _logger?.LogInformation("Parse from queue of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Parse from queue of {} ...", typeof(TInferenceModel).Name); if (string.IsNullOrWhiteSpace(jobId)) { @@ -362,7 +362,7 @@ LocalInputSource inputSource , CancellationToken ct = default) where TInferenceModel : GeneratedV1, new() { - _logger?.LogInformation("Asynchronous parsing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Asynchronous parsing of {} ...", typeof(TInferenceModel).Name); if (pollingOptions == null) { @@ -411,7 +411,7 @@ UrlInputSource inputSource , CancellationToken ct = default) where TInferenceModel : GeneratedV1, new() { - _logger?.LogInformation("Asynchronous parsing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Asynchronous parsing of {} ...", typeof(TInferenceModel).Name); if (pollingOptions == null) { @@ -452,7 +452,7 @@ LocalInputSource inputSource , PageOptions pageOptions = null) where TInferenceModel : class, new() { - _logger?.LogInformation("Synchronous parsing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Synchronous parsing of {} ...", typeof(TInferenceModel).Name); if (predictOptions == null) { @@ -498,7 +498,7 @@ UrlInputSource inputSource , PredictOptions predictOptions = null) where TInferenceModel : class, new() { - _logger?.LogInformation("Synchronous parsing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Synchronous parsing of {} ...", typeof(TInferenceModel).Name); if (predictOptions == null) { @@ -542,7 +542,7 @@ LocalInputSource inputSource , PageOptions pageOptions = null) where TInferenceModel : class, new() { - _logger?.LogInformation("Enqueuing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Enqueuing of {} ...", typeof(TInferenceModel).Name); if (predictOptions == null) { @@ -588,7 +588,7 @@ UrlInputSource inputSource , PredictOptions predictOptions = null) where TInferenceModel : class, new() { - _logger?.LogInformation("Enqueuing of {} ...", typeof(TInferenceModel).Name); + Logger?.LogInformation("Enqueuing of {} ...", typeof(TInferenceModel).Name); if (predictOptions == null) { @@ -623,7 +623,7 @@ public async Task> ParseQueuedAsync> ExecuteWorkflowAsync( WorkflowOptions workflowOptions = null, PageOptions pageOptions = null) { - _logger?.LogInformation("Sending '{Filename}' to workflow '{WorkflowId}'...", inputSource.Filename, workflowId); + Logger?.LogInformation("Sending '{Filename}' to workflow '{WorkflowId}'...", inputSource.Filename, workflowId); if (pageOptions != null && inputSource.IsPdf()) { @@ -786,7 +786,7 @@ public async Task> ExecuteWorkflowAsync( UrlInputSource inputSource, WorkflowOptions workflowOptions = null) { - _logger?.LogInformation("Asynchronous parsing of {} ...", inputSource.FileUrl); + Logger?.LogInformation("Asynchronous parsing of {} ...", inputSource.FileUrl); if (workflowOptions == null) { @@ -839,7 +839,7 @@ public AsyncPredictResponse LoadPrediction( /// /// /// - /// Cancellation token. + /// Cancellation token. /// /// /// @@ -848,33 +848,37 @@ private async Task> PollForResultsAsync enqueueResponse, CustomEndpoint endpoint, AsyncPollingOptions pollingOptions, - CancellationToken ct = default) + CancellationToken cancellationToken = default) where TInferenceModel : GeneratedV1, new() { - var maxRetries = pollingOptions.MaxRetries + 1; + var maxRetries = pollingOptions.MaxRetries; var jobId = enqueueResponse.Job.Id; - _logger?.LogInformation("Enqueued with job ID: {}", jobId); - _logger?.LogInformation( + Logger?.LogInformation("Enqueued with job ID: {}", jobId); + Logger?.LogInformation( "Waiting {} seconds before attempting to retrieve the document...", pollingOptions.InitialDelaySec); - await Task.Delay(pollingOptions.InitialDelayMilliSec, ct); - var retryCount = 1; + await Task.Delay(pollingOptions.InitialDelayMilliSec, cancellationToken); + var tryCounter = 0; AsyncPredictResponse response; - while (retryCount < maxRetries) + while (tryCounter < maxRetries) { - var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(retryCount); - await Task.Delay(retryDelayMilliSec, ct); - _logger?.LogInformation("Attempting to retrieve: {RetryCount} of {MaxRetries}", retryCount, maxRetries); - response = await ParseQueuedAsync(endpoint, jobId, ct); + cancellationToken.ThrowIfCancellationRequested(); + + Logger?.LogInformation( + "Attempting to retrieve: {RetryCount} of {MaxRetries}", + tryCounter + 1, + maxRetries); + + response = await ParseQueuedAsync(endpoint, jobId, cancellationToken); if (response.Document != null) { return response; } - - retryCount++; + tryCounter++; + await ThrottlePollingAttemptAsync(tryCounter, maxRetries, pollingOptions, cancellationToken); } - throw new MindeeException($"Could not complete after {retryCount} attempts."); + throw new MindeeException($"Could not complete after {tryCounter} attempts."); } /// @@ -890,7 +894,7 @@ private async Task> PollForResultsAsync /// /// - /// Cancellation token. + /// Cancellation token. /// /// /// @@ -898,33 +902,37 @@ private async Task> PollForResultsAsync> PollForResultsAsync( AsyncPredictResponse enqueueResponse, AsyncPollingOptions pollingOptions, - CancellationToken ct = default) + CancellationToken cancellationToken = default) where TInferenceModel : class, new() { var maxRetries = pollingOptions.MaxRetries + 1; var jobId = enqueueResponse.Job.Id; - _logger?.LogInformation("Enqueued with job ID: {}", jobId); - _logger?.LogInformation( + Logger?.LogInformation("Enqueued with job ID: {}", jobId); + Logger?.LogInformation( "Waiting {} seconds before attempting to retrieve the document...", pollingOptions.InitialDelaySec); - await Task.Delay(pollingOptions.InitialDelayMilliSec, ct); - var retryCount = 1; + await Task.Delay(pollingOptions.InitialDelayMilliSec, cancellationToken); + var tryCounter = 1; AsyncPredictResponse response; - while (retryCount < maxRetries) + while (tryCounter < maxRetries) { - var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(retryCount); - await Task.Delay(retryDelayMilliSec, ct); - _logger?.LogInformation("Attempting to retrieve: {RetryCount} of {MaxRetries}", retryCount, maxRetries); - response = await ParseQueuedAsync(jobId, ct); + cancellationToken.ThrowIfCancellationRequested(); + + Logger?.LogInformation( + "Attempting to retrieve: {RetryCount} of {MaxRetries}", + tryCounter + 1, + maxRetries); + + response = await ParseQueuedAsync(jobId, cancellationToken); if (response.Document != null) { return response; } - - retryCount++; + tryCounter++; + await ThrottlePollingAttemptAsync(tryCounter, maxRetries, pollingOptions, cancellationToken); } - throw new MindeeException($"Could not complete after {retryCount} attempts."); + throw new MindeeException($"Could not complete after {tryCounter} attempts."); } } } diff --git a/src/Mindee/V1/Parsing/Standard/DateField.cs b/src/Mindee/V1/Parsing/Standard/DateField.cs index 252c0234..b4e1a609 100644 --- a/src/Mindee/V1/Parsing/Standard/DateField.cs +++ b/src/Mindee/V1/Parsing/Standard/DateField.cs @@ -49,7 +49,7 @@ public DateField( } catch (FormatException) { - logger?.LogWarning("Unable to parse the date: {}", Value); + logger?.LogWarning("Unable to parse the date: {DateString}", Value); } } diff --git a/src/Mindee/V2/Client.cs b/src/Mindee/V2/Client.cs index facf20c3..e7d2b4a3 100644 --- a/src/Mindee/V2/Client.cs +++ b/src/Mindee/V2/Client.cs @@ -4,7 +4,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; +using Mindee.ClientOptions; using Mindee.Exceptions; using Mindee.Extensions.DependencyInjection; using Mindee.Input; @@ -26,28 +26,23 @@ namespace Mindee.V2 /// /// The entry point to use the Mindee V2 API features. /// - public sealed class Client + public sealed class Client : BaseClient { - private readonly ILogger _logger; private readonly HttpApiV2 _mindeeApi; /// /// /// The required API key to use the Mindee V2 API. /// Factory for the logger. - public Client(string apiKey, ILoggerFactory loggerFactory = null) + public Client(string apiKey, ILoggerFactory loggerFactory = null) : base(loggerFactory) { - var loggerFactoryInstance = loggerFactory ?? NullLoggerFactory.Instance; - _logger = loggerFactoryInstance.CreateLogger(); - var serviceCollection = new ServiceCollection(); serviceCollection.AddMindeeApiV2(options => { options.ApiKey = apiKey; - }, loggerFactoryInstance); + }, LoggerFactory); var serviceProvider = serviceCollection.BuildServiceProvider(); - _mindeeApi = serviceProvider.GetRequiredService(); } @@ -57,24 +52,17 @@ public Client(string apiKey, ILoggerFactory loggerFactory = null) /// /// /// - public Client(SettingsV2 settings, ILoggerFactory logger = null) + public Client(SettingsV2 settings, ILoggerFactory logger = null) : base(logger) { - var loggerFactory = logger ?? NullLoggerFactory.Instance; var serviceCollection = new ServiceCollection(); serviceCollection.AddMindeeApiV2(options => { options.ApiKey = settings.ApiKey; options.MindeeBaseUrl = settings.MindeeBaseUrl; options.RequestTimeoutSeconds = settings.RequestTimeoutSeconds; - }, loggerFactory); - var serviceProvider = serviceCollection.BuildServiceProvider(); - - if (logger != null) - { - MindeeLogger.Assign(logger); - _logger = MindeeLogger.GetLogger(); - } + }, LoggerFactory); + var serviceProvider = serviceCollection.BuildServiceProvider(); _mindeeApi = serviceProvider.GetRequiredService(); } @@ -84,11 +72,9 @@ public Client(SettingsV2 settings, ILoggerFactory logger = null) /// /// /// - public Client(HttpApiV2 httpApi, ILoggerFactory logger = null) + public Client(HttpApiV2 httpApi, ILoggerFactory logger = null) : base(logger) { _mindeeApi = httpApi; - var loggerFactory = logger ?? NullLoggerFactory.Instance; - _logger = loggerFactory.CreateLogger(); } /// @@ -114,10 +100,10 @@ InputSource inputSource switch (inputSource) { case LocalInputSource: - _logger?.LogInformation("Enqueuing: local source"); + Logger?.LogInformation("Enqueuing: local source"); break; case UrlInputSource: - _logger?.LogInformation("Enqueuing: URL source"); + Logger?.LogInformation("Enqueuing: URL source"); break; case null: throw new ArgumentNullException(nameof(inputSource)); @@ -138,7 +124,7 @@ InputSource inputSource /// public async Task GetJobFromUrlAsync(string pollingUrl, CancellationToken ct = default) { - _logger?.LogInformation("Getting Job at: {JobURL}", pollingUrl); + Logger?.LogInformation("Getting Job by URL: {JobURL}", pollingUrl); return await _mindeeApi.ReqGetJobByUrlAsync(pollingUrl, ct); } @@ -153,7 +139,7 @@ public async Task GetJobFromUrlAsync(string pollingUrl, Cancellatio public async Task GetResultFromUrlAsync(string resultUrl, CancellationToken ct = default) where TResponse : BaseResponse, new() { - _logger?.LogInformation("Getting result at: {ResultUrl}", resultUrl); + Logger?.LogInformation("Getting result by URL: {ResultUrl}", resultUrl); return await _mindeeApi.ReqGetResultByUrlAsync(resultUrl, ct); } @@ -168,12 +154,11 @@ public async Task GetResultFromUrlAsync(string resultUrl, public async Task GetResultAsync(string jobId, CancellationToken ct = default) where TResponse : BaseResponse, new() { - _logger?.LogInformation("Getting result with ID: {JobID}", jobId); + Logger?.LogInformation("Getting result with ID: {JobID}", jobId); if (string.IsNullOrWhiteSpace(jobId)) - { - throw new ArgumentNullException(jobId); - } + throw new ArgumentNullException(nameof(jobId), "jobId must not be null or blank."); + return await _mindeeApi.ReqGetResultByIdAsync(jobId, ct); } @@ -188,7 +173,11 @@ public async Task GetResultAsync(string jobId, Cancellatio /// public async Task GetJobAsync(string jobId, CancellationToken ct = default) { - _logger?.LogInformation("Getting job ID: {JobID}", jobId); + Logger?.LogInformation("Getting job ID: {JobID}", jobId); + + if (string.IsNullOrWhiteSpace(jobId)) + throw new ArgumentNullException(jobId, "jobId must not be null or blank."); + return await _mindeeApi.ReqGetJobByIdAsync(jobId, ct); } @@ -223,7 +212,7 @@ InputSource inputSource inputSource, parameters, ct); - _logger?.LogInformation( + Logger?.LogInformation( "Successfully enqueued document with job ID {JobID}", enqueueResponse.Job.Id); return await PollForProductResultsAsync( enqueueResponse, pollingOptions, false, ct); @@ -243,7 +232,7 @@ LocalInputSource inputSource , CancellationToken ct = default) where TAnnotationResponse : BaseRagAnnotationResponse, new() { - _logger?.LogInformation("Adding a document to the RAG database"); + Logger?.LogInformation("Adding a document to the RAG database"); return await _mindeeApi.ReqPostRagDocumentAsync(parameters, inputSource, ct); } @@ -280,7 +269,7 @@ public async Task GetRagDocumentAsync( string documentId, CancellationToken ct = default) where TAnnotationResponse : BaseRagAnnotationResponse, new() { - _logger?.LogInformation("Getting RAG document ID: {DocumentId}", documentId); + Logger?.LogInformation("Getting RAG document ID: {DocumentId}", documentId); return await _mindeeApi.ReqGetRagAnnotationAsync(documentId, ct); } @@ -317,7 +306,7 @@ public async Task UpdateRagAnnotationAsync parameters, CancellationToken ct = default) where TAnnotationResponse : BaseRagAnnotationResponse, new() { - _logger?.LogInformation("Updating RAG document ID: {DocumentId}", parameters.DocumentId); + Logger?.LogInformation("Updating RAG document ID: {DocumentId}", parameters.DocumentId); return await _mindeeApi.ReqPatchRagAnnotationAsync(parameters, ct); } @@ -353,7 +342,7 @@ BaseAnnotationParameters parameters public async Task DeleteExtractionRagDocumentAsync( string documentId, CancellationToken ct = default) { - _logger?.LogInformation("Deleting RAG document ID: {DocumentId}", documentId); + Logger?.LogInformation("Deleting RAG document ID: {DocumentId}", documentId); return await _mindeeApi.ReqDeleteExtractionRagDocumentAsync(documentId, ct); } @@ -397,39 +386,42 @@ BaseRagAnnotationResponse initialResponse , CancellationToken cancellationToken = default) where TAnnotationResponse : BaseRagAnnotationResponse, new() { - _logger?.LogInformation("Polling for RAG document ID: {DocumentId}", initialResponse.Id); - var maxRetries = pollingOptions.MaxRetries + 1; + Logger?.LogInformation("Polling for RAG document ID: {DocumentId}", initialResponse.Id); - _logger?.LogDebug( + Logger?.LogDebug( "Waiting {InitialDelaySec} seconds before attempting to retrieve the result...", pollingOptions.InitialDelaySec); + await Task.Delay(pollingOptions.InitialDelayMilliSec, cancellationToken); + + var tryCounter = 0; + var maxRetries = pollingOptions.MaxRetries; var documentId = initialResponse.Id; - var retryCount = 1; - while (retryCount < maxRetries) + while (tryCounter < maxRetries) { - var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(retryCount); - await Task.Delay(retryDelayMilliSec, cancellationToken); - _logger?.LogDebug( + cancellationToken.ThrowIfCancellationRequested(); + + Logger?.LogDebug( "Poll attempt {RetryCount} of {MaxRetries}", - retryCount, + tryCounter + 1, maxRetries); var response = await GetRagDocumentAsync(documentId, cancellationToken); - retryCount++; + tryCounter++; switch (response.Status) { case "Processing": - continue; + await ThrottlePollingAttemptAsync(tryCounter, maxRetries, pollingOptions, cancellationToken); + break; case "Failed": throw new MindeeException("RAG failed without an error payload."); default: return response; } } - throw new MindeeException($"RAG polling not complete after {retryCount} attempts."); + throw new MindeeException($"RAG polling not complete after {tryCounter} attempts."); } /// @@ -442,11 +434,11 @@ private bool CheckWebhooksDone(JobResponse jobResponse) if (areWebhooksDone) { - _logger?.LogDebug("All webhooks are completed."); + Logger?.LogDebug("All webhooks are completed."); return true; } - _logger?.LogDebug("Not all webhooks are completed."); + Logger?.LogDebug("Not all webhooks are completed."); return false; } @@ -459,7 +451,7 @@ private async Task PollOnJobAsync( bool waitForWebhooks = false, CancellationToken cancellationToken = default) { - _logger?.LogDebug( + Logger?.LogDebug( "Waiting {InitialDelaySec} seconds before attempting to retrieve the result...", pollingOptions.InitialDelaySec); @@ -472,7 +464,7 @@ private async Task PollOnJobAsync( { cancellationToken.ThrowIfCancellationRequested(); - _logger?.LogDebug( + Logger?.LogDebug( "Poll attempt {RetryCount} of {MaxRetries}", tryCounter + 1, maxRetries); @@ -481,7 +473,7 @@ private async Task PollOnJobAsync( if (jobResponse.Job.Status == "Processed") { - _logger?.LogDebug( + Logger?.LogDebug( "Job ID {JobID} completed processing at: {CompletedAt}", jobResponse.Job.Id, jobResponse.Job.CompletedAt); @@ -492,7 +484,7 @@ private async Task PollOnJobAsync( } } - // normally the mindee_api will throw on error, this is a fallback + // normally the API handler will throw an error, this is a fallback if (jobResponse.Job.Status == "Failed") { if (jobResponse.Job.Error != null) @@ -503,8 +495,7 @@ private async Task PollOnJobAsync( } tryCounter++; - var retryDelayMilliSec = pollingOptions.GetRetryDelayMilliSec(tryCounter); - await Task.Delay(retryDelayMilliSec, cancellationToken); + await ThrottlePollingAttemptAsync(tryCounter, maxRetries, pollingOptions, cancellationToken); } throw new MindeeException($"Couldn't retrieve the result after {tryCounter} tries."); diff --git a/src/Mindee/V2/ClientOptions/BaseAnnotationParameters.cs b/src/Mindee/V2/ClientOptions/BaseAnnotationParameters.cs index 3d7e3b0b..a1d780e4 100644 --- a/src/Mindee/V2/ClientOptions/BaseAnnotationParameters.cs +++ b/src/Mindee/V2/ClientOptions/BaseAnnotationParameters.cs @@ -16,9 +16,9 @@ public abstract class BaseAnnotationParameters public string DocumentId { get; } /// - /// Default constructor. + /// Base constructor. /// - /// + /// protected BaseAnnotationParameters(string documentId) { if (string.IsNullOrWhiteSpace(documentId)) diff --git a/src/Mindee/V2/ClientOptions/BaseRagDocumentUploadParameters.cs b/src/Mindee/V2/ClientOptions/BaseRagDocumentUploadParameters.cs index c5ddaa7f..a2fa1820 100644 --- a/src/Mindee/V2/ClientOptions/BaseRagDocumentUploadParameters.cs +++ b/src/Mindee/V2/ClientOptions/BaseRagDocumentUploadParameters.cs @@ -16,7 +16,7 @@ public abstract class BaseRagDocumentUploadParameters public string ModelId { get; } /// - /// Default constructor. + /// Base constructor. /// /// protected BaseRagDocumentUploadParameters(string modelId) diff --git a/src/Mindee/V2/Http/HttpApiV2.cs b/src/Mindee/V2/Http/HttpApiV2.cs index 1825669d..73541895 100644 --- a/src/Mindee/V2/Http/HttpApiV2.cs +++ b/src/Mindee/V2/Http/HttpApiV2.cs @@ -30,7 +30,7 @@ public abstract class HttpApiV2 protected ILogger? Logger; /// - /// Send a file to the asynchronous processing queue for inference processing. + /// Send a file to the asynchronous processing queue for a product. /// /// /// @@ -48,7 +48,7 @@ InputSource inputSource /// /// Get the status of an inference that was previously enqueued. /// - /// The job ID as returned by the predict_async route. + /// The job ID as returned by the enqueue call. /// Cancellation token. public abstract Task ReqGetJobByUrlAsync( string pollingUrl, CancellationToken cancellationToken = default); @@ -56,7 +56,7 @@ public abstract Task ReqGetJobByUrlAsync( /// /// Get the status of an inference that was previously enqueued. /// - /// The job ID as returned by the predict_async route. + /// The job ID as returned by the enqueue call. /// Cancellation token. public abstract Task ReqGetJobByIdAsync( string jobId, CancellationToken cancellationToken = default); @@ -136,12 +136,12 @@ public abstract Task SearchModelsObsolete( ModelSearchParameters parameters, CancellationToken cancellationToken = default); /// - /// Get the error from the server return. + /// Get the error from the server response. /// /// HTTP Status of the response /// String content of the response /// - protected ErrorResponse GetErrorFromContent(int statusCode, string? responseContent) + protected ErrorResponse GetErrorFromResponse(int statusCode, string? responseContent) { Logger?.LogInformation("Parsing error response ..."); try diff --git a/src/Mindee/V2/Http/MindeeApiV2.cs b/src/Mindee/V2/Http/MindeeApiV2.cs index c2887dd7..5f987060 100644 --- a/src/Mindee/V2/Http/MindeeApiV2.cs +++ b/src/Mindee/V2/Http/MindeeApiV2.cs @@ -73,10 +73,10 @@ InputSource inputSource public override async Task ReqGetSearchAsync( BaseSearchParameters parameters, CancellationToken cancellationToken = default) { - var searchType = typeof(TSearchResponse); - var productAttributes = searchType.GetCustomAttribute(); + var responseType = typeof(TSearchResponse); + var productAttributes = responseType.GetCustomAttribute(); if (productAttributes == null) - throw new Exception($"ProductAttributes must be set for class: {searchType.Name}"); + throw new Exception($"ProductAttributes must be set for class: {responseType.Name}"); var request = new RestRequest($"/v2/search/{productAttributes.Slug}"); Logger?.LogInformation("Searching {Slug} ...", productAttributes.Slug); @@ -90,7 +90,7 @@ public override async Task ReqGetSearchAsync( cancellationToken.ThrowIfCancellationRequested(); var response = JsonSerializer.Deserialize(GetResponseContent(restResponse)); - return response ?? throw new MindeeException($"Couldn't deserialize {searchType}."); + return response ?? throw new MindeeException($"Couldn't deserialize {responseType}."); } public override async Task ReqPostRagDocumentAsync( @@ -98,7 +98,13 @@ BaseRagDocumentUploadParameters parameters , LocalInputSource localInputSource , CancellationToken cancellationToken = default) { - var request = new RestRequest("/v2/products/extraction/rag-documents", Method.Post); + var responseType = typeof(TAnnotationResponse); + var productAttributes = responseType.GetCustomAttribute(); + if (productAttributes == null) + throw new Exception($"ProductAttributes must be set for class: {responseType.Name}"); + + var request = new RestRequest( + $"/v2/products/{productAttributes.Slug}/rag-documents", Method.Post); request.AddFile( "file", localInputSource.FileBytes, @@ -116,7 +122,13 @@ BaseRagDocumentUploadParameters parameters public override async Task ReqGetRagAnnotationAsync( string documentId, CancellationToken cancellationToken = default) { - var request = new RestRequest($"/v2/products/extraction/rag-documents/{documentId}"); + var responseType = typeof(TAnnotationResponse); + var productAttributes = responseType.GetCustomAttribute(); + if (productAttributes == null) + throw new Exception($"ProductAttributes must be set for class: {responseType.Name}"); + + var request = new RestRequest( + $"/v2/products/{productAttributes.Slug}/rag-documents/{documentId}"); Logger?.LogDebug("HTTP GET to {RequestResource} ...", request.Resource); return await ExecuteRagAnnotationRequest(request, cancellationToken); @@ -125,8 +137,13 @@ public override async Task ReqGetRagAnnotationAsync ReqPatchRagAnnotationAsync( BaseAnnotationParameters parameters, CancellationToken cancellationToken = default) { + var responseType = typeof(TAnnotationResponse); + var productAttributes = responseType.GetCustomAttribute(); + if (productAttributes == null) + throw new Exception($"ProductAttributes must be set for class: {responseType.Name}"); + var request = new RestRequest( - $"/v2/products/extraction/rag-documents/{parameters.DocumentId}", Method.Patch); + $"/v2/products/{productAttributes.Slug}/rag-documents/{parameters.DocumentId}", Method.Patch); request.AddJsonBody(parameters.GetRequestParameters()); Logger?.LogDebug("HTTP PATCH to {RequestResource} ...", request.Resource); @@ -269,7 +286,7 @@ private string GetResponseContent(RestResponse restResponse) if (statusCode is <= 199 or >= 400) { throw new MindeeHttpExceptionV2( - GetErrorFromContent(statusCode, restResponse.Content)); + GetErrorFromResponse(statusCode, restResponse.Content)); } if (restResponse.Content == null) diff --git a/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedDynamicField.cs b/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedDynamicField.cs index c7d1ba1d..c24b3109 100644 --- a/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedDynamicField.cs +++ b/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedDynamicField.cs @@ -30,22 +30,39 @@ public class AnnotatedDynamicField public FieldType Type { get; } /// - /// Default constructor. + /// Constructor for a simple field. /// - /// - /// - /// - /// - public AnnotatedDynamicField( - FieldType type - , AnnotatedSimpleField simpleField = null - , AnnotatedObjectField objectField = null - , AnnotatedListField listField = null) + /// + public AnnotatedDynamicField(AnnotatedSimpleField field) { - SimpleField = simpleField; - ObjectField = objectField; - ListField = listField; - Type = type; + SimpleField = field; + ObjectField = null; + ListField = null; + Type = FieldType.SimpleField; + } + + /// + /// Constructor for a list field. + /// + /// + public AnnotatedDynamicField(AnnotatedListField field) + { + SimpleField = null; + ObjectField = null; + ListField = field; + Type = FieldType.ListField; + } + + /// + /// Constructor for an object field. + /// + /// + public AnnotatedDynamicField(AnnotatedObjectField field) + { + SimpleField = null; + ObjectField = field; + ListField = null; + Type = FieldType.ObjectField; } } } diff --git a/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedDynamicFieldJsonConverter.cs b/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedDynamicFieldJsonConverter.cs index 02a4adbb..e9a39e4d 100644 --- a/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedDynamicFieldJsonConverter.cs +++ b/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedDynamicFieldJsonConverter.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; @@ -29,28 +30,23 @@ public override AnnotatedDynamicField Read(ref Utf8JsonReader reader, Type typeT string guidelines = jsonObject["guidelines"]?.Deserialize(options); bool selected = jsonObject["selected"]?.Deserialize(options) ?? false; - var listField = new AnnotatedListField(selected, guidelines); + var items = new List(); foreach (var item in itemsArray) - { - listField.Items.Add(item.Deserialize(options)); - } + items.Add(item.Deserialize(options)); - return new AnnotatedDynamicField( - FieldType.ListField, listField: listField); + var listField = new AnnotatedListField(items, selected, guidelines); + + return new AnnotatedDynamicField(listField); } if (jsonObject.TryGetPropertyValue("fields", out var nestedFieldsNode) && nestedFieldsNode is JsonObject) { - return new AnnotatedDynamicField( - FieldType.ObjectField, - objectField: jsonObject.Deserialize(options)); + return new AnnotatedDynamicField(jsonObject.Deserialize(options)); } // -------- SIMPLE FIELD -------- if (jsonObject.ContainsKey("value")) { - return new AnnotatedDynamicField( - FieldType.SimpleField, - simpleField: jsonObject.Deserialize(options)); + return new AnnotatedDynamicField(jsonObject.Deserialize(options)); } throw new JsonException($"Unknown field: {jsonObject.ToJsonString()}"); diff --git a/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedListField.cs b/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedListField.cs index f06318f5..d1903169 100644 --- a/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedListField.cs +++ b/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedListField.cs @@ -17,15 +17,17 @@ public class AnnotatedListField : AnnotatedBaseField /// List of dynamic fields, prefer SimpleItems or ObjectItems. /// [JsonPropertyName("items")] - public List Items { get; set; } = []; + public List Items { get; set; } /// /// Default constructor. /// + /// /// /// - public AnnotatedListField(bool selected, string guidelines) : base(selected, guidelines) + public AnnotatedListField(List items, bool selected, string guidelines) : base(selected, guidelines) { + Items = items; } /// diff --git a/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedObjectField.cs b/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedObjectField.cs index 48972c9c..a96a2d5f 100644 --- a/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedObjectField.cs +++ b/src/Mindee/V2/Product/Extraction/RagDocuments/AnnotatedObjectField.cs @@ -18,7 +18,11 @@ public class AnnotatedObjectField : AnnotatedBaseField /// /// /// - public AnnotatedObjectField(bool selected, string guidelines) : base(selected, guidelines) - { } + /// + public AnnotatedObjectField(AnnotatedFields fields, bool selected, string guidelines) : + base(selected, guidelines) + { + this.Fields = fields; + } } } diff --git a/src/Mindee/V2/Product/Extraction/RagDocuments/ExtractionRagAnnotationResponse.cs b/src/Mindee/V2/Product/Extraction/RagDocuments/ExtractionRagAnnotationResponse.cs index be34e5b1..e9f89e0b 100644 --- a/src/Mindee/V2/Product/Extraction/RagDocuments/ExtractionRagAnnotationResponse.cs +++ b/src/Mindee/V2/Product/Extraction/RagDocuments/ExtractionRagAnnotationResponse.cs @@ -1,12 +1,14 @@ using System; using System.Text.Json.Serialization; using Mindee.V2.Parsing; +using Mindee.V2.Product; namespace Mindee.V2.Product.Extraction.RagDocuments { /// /// Response for a RAG document. /// + [ProductAttributes("extraction")] public class ExtractionRagAnnotationResponse : BaseRagAnnotationResponse { /// diff --git a/src/Mindee/V2/Product/Extraction/RagDocuments/Params/RagDocumentUploadParameters.cs b/src/Mindee/V2/Product/Extraction/RagDocuments/Params/RagDocumentUploadParameters.cs index cb5dd7f1..3bc44445 100644 --- a/src/Mindee/V2/Product/Extraction/RagDocuments/Params/RagDocumentUploadParameters.cs +++ b/src/Mindee/V2/Product/Extraction/RagDocuments/Params/RagDocumentUploadParameters.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using Mindee.V2.ClientOptions; namespace Mindee.V2.Product.Extraction.RagDocuments.Params diff --git a/src/Mindee/V2/Product/Extraction/RagDocuments/RagAnnotation.cs b/src/Mindee/V2/Product/Extraction/RagDocuments/RagAnnotation.cs index efaef7a5..7b2317b5 100644 --- a/src/Mindee/V2/Product/Extraction/RagDocuments/RagAnnotation.cs +++ b/src/Mindee/V2/Product/Extraction/RagDocuments/RagAnnotation.cs @@ -12,5 +12,21 @@ public class RagAnnotation /// [JsonPropertyName("fields")] public AnnotatedFields Fields { get; set; } + + /// + /// Empty constructor. + /// + public RagAnnotation() + { + } + + /// + /// Default constructor. + /// + /// + public RagAnnotation(AnnotatedFields fields) + { + this.Fields = fields; + } } } diff --git a/tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs b/tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs index 50b22537..d0525629 100644 --- a/tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs +++ b/tests/Mindee.IntegrationTests/V2/Product/Extraction/RagDocumentsTest.cs @@ -9,14 +9,14 @@ namespace Mindee.IntegrationTests.V2.Product.Extraction [Trait("Category", "Integration")] public class RagDocumentsTest { - private readonly string? _extractionModelId; + private readonly string? _modelId; private readonly Client _client; public RagDocumentsTest() { var apiKey = Environment.GetEnvironmentVariable("MindeeV2__ApiKey"); _client = TestingUtilities.GetOrGenerateMindeeClientV2(apiKey); - _extractionModelId = Environment.GetEnvironmentVariable("MindeeV2__Findoc__Model__Id"); + _modelId = Environment.GetEnvironmentVariable("MindeeV2__Findoc__Model__Id"); } [Fact(Timeout = 180000, DisplayName = "should perform the entire lifecycle of a RAG document")] @@ -24,7 +24,7 @@ public async Task RagDocument_Lifecycle_MustSucceed() { var inputSource = new LocalInputSource( Constants.V2ProductDir + "extraction/financial_document/default_sample.jpg"); - var parameters = new RagDocumentUploadParameters(modelId: _extractionModelId); + var parameters = new RagDocumentUploadParameters(modelId: _modelId); var postResponse = await _client.UploadAndGetRagDocumentPollAsync( inputSource, parameters); diff --git a/tests/Mindee.UnitTests/ClientOptions/BaseClientTest.cs b/tests/Mindee.UnitTests/ClientOptions/BaseClientTest.cs new file mode 100644 index 00000000..d59f2308 --- /dev/null +++ b/tests/Mindee.UnitTests/ClientOptions/BaseClientTest.cs @@ -0,0 +1,67 @@ +using System.Diagnostics; +using Mindee.ClientOptions; + +namespace Mindee.UnitTests.ClientOptions +{ + public class BaseClientTest + { + private class TestableBaseClient : BaseClient + { + public Task TestThrottlePollingAttemptAsync( + int tryCounter, + int maxRetries, + BasePollingOptions pollingOptions, + CancellationToken cancellationToken = default) + { + return ThrottlePollingAttemptAsync(tryCounter, maxRetries, pollingOptions, cancellationToken); + } + } + + private class TestPollingOptions : BasePollingOptions + { + public TestPollingOptions(int intervalMilliSec) : base(1.0, 1.0, 1) + { + IntervalMilliSec = intervalMilliSec; + } + } + + [Fact(DisplayName = "should delay for the specified polling interval when retries remain")] + public async Task PollingAttempt_WhenTryCounterLessThanMaxRetries_DelaysForInterval() + { + var client = new TestableBaseClient(); + var options = new TestPollingOptions(intervalMilliSec: 100); + var stopwatch = Stopwatch.StartNew(); + + await client.TestThrottlePollingAttemptAsync(tryCounter: 1, maxRetries: 3, options); + stopwatch.Stop(); + + Assert.True(stopwatch.ElapsedMilliseconds >= 90); + } + + [Fact(DisplayName = "should return immediately without delay when max retries are reached")] + public async Task PollingAttempt_WhenTryCounterEqualsMaxRetries_DoesNotDelay() + { + var client = new TestableBaseClient(); + var options = new TestPollingOptions(intervalMilliSec: 1000); + var stopwatch = Stopwatch.StartNew(); + + await client.TestThrottlePollingAttemptAsync(tryCounter: 3, maxRetries: 3, options); + stopwatch.Stop(); + + Assert.True(stopwatch.ElapsedMilliseconds < 200); + } + + [Fact(DisplayName = "should throw when the cancellation token is canceled")] + public async Task PollingAttempt_WithCancelledToken_ThrowsException() + { + var client = new TestableBaseClient(); + var options = new TestPollingOptions(intervalMilliSec: 5000); + using var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + + await Assert.ThrowsAnyAsync(() => + client.TestThrottlePollingAttemptAsync( + tryCounter: 0, maxRetries: 3, options, cancellationTokenSource.Token)); + } + } +} diff --git a/tests/Mindee.UnitTests/V1/AsyncPollingOptionsTest.cs b/tests/Mindee.UnitTests/V1/AsyncPollingOptionsTest.cs index aa85d437..91589e13 100644 --- a/tests/Mindee.UnitTests/V1/AsyncPollingOptionsTest.cs +++ b/tests/Mindee.UnitTests/V1/AsyncPollingOptionsTest.cs @@ -16,8 +16,6 @@ public void ValidPollingOptions_MustInit() Assert.Equal(100, options.MaxRetries); Assert.Equal(3000, options.InitialDelayMilliSec); Assert.Equal(5000, options.IntervalMilliSec); - Assert.Equal(5000, options.GetRetryDelayMilliSec(1)); - Assert.Equal(5000, options.GetRetryDelayMilliSec(2)); } [Fact] diff --git a/tests/Mindee.UnitTests/V2/ClientOptions/PollingOptionsTest.cs b/tests/Mindee.UnitTests/V2/ClientOptions/PollingOptionsTest.cs index de1ac898..b6c0447b 100644 --- a/tests/Mindee.UnitTests/V2/ClientOptions/PollingOptionsTest.cs +++ b/tests/Mindee.UnitTests/V2/ClientOptions/PollingOptionsTest.cs @@ -15,9 +15,6 @@ public void PollingOptions_ValidOptions_MustInit() Assert.Equal(100, options.MaxRetries); Assert.Equal(3000, options.InitialDelayMilliSec); Assert.Equal(5000, options.IntervalMilliSec); - Assert.Equal(5000, options.GetRetryDelayMilliSec(1)); - Assert.Equal(5000, options.GetRetryDelayMilliSec(2)); - Assert.Equal(5000, options.GetRetryDelayMilliSec(10)); } [Fact] diff --git a/tests/Mindee.UnitTests/V2/Product/ClassificationTest.cs b/tests/Mindee.UnitTests/V2/Product/ClassificationTest.cs index 5e19feac..b01f0c48 100644 --- a/tests/Mindee.UnitTests/V2/Product/ClassificationTest.cs +++ b/tests/Mindee.UnitTests/V2/Product/ClassificationTest.cs @@ -24,7 +24,7 @@ public void Parameters_MustInit() [Fact] public void Classification_WhenSingle_MustHaveValidProperties() { - var response = GetInference("classification/default_sample.json"); + var response = LoadResponse("classification/default_sample.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -44,7 +44,7 @@ public void Classification_WhenSingle_MustHaveValidProperties() [Fact] public void Classification_WithExtraction_MustHaveValidProperties() { - var response = GetInference("classification/default_sample_extraction.json"); + var response = LoadResponse("classification/default_sample_extraction.json"); Assert.NotNull(response.Inference); Assert.Equal( "invoice", @@ -68,7 +68,7 @@ public void Classification_WithExtraction_MustHaveValidProperties() ); } - private static ClassificationResponse GetInference(string path) + private static ClassificationResponse LoadResponse(string path) { var localResponse = new LocalResponse( File.ReadAllText(Constants.V2ProductPath + path)); diff --git a/tests/Mindee.UnitTests/V2/Product/CropTest.cs b/tests/Mindee.UnitTests/V2/Product/CropTest.cs index 89ecf384..cad0eed9 100644 --- a/tests/Mindee.UnitTests/V2/Product/CropTest.cs +++ b/tests/Mindee.UnitTests/V2/Product/CropTest.cs @@ -24,7 +24,7 @@ public void Parameters_MustInit() [Fact] public void Crop_WhenSingle_MustHaveValidProperties() { - var response = GetInference("crop/crop_single.json"); + var response = LoadResponse("crop/crop_single.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -56,7 +56,7 @@ public void Crop_WhenSingle_MustHaveValidProperties() [Fact] public void Crop_WhenMultiple_MustHaveValidProperties() { - var response = GetInference("crop/crop_multiple.json"); + var response = LoadResponse("crop/crop_multiple.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -101,7 +101,7 @@ public void Crop_WhenMultiple_MustHaveValidProperties() [Fact(DisplayName = "extraction properties must be valid")] public void Crop_WithExtraction_MustHaveValidProperties() { - var response = GetInference("crop/default_sample_extraction.json"); + var response = LoadResponse("crop/default_sample_extraction.json"); Assert.NotNull(response.Inference); var crops = response.Inference.Result.Crops; @@ -145,7 +145,7 @@ public void Crop_WithExtraction_MustHaveValidProperties() [Fact(DisplayName = "crop_single.rst – RST display must be parsed and exposed")] public void RstDisplay_MustBeAccessible() { - var resp = GetInference("crop/crop_single.json"); + var resp = LoadResponse("crop/crop_single.json"); var rstReference = File.ReadAllText( Constants.V2ProductPath + "crop/crop_single.rst"); @@ -168,7 +168,7 @@ private static string NormalizeLineEndings(string input) return input.Replace("\r\n", "\n").Replace("\r", "\n"); } - private static CropResponse GetInference(string path) + private static CropResponse LoadResponse(string path) { var localResponse = new LocalResponse( File.ReadAllText(Constants.V2ProductPath + path)); diff --git a/tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionParametersTest.cs b/tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionParametersTest.cs index 70aa9b8a..14a4ebfb 100644 --- a/tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionParametersTest.cs +++ b/tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionParametersTest.cs @@ -1,5 +1,7 @@ +using System.Reflection; using System.Text.Encodings.Web; using System.Text.Json; +using Mindee.V2.Product; using Mindee.V2.Product.Extraction.Params; namespace Mindee.UnitTests.V2.Product.Extraction @@ -8,51 +10,67 @@ namespace Mindee.UnitTests.V2.Product.Extraction [Trait("Category", "ExtractionInference Parameter")] public class ExtractionParametersTest { - private const string ReplacePath = Constants.V2ProductPath + "extraction/data_schema_replace_param.json"; - private readonly Dictionary DataSchemaDict; - private readonly DataSchema DataSchemaInstance; - private readonly string DataSchemaString; + private const string ModelId = "test-model-id"; - public ExtractionParametersTest() + [Fact(DisplayName = "should init with minimum values")] + public void Parameters_MustInit() { - var fileContent = File.ReadAllText(ReplacePath).Trim(); - DataSchemaDict = JsonSerializer.Deserialize>(fileContent) ?? - new Dictionary(); - DataSchemaString = JsonSerializer.Serialize(DataSchemaDict, - new JsonSerializerOptions - { - WriteIndented = false, - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping - }); - DataSchemaInstance = new DataSchema(DataSchemaDict); - } + var productParams = new ExtractionParameters(ModelId); + Assert.Equal(ModelId, productParams.ModelId); - [Fact] - public void DataSchemaNull_ShouldInitializeWithoutIssues() - { - var inferenceParameters = new ExtractionParameters("test"); - Assert.Null(inferenceParameters.DataSchema); + var productAttributes = productParams.GetType().GetCustomAttribute(); + Assert.Equal("extraction", productAttributes?.Slug); } - [Fact] - public void DataSchemaStr_ShouldInitialize() + [Trait("Feature", "Data Schema")] + public class DataSchemaTests { - var inferenceParameters = new ExtractionParameters("test", dataSchema: DataSchemaString); - Assert.Equal(DataSchemaString, inferenceParameters.DataSchema.ToString()); - } + private const string ReplacePath = Constants.V2ProductPath + "extraction/data_schema_replace_param.json"; + private readonly Dictionary DataSchemaDict; + private readonly DataSchema DataSchemaInstance; + private readonly string DataSchemaString; - [Fact] - public void DataSchemaDict_ShouldInitialize() - { - var inferenceParameters = new ExtractionParameters("test", dataSchema: DataSchemaDict); - Assert.Equal(DataSchemaString, inferenceParameters.DataSchema.ToString()); - } + public DataSchemaTests() + { + var fileContent = File.ReadAllText(ReplacePath).Trim(); + DataSchemaDict = JsonSerializer.Deserialize>(fileContent) ?? + new Dictionary(); + DataSchemaString = JsonSerializer.Serialize(DataSchemaDict, + new JsonSerializerOptions + { + WriteIndented = false, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }); + DataSchemaInstance = new DataSchema(DataSchemaDict); + } - [Fact] - public void DataSchemaInstance_ShouldInitialize() - { - var inferenceParameters = new ExtractionParameters("test", dataSchema: DataSchemaInstance); - Assert.Equal(DataSchemaString, inferenceParameters.DataSchema.ToString()); + [Fact(DisplayName = "should leave unset when not provided")] + public void DataSchema_ShouldLeaveUnsetWhenNotProvided() + { + var inferenceParameters = new ExtractionParameters(ModelId); + Assert.Null(inferenceParameters.DataSchema); + } + + [Fact(DisplayName = "should initialize from a string")] + public void DataSchemaString_ShouldInitialize() + { + var inferenceParameters = new ExtractionParameters(ModelId, dataSchema: DataSchemaString); + Assert.Equal(DataSchemaString, inferenceParameters.DataSchema.ToString()); + } + + [Fact(DisplayName = "should initialize from a dictionary")] + public void DataSchemaDict_ShouldInitialize() + { + var inferenceParameters = new ExtractionParameters(ModelId, dataSchema: DataSchemaDict); + Assert.Equal(DataSchemaString, inferenceParameters.DataSchema.ToString()); + } + + [Fact(DisplayName = "should initialize from an object instance")] + public void DataSchemaInstance_ShouldInitialize() + { + var inferenceParameters = new ExtractionParameters(ModelId, dataSchema: DataSchemaInstance); + Assert.Equal(DataSchemaString, inferenceParameters.DataSchema.ToString()); + } } } } diff --git a/tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionTest.cs b/tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionResponseTest.cs similarity index 89% rename from tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionTest.cs rename to tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionResponseTest.cs index e02812c8..f1ac88a3 100644 --- a/tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionTest.cs +++ b/tests/Mindee.UnitTests/V2/Product/Extraction/ExtractionResponseTest.cs @@ -1,31 +1,18 @@ -using System.Reflection; using Mindee.Geometry; using Mindee.V2.Parsing; using Mindee.V2.Parsing.Inference.Field; -using Mindee.V2.Product; using Mindee.V2.Product.Extraction; -using Mindee.V2.Product.Extraction.Params; namespace Mindee.UnitTests.V2.Product.Extraction { [Trait("Category", "V2")] - [Trait("Category", "ExtractionInference")] - public class ExtractionTest + [Trait("Category", "Extraction Response")] + public class ExtractionResponseTest { - [Fact] - public void Parameters_MustInit() - { - var productParams = new ExtractionParameters("invalid-model-id"); - Assert.Equal("invalid-model-id", productParams.ModelId); - - var productAttributes = productParams.GetType().GetCustomAttribute(); - Assert.Equal("extraction", productAttributes?.Slug); - } - [Fact] public void FinancialDocument_WhenEmpty_MustHaveValidProperties() { - var response = GetResponse("extraction/financial_document/blank.json"); + var response = LoadResponse("extraction/financial_document/blank.json"); AssertInferenceResponse(response); var fields = response.Inference.Result.Fields; @@ -64,7 +51,7 @@ public void FinancialDocument_WhenEmpty_MustHaveValidProperties() [Fact] public void FinancialDocument_WhenComplete_MustHaveValidProperties() { - var response = GetResponse("extraction/financial_document/complete.json"); + var response = LoadResponse("extraction/financial_document/complete.json"); AssertInferenceResponse(response); var activeOptions = response.Inference.ActiveOptions; @@ -100,7 +87,7 @@ public void FinancialDocument_WhenComplete_MustHaveValidProperties() [Fact] public void DeepNestedFields_mustExposeCorrectTypes() { - var response = GetResponse("extraction/deep_nested_fields.json"); + var response = LoadResponse("extraction/deep_nested_fields.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -137,7 +124,7 @@ public void DeepNestedFields_mustExposeCorrectTypes() [Fact] public void StandardFieldTypes_mustExposeFileValues() { - var response = GetResponse("extraction/standard_field_types.json"); + var response = LoadResponse("extraction/standard_field_types.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -159,7 +146,7 @@ public void StandardFieldTypes_mustExposeFileValues() [Fact] public void StandardFieldTypes_mustExposeSimpleFieldValues() { - var response = GetResponse("extraction/standard_field_types.json"); + var response = LoadResponse("extraction/standard_field_types.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -200,7 +187,7 @@ public void StandardFieldTypes_mustExposeSimpleFieldValues() [Fact] public void StandardFieldTypes_mustExposeSimpleListFieldValues() { - var response = GetResponse("extraction/standard_field_types.json"); + var response = LoadResponse("extraction/standard_field_types.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -222,7 +209,7 @@ public void StandardFieldTypes_mustExposeSimpleListFieldValues() [Fact] public void StandardFieldTypes_mustExposeObjectFieldValues() { - var response = GetResponse("extraction/standard_field_types.json"); + var response = LoadResponse("extraction/standard_field_types.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -247,7 +234,7 @@ public void StandardFieldTypes_mustExposeObjectFieldValues() [Fact] public void StandardFieldTypes_mustExposeObjectListFieldValues() { - var response = GetResponse("extraction/standard_field_types.json"); + var response = LoadResponse("extraction/standard_field_types.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -281,7 +268,7 @@ public void StandardFieldTypes_mustExposeObjectListFieldValues() [Fact] public void StandardFieldTypes_mustHaveLocations() { - var response = GetResponse("extraction/standard_field_types.json"); + var response = LoadResponse("extraction/standard_field_types.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -303,7 +290,7 @@ public void StandardFieldTypes_mustHaveLocations() [Fact] public void RstOutput_mustBeValid() { - var response = GetResponse("extraction/standard_field_types.json"); + var response = LoadResponse("extraction/standard_field_types.json"); var rstOutput = File.ReadAllText( Constants.V2ProductPath + "extraction/standard_field_types.rst"); @@ -320,7 +307,7 @@ public void RstOutput_mustBeValid() [Fact] public void RawText_whenActivated_mustExposeProperties() { - var response = GetResponse("extraction/raw_texts.json"); + var response = LoadResponse("extraction/raw_texts.json"); AssertInferenceResponse(response); var activeOptions = response.Inference.ActiveOptions; @@ -349,7 +336,7 @@ public void RawText_whenActivated_mustExposeProperties() [Fact] public void Rag_whenMatched_mustExposeProperties() { - var response = GetResponse("extraction/rag_matched.json"); + var response = LoadResponse("extraction/rag_matched.json"); AssertInferenceResponse(response); var activeOptions = response.Inference.ActiveOptions; @@ -367,7 +354,7 @@ public void Rag_whenMatched_mustExposeProperties() [Fact] public void Rag_whenNotMatched_mustExposeProperties() { - var response = GetResponse("extraction/rag_not_matched.json"); + var response = LoadResponse("extraction/rag_not_matched.json"); AssertInferenceResponse(response); var activeOptions = response.Inference.ActiveOptions; @@ -382,7 +369,7 @@ public void Rag_whenNotMatched_mustExposeProperties() Assert.Null(rag.RetrievedDocumentId); } - private static ExtractionResponse GetResponse(string path) + private static ExtractionResponse LoadResponse(string path) { var localResponse = new LocalResponse( File.ReadAllText(Constants.V2ProductPath + path)); diff --git a/tests/Mindee.UnitTests/V2/Product/Extraction/RagDocumentsTest.cs b/tests/Mindee.UnitTests/V2/Product/Extraction/RagDocumentsTest.cs index 24873932..d2d3f49d 100644 --- a/tests/Mindee.UnitTests/V2/Product/Extraction/RagDocumentsTest.cs +++ b/tests/Mindee.UnitTests/V2/Product/Extraction/RagDocumentsTest.cs @@ -1,3 +1,5 @@ +using System.Text.Json; +using System.Text.Json.Nodes; using Mindee.V2.Parsing; using Mindee.V2.Product.Extraction.RagDocuments; using Mindee.V2.Product.Extraction.RagDocuments.Params; @@ -8,6 +10,14 @@ namespace Mindee.UnitTests.V2.Product.Extraction [Trait("Category", "ExtractionRagDocuments")] public class RagDocumentsTest { + private static readonly string _expectedAnnotation; + + static RagDocumentsTest() + { + string rawJson = File.ReadAllText(Constants.V2ProductPath + "extraction/rag_documents/test_annotation.json"); + _expectedAnnotation = JsonSerializer.Serialize(JsonSerializer.Deserialize(rawJson)); + } + [Fact(DisplayName = "should init POST parameters")] public void PostParameters_MustInit() { @@ -19,7 +29,17 @@ public void PostParameters_MustInit() [Fact(DisplayName = "should init PATCH parameters")] public void PatchParameters_MustInit() { - var annotation = new RagAnnotation(); + var fields = new AnnotatedFields(); + fields.Add("simple", new AnnotatedDynamicField( + new AnnotatedSimpleField(true, false, null) + )); + fields.Add("list", new AnnotatedDynamicField( + new AnnotatedListField(new List(), false, null) + )); + fields.Add("object", new AnnotatedDynamicField( + new AnnotatedObjectField(new AnnotatedFields(), false, null) + )); + var annotation = new RagAnnotation(fields); var parameters = new RagDocumentAnnotationParameters( documentId: "invalid-document-id" , status: "Active" @@ -27,7 +47,13 @@ public void PatchParameters_MustInit() var reqParams = parameters.GetRequestParameters(); Assert.Equal("invalid-document-id", parameters.DocumentId); Assert.Equal("Active", reqParams["status"]); - Assert.Equal(annotation, reqParams["annotation"]); + + var actualJsonString = JsonSerializer.Serialize(reqParams["annotation"]); + + // Parse both to JsonNode to compare structure/values while ignoring property order + var expectedNode = JsonNode.Parse(_expectedAnnotation); + var actualNode = JsonNode.Parse(actualJsonString); + Assert.True(JsonNode.DeepEquals(expectedNode, actualNode)); } [Fact(DisplayName = "should load a POST response from a JSON string")] diff --git a/tests/Mindee.UnitTests/V2/Product/OcrTest.cs b/tests/Mindee.UnitTests/V2/Product/OcrTest.cs index 5e19ccc0..2b1dbbfa 100644 --- a/tests/Mindee.UnitTests/V2/Product/OcrTest.cs +++ b/tests/Mindee.UnitTests/V2/Product/OcrTest.cs @@ -23,7 +23,7 @@ public void Parameters_MustInit() [Fact] public void Ocr_WhenSingle_MustHaveValidProperties() { - var response = GetInference("ocr/ocr_single.json"); + var response = LoadResponse("ocr/ocr_single.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -54,7 +54,7 @@ public void Ocr_WhenSingle_MustHaveValidProperties() [Fact] public void Ocr_WhenMultiple_MustHaveValidProperties() { - var response = GetInference("ocr/ocr_multiple.json"); + var response = LoadResponse("ocr/ocr_multiple.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -77,7 +77,7 @@ public void Ocr_WhenMultiple_MustHaveValidProperties() } } - private static OcrResponse GetInference(string path) + private static OcrResponse LoadResponse(string path) { var localResponse = new LocalResponse( File.ReadAllText(Constants.V2ProductPath + path)); diff --git a/tests/Mindee.UnitTests/V2/Product/SplitTest.cs b/tests/Mindee.UnitTests/V2/Product/SplitTest.cs index d505cb03..4eaf4ebd 100644 --- a/tests/Mindee.UnitTests/V2/Product/SplitTest.cs +++ b/tests/Mindee.UnitTests/V2/Product/SplitTest.cs @@ -23,7 +23,7 @@ public void Parameters_MustInit() [Fact] public void Split_WhenSingle_MustHaveValidProperties() { - var response = GetInference("split/split_single.json"); + var response = LoadResponse("split/split_single.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -47,7 +47,7 @@ public void Split_WhenSingle_MustHaveValidProperties() [Fact] public void Split_WhenMultiple_MustHaveValidProperties() { - var response = GetInference("split/split_multiple.json"); + var response = LoadResponse("split/split_multiple.json"); AssertInferenceResponse(response); var inference = response.Inference; @@ -79,7 +79,7 @@ public void Split_WhenMultiple_MustHaveValidProperties() [Fact] public void Split_WithExtraction_MustHaveValidProperties() { - var response = GetInference("split/default_sample_extraction.json"); + var response = LoadResponse("split/default_sample_extraction.json"); Assert.NotNull(response.Inference); var splits = response.Inference.Result.Splits; @@ -118,7 +118,7 @@ public void Split_WithExtraction_MustHaveValidProperties() ); } - private static SplitResponse GetInference(string path) + private static SplitResponse LoadResponse(string path) { var localResponse = new LocalResponse( File.ReadAllText(Constants.V2ProductPath + path));