From 64787107db7523bafd7200474e91cbac15582532 Mon Sep 17 00:00:00 2001 From: evgTSV Date: Sat, 24 Oct 2026 20:10:12 +0500 Subject: [PATCH 1/6] Add sync worker, deck and note endpoints Commands are applied to the local collection right away and answered through the message's reply, then pushed to the sync server once the debounce window closes. Sync failures are classified and logged by AnkiSyncService along with the final sync status. Request bodies are bound with FSharp.SystemTextJson and validated with DataAnnotations; missing, null and invalid fields come back as 422 with per-field errors. Co-Authored-By: Claude Opus 5.5 --- Ankit.sln | 2 +- paket.dependencies | 2 + paket.lock | 6 ++ src/Ankit.Server/AnkiSyncWorker.fs | 91 +++++++++++++++++++ src/Ankit.Server/Ankit.Server.fsproj | 15 ++++ src/Ankit.Server/Dto.fs | 129 +++++++++++++++++++++++++++ src/Ankit.Server/Env.fs | 40 +++++++++ src/Ankit.Server/ErrorHandler.fs | 113 +++++++++++++++++++++++ src/Ankit.Server/Handlers.fs | 76 ++++++++++++++++ src/Ankit.Server/Program.fs | 68 +++++++++++++- src/Ankit.Server/Services.fs | 107 ++++++++++++++++++++++ src/Ankit.Server/Utils.fs | 35 ++++++++ src/Ankit.Server/paket.references | 4 +- 13 files changed, 684 insertions(+), 4 deletions(-) create mode 100644 src/Ankit.Server/AnkiSyncWorker.fs create mode 100644 src/Ankit.Server/Dto.fs create mode 100644 src/Ankit.Server/Env.fs create mode 100644 src/Ankit.Server/ErrorHandler.fs create mode 100644 src/Ankit.Server/Handlers.fs create mode 100644 src/Ankit.Server/Services.fs create mode 100644 src/Ankit.Server/Utils.fs diff --git a/Ankit.sln b/Ankit.sln index 718ffc3..978182f 100644 --- a/Ankit.sln +++ b/Ankit.sln @@ -8,7 +8,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution global.json = global.json paket.dependencies = paket.dependencies README.md = README.md - Scriptsuild.fsx = Scriptsuild.fsx + build.fsx = Scripts/build.fsx EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ankit.Native", "src\Ankit.Native\Ankit.Native.csproj", "{452916EC-79B6-4C8B-8528-369D929FD39B}" diff --git a/paket.dependencies b/paket.dependencies index 540bf45..48ff1ea 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -5,6 +5,8 @@ group WebServer framework: net10.0 nuget Oxpecker 2.1.1 + nuget FsToolkit.ErrorHandling 5.2.0 + nuget FSharp.SystemTextJson 1.4.36 group Native source https://api.nuget.org/v3/index.json diff --git a/paket.lock b/paket.lock index 93e1a1b..b7ac3fe 100644 --- a/paket.lock +++ b/paket.lock @@ -46,6 +46,11 @@ RESTRICTION: == net10.0 NUGET remote: https://api.nuget.org/v3/index.json FSharp.Core (10.1.402) + FSharp.SystemTextJson (1.4.36) + FSharp.Core (>= 4.7) + System.Text.Json (>= 6.0.10) + FsToolkit.ErrorHandling (5.2) + FSharp.Core (>= 9.0.300) Microsoft.Extensions.ObjectPool (10.0.13) Microsoft.IO.RecyclableMemoryStream (3.0.1) Oxpecker (2.1.1) @@ -55,5 +60,6 @@ NUGET Oxpecker.ViewEngine (2.1) FSharp.Core (>= 10.0.100) Microsoft.Extensions.ObjectPool (>= 10.0) + System.Text.Json (10.0.13) TypeShape (10.0) FSharp.Core (>= 4.5.4) diff --git a/src/Ankit.Server/AnkiSyncWorker.fs b/src/Ankit.Server/AnkiSyncWorker.fs new file mode 100644 index 0000000..0d27857 --- /dev/null +++ b/src/Ankit.Server/AnkiSyncWorker.fs @@ -0,0 +1,91 @@ +namespace Ankit.Server + +open System +open System.Threading +open System.Threading.Tasks +open Ankit.Native +open Ankit.Server.Env +open Ankit.Server.Services +open Microsoft.Extensions.Hosting +open System.Threading.Channels +open Microsoft.Extensions.Logging +open Microsoft.Extensions.Options + +[] +type AnkiError = + | CreateDeckError + | AddNoteError + +type AnkiCommand = + | CreateDeck of Anki.Decks.Deck + | AddNote of Anki.Notes.AddNoteRequest + +type AnkiSyncMsg = + | AnkiSyncMsg of cmd: AnkiCommand * reply: TaskCompletionSource> + +type AnkiSyncWorker(backend: RslibBackend, syncService: ISyncService, ch: Channel, logger: ILogger, settings: IOptionsMonitor) = + inherit BackgroundService() + + let getDebounceTime() = + TimeSpan.FromMilliseconds(float settings.CurrentValue.SyncDebounceTime) + + let applyCommand (cmd: AnkiCommand) : Result = + try + match cmd with + | CreateDeck deck -> + backend.AddDeck deck |> ignore + | AddNote noteRequest -> + backend.AddNote noteRequest |> ignore + Ok () + with :? RslibException as e -> + logger.LogError(e, "Failed to apply {Command}", cmd) + match cmd with + | CreateDeck _ -> Error AnkiError.CreateDeckError + | AddNote _ -> Error AnkiError.AddNoteError + + override this.ExecuteAsync(stoppingToken) = + task { + logger.LogInformation("AnkiSyncWorker is starting.") + + let rd = ch.Reader + let requiresSync = ref false + + let applyPending() = + let mutable msg = Unchecked.defaultof + while rd.TryRead(&msg) do + let (AnkiSyncMsg(cmd, reply)) = msg + let result = applyCommand cmd + if Result.isOk result then requiresSync.Value <- true + reply.TrySetResult result |> ignore + + try + let mutable running = true + while running do + let! canRead = rd.WaitToReadAsync(stoppingToken) + if not canRead then + running <- false + else + applyPending() + use window = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken) + window.CancelAfter(getDebounceTime()) + try + let mutable windowOpen = true + while windowOpen do + let! more = rd.WaitToReadAsync(window.Token) + if more then applyPending() else windowOpen <- false + with :? OperationCanceledException when not stoppingToken.IsCancellationRequested -> + () + + if requiresSync.Value then + requiresSync.Value <- false + syncService.TrySync() |> ignore + with :? OperationCanceledException when stoppingToken.IsCancellationRequested -> + () + + let mutable msg = Unchecked.defaultof + while rd.TryRead(&msg) do + let (AnkiSyncMsg(_, reply)) = msg + reply.TrySetCanceled(stoppingToken) |> ignore + + logger.LogInformation("AnkiSyncWorker is stopping due to cancellation.") + } diff --git a/src/Ankit.Server/Ankit.Server.fsproj b/src/Ankit.Server/Ankit.Server.fsproj index c5cbd04..5fb1153 100644 --- a/src/Ankit.Server/Ankit.Server.fsproj +++ b/src/Ankit.Server/Ankit.Server.fsproj @@ -7,6 +7,13 @@ + + + + + + + @@ -18,5 +25,13 @@ + + + + + + + + diff --git a/src/Ankit.Server/Dto.fs b/src/Ankit.Server/Dto.fs new file mode 100644 index 0000000..9c97be5 --- /dev/null +++ b/src/Ankit.Server/Dto.fs @@ -0,0 +1,129 @@ +module Ankit.Server.Dto + +open System +open System.ComponentModel.DataAnnotations +open System.Threading.Tasks +open Ankit.Server.Utils +open Microsoft.AspNetCore.Http +open Microsoft.FSharp.Reflection +open Oxpecker + +open type Microsoft.AspNetCore.Http.StatusCodes + +[] +type CreateDeckDto = { + [] + Name: string +} + +[] +type AddNoteDto = { + [] + DeckId: int64 + + [] + NoteTypeId: int64 + + [] + Fields: string array + + Tags: string array option +} + +[] +type FieldError = { + field: string + issue: string + rejectedValue: obj option +} + +[] +type ValidationError = { + code: int + message: string + traceId: string + timestamp: string + path: string + fieldErrors: FieldError[] +} + +[] +type ApiError = { + code: int + message: string + traceId: string + timestamp: string + path: string + details: obj option +} + +let fetchInvalidFieldsFromModel (model: 'T) (errors: ValidationResult seq) = + // Option fields report the wrapped value, None and null report nothing + let unwrap (value: obj) : obj option = + if isNull value then + None + else + let ty = value.GetType() + if ty.IsGenericType && ty.GetGenericTypeDefinition() = typedefof> then + FSharpValue.GetUnionFields(value, ty) |> snd |> Array.tryHead + else + Some value + + let lookUpValue (name: string) : obj option = + let m = box model + if isNull m || not (FSharpType.IsRecord(m.GetType())) then + None + else + FSharpType.GetRecordFields(m.GetType()) + |> Array.tryFind (fun p -> p.Name = name) + |> Option.bind (fun p -> FSharpValue.GetRecordField(m, p) |> unwrap) + + errors + |> Seq.collect (fun err -> + // A rule may cover several members, each of them gets its own entry + match err.MemberNames |> Seq.map _.Trim() |> List.ofSeq with + | [] -> [ { field = ""; issue = err.ErrorMessage; rejectedValue = None } ] + | names -> + names + |> List.map (fun name -> { + // Same naming as the JSON body, so it matches the fields the client sent + field = jsonOptions.PropertyNamingPolicy.ConvertName name + issue = err.ErrorMessage + rejectedValue = lookUpValue name + })) + |> Seq.toArray + +let buildValidationError (invalidFields: FieldError[]) (ctx: HttpContext) = { + ValidationError.code = Status422UnprocessableEntity + message = "Validation failed" + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + fieldErrors = invalidFields +} + +let validateModel (f: 'T -> EndpointHandler) : ModelValidationResult<'T> -> HttpContext -> Task = + (fun res ctx -> + match res with + | ModelValidationResult.Valid dto -> f dto ctx + | ModelValidationResult.Invalid(model, errors) -> + task { + let invalidFields = fetchInvalidFieldsFromModel model errors.All + let valError = buildValidationError invalidFields ctx + + ctx.SetStatusCode(Status422UnprocessableEntity) + return! json valError ctx + }) + +let toDeck (dto: CreateDeckDto) : Anki.Decks.Deck = + // Mirrors Deck::new_normal in rslib + Anki.Decks.Deck( + Name = dto.Name.Trim(), + Common = Anki.Decks.Deck.Types.Common(StudyCollapsed = true, BrowserCollapsed = true), + Normal = Anki.Decks.Deck.Types.Normal(ConfigId = 1L)) + +let toAddNoteRequest (dto: AddNoteDto) : Anki.Notes.AddNoteRequest = + let note = Anki.Notes.Note(NotetypeId = dto.NoteTypeId) + note.Fields.AddRange dto.Fields + dto.Tags |> Option.iter note.Tags.AddRange + Anki.Notes.AddNoteRequest(Note = note, DeckId = dto.DeckId) diff --git a/src/Ankit.Server/Env.fs b/src/Ankit.Server/Env.fs new file mode 100644 index 0000000..a3e8d2d --- /dev/null +++ b/src/Ankit.Server/Env.fs @@ -0,0 +1,40 @@ +module Ankit.Server.Env + +open System + +let tryGetEnv (key: string) : string option = + match Environment.GetEnvironmentVariable(key) with + | null -> None + | value -> Some value + +let tryGetEnvOrDefault (key: string) (defaultValue: string) : string = + tryGetEnv key |> Option.defaultValue defaultValue + +[] +let AnkitServerSettingsSection = "Ankit" + +[] +type AnkitServerSettings = { + SyncDebounceTime: int64 + CollectionPath: string +} + +[] +let SyncServerSettingsSection = "SyncServer" + +[] +type SyncServerSettings = { + Host: string + Port: int + UseHttps: bool + Username: string + Password: string +} with + member this.Endpoint = + UriBuilder( + Host = this.Host, + Port = this.Port, + Scheme = if this.UseHttps then "https" else "http" + ).ToString() + +let defaultLangs = ["en"] diff --git a/src/Ankit.Server/ErrorHandler.fs b/src/Ankit.Server/ErrorHandler.fs new file mode 100644 index 0000000..6aec603 --- /dev/null +++ b/src/Ankit.Server/ErrorHandler.fs @@ -0,0 +1,113 @@ +module Ankit.Server.ErrorHandler + +open System +open System.Text.Json +open System.Threading.Tasks +open Ankit.Server.Dto +open Microsoft.AspNetCore.Http +open Microsoft.Extensions.Logging +open Oxpecker + +open type Microsoft.AspNetCore.Http.TypedResults +open type Microsoft.AspNetCore.Http.StatusCodes + +let notFoundHandler: EndpointHandler = + fun ctx -> + ctx.Write + <| NotFound {| + message = "Requested method doesn't exist" + |} + +let private (|RejectedJsonField|_|) (ex: exn) : string option = + match ex with + | :? JsonException as ex when ex.Message.StartsWith("Missing") -> + Some (ex.Message.Split(' ') |> Array.last) + | :? JsonException as ex when ex.Message.EndsWith("but was null.") -> + let path = ex.Message.Split(' ') |> Array.head + Some (path.Substring(path.LastIndexOf('.') + 1)) + | _ -> None + +let errorHandler (ctx: HttpContext) (next: RequestDelegate) = + task { + let logger = ctx.GetLogger("Ankit.Server.ErrorHandler") + + try + return! next.Invoke(ctx) + with + | :? OperationCanceledException when ctx.RequestAborted.IsCancellationRequested -> + logger.LogDebug("Request was aborted by the client") + | :? ModelBindException as ex -> + logger.LogWarning(ex, "Reading request body for validation errors") + + match ex.InnerException with + | RejectedJsonField field -> + let fieldErr = { + field = field + issue = ex.InnerException.Message + rejectedValue = None + } + + let code = Status422UnprocessableEntity + + let valError = { + ValidationError.code = code + message = "Validation failed" + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + fieldErrors = [| fieldErr |] + } + + ctx.SetStatusCode(code) + return! json valError ctx + | ex -> + logger.LogWarning(ex, "Model bind error") + + let code = Status400BadRequest + + let error = { + code = code + message = "Model binding failed: " + ex.Message + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + details = Some {| errors = ex.Message |} + } + + ctx.SetStatusCode(code) + return! json error ctx + + | :? RouteParseException as ex -> + logger.LogWarning(ex, "Route parse error") + + let code = Status400BadRequest + + let error = { + code = code + message = "Invalid route:" + ex.Message + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + details = None + } + + ctx.SetStatusCode(code) + return! json error ctx + | ex -> + logger.LogError(ex, "Unhandled error") + + let code = Status500InternalServerError + + let error = { + code = code + message = "Internal server error: an unexpected error occurred" + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + details = None + } + + ctx.SetStatusCode(code) + return! json error ctx + } + :> Task diff --git a/src/Ankit.Server/Handlers.fs b/src/Ankit.Server/Handlers.fs new file mode 100644 index 0000000..ecaba6b --- /dev/null +++ b/src/Ankit.Server/Handlers.fs @@ -0,0 +1,76 @@ +module Ankit.Server.Handlers + +open System +open System.Threading.Channels +open System.Threading.Tasks +open Ankit.Server +open Ankit.Server.Dto +open Microsoft.AspNetCore.Http +open Microsoft.Extensions.Logging +open Oxpecker +open type Microsoft.AspNetCore.Http.StatusCodes + +let ping = + (setStatusCode Status200OK) >=> (text "PONG!") + +let writeError (ctx: HttpContext) (code: int) (msg: string) (details: obj option) = + let logger = ctx.GetLogger() + logger.LogWarning("Request failed with {StatusCode}: {Message}", code, msg) + + let error = { + code = code + message = msg + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + details = details + } + + ctx.SetStatusCode(code) + + ctx.WriteJson <| error + +let private describe (error: AnkiError) = + match error with + | AnkiError.CreateDeckError -> "Failed to create the deck" + | AnkiError.AddNoteError -> "Failed to add the note, check the deck, the notetype and the number of fields" + +let private runCommand (ctx: HttpContext) (cmd: AnkiCommand) : Task = + task { + let reply = TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously) + let ch = ctx.GetService>() + do! ch.Writer.WriteAsync(AnkiSyncMsg(cmd, reply), ctx.RequestAborted) + try + match! reply.Task.WaitAsync(ctx.RequestAborted) with + | Ok () -> ctx.SetStatusCode Status204NoContent + | Error e -> return! writeError ctx Status422UnprocessableEntity (describe e) None + with :? TaskCanceledException when not ctx.RequestAborted.IsCancellationRequested -> + return! writeError ctx Status503ServiceUnavailable "Server is shutting down" None + } + +module ModelBinder = + let bindOrDeclineJson<'T when 'T: not null> (f: 'T -> EndpointHandler) : EndpointHandler = + bindAndValidateJson<'T> <| Dto.validateModel f + + let bindOrDeclineJson1 (f: 'T0 -> 'T1 -> EndpointHandler) : 'T1 -> EndpointHandler = + fun x -> bindOrDeclineJson (f >> (fun g -> g x)) + + let bindOrDeclineQuery<'T when 'T: not null> (f: 'T -> EndpointHandler) : EndpointHandler = + bindAndValidateQuery<'T> <| Dto.validateModel f + + let bindOrDeclineQuery1 (f: 'T0 -> 'T1 -> EndpointHandler) : 'T1 -> EndpointHandler = + fun x -> bindOrDeclineQuery (f >> (fun g -> g x)) + +let createDeck (createDeck: CreateDeckDto) : EndpointHandler = + fun (ctx: HttpContext) -> + task { + let deck = toDeck createDeck + return! runCommand ctx (CreateDeck deck) + } + +let addNote (addNote: AddNoteDto) : EndpointHandler = + fun (ctx: HttpContext) -> + task { + let note = toAddNoteRequest addNote + return! runCommand ctx (AddNote note) + } diff --git a/src/Ankit.Server/Program.fs b/src/Ankit.Server/Program.fs index e0827a9..1350d9f 100644 --- a/src/Ankit.Server/Program.fs +++ b/src/Ankit.Server/Program.fs @@ -1,2 +1,66 @@ -// For more information see https://aka.ms/fsharp-console-apps -printfn "Hello from F#" +module Ankit.Server.Main + +open System +open System.IO +open System.Threading.Channels +open Anki.Collection +open Ankit.Server +open Ankit.Server.Handlers.ModelBinder +open Ankit.Server.Env +open Ankit.Server.Services +open Ankit.Native +open Ankit.Server.Utils +open Oxpecker +open Microsoft.AspNetCore.Builder +open Microsoft.Extensions.Configuration +open Microsoft.Extensions.DependencyInjection +open Microsoft.Extensions.Options + +let endpoints = [ + GET [ + route "/ping" Handlers.ping + ] + POST [ + route "/decks" <| bindOrDeclineJson Handlers.createDeck + route "/notes" <| bindOrDeclineJson Handlers.addNote + ] +] + +let openBackend (sp: IServiceProvider) : RslibBackend = + let collectionPath = sp.GetRequiredService>().Value.CollectionPath |> Path.GetFullPath + Directory.CreateDirectory(Path.GetDirectoryName collectionPath) |> ignore + + let backend = RslibBackend.Open defaultLangs + backend.OpenCollection(OpenCollectionRequest( + CollectionPath = collectionPath, + MediaFolderPath = Path.ChangeExtension(collectionPath, ".media"), + MediaDbPath = Path.ChangeExtension(collectionPath, ".media.db2"))) + backend + +let configureServices (configuration: IConfiguration) (services: IServiceCollection) = + services + .Configure(configuration.GetSection AnkitServerSettingsSection) + .Configure(configuration.GetSection SyncServerSettingsSection) + .AddRouting() + .AddOxpecker() + .AddSingleton(SystemTextJsonSerializer(jsonOptions)) + .AddSingleton(openBackend) + .AddSingleton>(Channel.CreateUnbounded(UnboundedChannelOptions(SingleReader = true))) + .AddSingleton() + .AddHostedService() + |> ignore + +let configureApp (app: IApplicationBuilder) = + app + .UseRouting() + .Use(ErrorHandler.errorHandler) + .UseOxpecker(endpoints) + .Run(ErrorHandler.notFoundHandler) + +let builder = WebApplication.CreateBuilder() +configureServices builder.Configuration builder.Services + +let app = builder.Build() +configureApp app + +app.Run() diff --git a/src/Ankit.Server/Services.fs b/src/Ankit.Server/Services.fs new file mode 100644 index 0000000..43b0c6d --- /dev/null +++ b/src/Ankit.Server/Services.fs @@ -0,0 +1,107 @@ +module Ankit.Server.Services + +open Anki.Backend +open Anki.Sync +open Ankit.Native +open Ankit.Server.Env +open Microsoft.Extensions.Logging +open Microsoft.Extensions.Options +open FsToolkit.ErrorHandling + +open type SyncCollectionResponse.Types +open type SyncStatusResponse.Types + +[] +type SyncFailed = + | InvalidCredentials + | Aborted + | NetworkError of message: string + /// The server refused the sync and explained why, e.g. the account is locked + | ServerRejected of message: string + /// The collections diverged and one side has to overwrite the other, which needs a human decision + | FullSyncRequired of SyncCollectionResponse.Types.ChangesRequired + | BackendFailed of kind: BackendError.Types.Kind * message: string + | NativeFailed of message: string + +type SyncResult<'a> = Result<'a, SyncFailed> +type BackendErrorKind = BackendError.Types.Kind + +[] +type ISyncService = + abstract TrySync: unit -> SyncResult + +type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor, logger: ILogger) = + + let classify (error: BackendError) : SyncFailed = + match error.Kind with + | BackendErrorKind.SyncAuthError -> SyncFailed.InvalidCredentials + | BackendErrorKind.Interrupted -> SyncFailed.Aborted + | BackendErrorKind.NetworkError -> SyncFailed.NetworkError error.Message + | BackendErrorKind.SyncServerMessage -> SyncFailed.ServerRejected error.Message + | kind -> SyncFailed.BackendFailed(kind, error.Message) + + let call (f: unit -> 'a) : SyncResult<'a> = + try + Ok <| f() + with + | :? BackendException as e -> Error <| classify e.Error + | :? RslibException as e -> Error <| SyncFailed.NativeFailed e.Message + + /// The server may redirect us to another shard, subsequent requests have to go there + let followRedirect (auth: SyncAuth) (newEndpoint: string option) : SyncAuth = + match newEndpoint with + | Some endpoint when endpoint <> auth.Endpoint -> + logger.LogInformation("Sync server moved the collection to {Endpoint}", endpoint) + let redirected = auth.Clone() + redirected.Endpoint <- endpoint + redirected + | _ -> auth + + let getAuth() : SyncResult = + let loginRequest = SyncLoginRequest( + Username = syncServerSettings.CurrentValue.Username, + Password = syncServerSettings.CurrentValue.Password, + Endpoint = syncServerSettings.CurrentValue.Endpoint) + call (fun () -> backend.SyncLogin loginRequest) + + let syncAll (auth: SyncAuth) : SyncResult = + result { + let request = SyncCollectionRequest(Auth = auth, SyncMedia = true) + let! response = call (fun () -> backend.SyncCollection request) + + if not (System.String.IsNullOrEmpty response.ServerMessage) then + logger.LogWarning("Sync server says: {ServerMessage}", response.ServerMessage) + + let auth = followRedirect auth (if response.HasNewEndpoint then Some response.NewEndpoint else None) + + match response.Required with + | ChangesRequired.NoChanges + | ChangesRequired.NormalSync -> return auth + | required -> return! Error <| SyncFailed.FullSyncRequired required + } + + let getSyncStatus (auth: SyncAuth) : SyncResult = + call (fun () -> backend.SyncStatus auth) + + let logSyncStatus (status: SyncStatusResponse) = + match status.Required with + | Required.NoChanges -> + logger.LogInformation("Sync finished, collection is up to date") + | required -> + logger.LogWarning("Sync finished, but the collection still requires {Required}", required) + + let logFailure (failure: SyncFailed) = + match failure with + | SyncFailed.Aborted -> logger.LogInformation("Sync was aborted") + | SyncFailed.NetworkError message -> logger.LogWarning("Sync failed, sync server is unreachable: {Message}", message) + | failure -> logger.LogError("Sync failed: {Failure}", failure) + + interface ISyncService with + member this.TrySync() = + result { + let! auth = getAuth() + let! auth = syncAll auth + let! status = getSyncStatus auth + logSyncStatus status + } + |> Result.teeError logFailure diff --git a/src/Ankit.Server/Utils.fs b/src/Ankit.Server/Utils.fs new file mode 100644 index 0000000..bdf1435 --- /dev/null +++ b/src/Ankit.Server/Utils.fs @@ -0,0 +1,35 @@ +module Ankit.Server.Utils + +open System.ComponentModel.DataAnnotations +open System.Text.Json +open System.Text.Json.Serialization + +let jsonOptions = + let opts = JsonSerializerOptions() + opts.PropertyNamingPolicy <- JsonNamingPolicy.CamelCase + opts.DefaultIgnoreCondition <- JsonIgnoreCondition.Never + opts.AllowDuplicateProperties <- false + opts.AllowTrailingCommas <- false + opts.PropertyNameCaseInsensitive <- true + opts.WriteIndented <- true + + JsonFSharpOptions + .Default() + .WithUnionUnwrapFieldlessTags() + .WithUnionTagNamingPolicy(JsonNamingPolicy.SnakeCaseUpper) + .WithUnionNamedFields() + .WithUnionTagName("code") + .WithUnionInternalTag() + .WithSkippableOptionFields() + .AddToJsonSerializerOptions(opts) + + opts + +type NumericIdAttribute() = + inherit ValidationAttribute("Must be a positive id") + + override this.IsValid(value) = + match value with + | :? int64 as i -> i >= 1L + | :? int32 as i -> i >= 1 + | _ -> true diff --git a/src/Ankit.Server/paket.references b/src/Ankit.Server/paket.references index db4f7f8..d955110 100644 --- a/src/Ankit.Server/paket.references +++ b/src/Ankit.Server/paket.references @@ -1,2 +1,4 @@ group WebServer - Oxpecker \ No newline at end of file + Oxpecker + FsToolkit.ErrorHandling + FSharp.SystemTextJson From d561e922be678cf8ddf6d5387fb47ec3ec767014 Mon Sep 17 00:00:00 2001 From: evgTSV Date: Sat, 24 Oct 2026 20:13:21 +0500 Subject: [PATCH 2/6] Add appsettings.json for the server Defaults for the Ankit and SyncServer sections, listening on port 5000 to match compose.yaml. Sync credentials are left empty and are meant to come from environment variables or user-secrets. Co-Authored-By: Claude Opus 5.5 --- src/Ankit.Server/Ankit.Server.fsproj | 1 + src/Ankit.Server/appsettings.json | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 src/Ankit.Server/appsettings.json diff --git a/src/Ankit.Server/Ankit.Server.fsproj b/src/Ankit.Server/Ankit.Server.fsproj index 5fb1153..2d6ea22 100644 --- a/src/Ankit.Server/Ankit.Server.fsproj +++ b/src/Ankit.Server/Ankit.Server.fsproj @@ -21,6 +21,7 @@ .dockerignore + diff --git a/src/Ankit.Server/appsettings.json b/src/Ankit.Server/appsettings.json new file mode 100644 index 0000000..e244c13 --- /dev/null +++ b/src/Ankit.Server/appsettings.json @@ -0,0 +1,20 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "Urls": "http://0.0.0.0:5000", + "Ankit": { + "SyncDebounceTime": 5000, + "CollectionPath": "data/collection.anki2" + }, + "SyncServer": { + "Host": "localhost", + "Port": 8080, + "UseHttps": false, + "Username": "", + "Password": "" + } +} From c3c40af27521c9825adbeadd98cd91a6aa197e2c Mon Sep 17 00:00:00 2001 From: evgTSV Date: Sat, 24 Oct 2026 21:30:06 +0500 Subject: [PATCH 3/6] Add full sync endpoint POST /sync/full with "upload" or "download" resolves FullSyncRequired, which the first sync of a non-empty collection always runs into. The full sync goes through the worker like the other commands, media is synced right after it, and failures come back as 502. Co-Authored-By: Claude Opus 5.5 --- src/Ankit.Server/AnkiSyncWorker.fs | 35 +++++++++++++++++++----------- src/Ankit.Server/Dto.fs | 8 +++++++ src/Ankit.Server/Handlers.fs | 18 ++++++++++----- src/Ankit.Server/Program.fs | 1 + src/Ankit.Server/Services.fs | 23 ++++++++++++++++++++ 5 files changed, 67 insertions(+), 18 deletions(-) diff --git a/src/Ankit.Server/AnkiSyncWorker.fs b/src/Ankit.Server/AnkiSyncWorker.fs index 0d27857..284ee28 100644 --- a/src/Ankit.Server/AnkiSyncWorker.fs +++ b/src/Ankit.Server/AnkiSyncWorker.fs @@ -15,10 +15,12 @@ open Microsoft.Extensions.Options type AnkiError = | CreateDeckError | AddNoteError + | FullSyncError of SyncFailed type AnkiCommand = | CreateDeck of Anki.Decks.Deck | AddNote of Anki.Notes.AddNoteRequest + | FullSync of SyncDirection type AnkiSyncMsg = | AnkiSyncMsg of cmd: AnkiCommand * reply: TaskCompletionSource> @@ -30,18 +32,22 @@ type AnkiSyncWorker(backend: RslibBackend, syncService: ISyncService, ch: Channe TimeSpan.FromMilliseconds(float settings.CurrentValue.SyncDebounceTime) let applyCommand (cmd: AnkiCommand) : Result = - try - match cmd with - | CreateDeck deck -> - backend.AddDeck deck |> ignore - | AddNote noteRequest -> - backend.AddNote noteRequest |> ignore - Ok () - with :? RslibException as e -> - logger.LogError(e, "Failed to apply {Command}", cmd) - match cmd with - | CreateDeck _ -> Error AnkiError.CreateDeckError - | AddNote _ -> Error AnkiError.AddNoteError + match cmd with + | FullSync direction -> + syncService.FullSync direction |> Result.mapError AnkiError.FullSyncError + | CreateDeck _ + | AddNote _ -> + try + match cmd with + | CreateDeck deck -> backend.AddDeck deck |> ignore + | AddNote noteRequest -> backend.AddNote noteRequest |> ignore + | FullSync _ -> () + Ok () + with :? RslibException as e -> + logger.LogError(e, "Failed to apply {Command}", cmd) + match cmd with + | CreateDeck _ -> Error AnkiError.CreateDeckError + | _ -> Error AnkiError.AddNoteError override this.ExecuteAsync(stoppingToken) = task { @@ -55,7 +61,10 @@ type AnkiSyncWorker(backend: RslibBackend, syncService: ISyncService, ch: Channe while rd.TryRead(&msg) do let (AnkiSyncMsg(cmd, reply)) = msg let result = applyCommand cmd - if Result.isOk result then requiresSync.Value <- true + match cmd, result with + | FullSync _, Ok () -> requiresSync.Value <- false + | _, Ok () -> requiresSync.Value <- true + | _, Error _ -> () reply.TrySetResult result |> ignore try diff --git a/src/Ankit.Server/Dto.fs b/src/Ankit.Server/Dto.fs index 9c97be5..c58fbcd 100644 --- a/src/Ankit.Server/Dto.fs +++ b/src/Ankit.Server/Dto.fs @@ -3,6 +3,7 @@ module Ankit.Server.Dto open System open System.ComponentModel.DataAnnotations open System.Threading.Tasks +open Ankit.Server.Services open Ankit.Server.Utils open Microsoft.AspNetCore.Http open Microsoft.FSharp.Reflection @@ -30,6 +31,13 @@ type AddNoteDto = { Tags: string array option } +[] +type FullSyncDto = { + /// "upload" replaces the server's collection with ours, "download" replaces ours with the server's + [] + Direction: SyncDirection +} + [] type FieldError = { field: string diff --git a/src/Ankit.Server/Handlers.fs b/src/Ankit.Server/Handlers.fs index ecaba6b..8b5c8f5 100644 --- a/src/Ankit.Server/Handlers.fs +++ b/src/Ankit.Server/Handlers.fs @@ -34,6 +34,12 @@ let private describe (error: AnkiError) = match error with | AnkiError.CreateDeckError -> "Failed to create the deck" | AnkiError.AddNoteError -> "Failed to add the note, check the deck, the notetype and the number of fields" + | AnkiError.FullSyncError failure -> $"Full sync failed: %A{failure}" + +let private statusOf (error: AnkiError) = + match error with + | AnkiError.FullSyncError _ -> Status502BadGateway + | _ -> Status422UnprocessableEntity let private runCommand (ctx: HttpContext) (cmd: AnkiCommand) : Task = task { @@ -43,7 +49,7 @@ let private runCommand (ctx: HttpContext) (cmd: AnkiCommand) : Task = try match! reply.Task.WaitAsync(ctx.RequestAborted) with | Ok () -> ctx.SetStatusCode Status204NoContent - | Error e -> return! writeError ctx Status422UnprocessableEntity (describe e) None + | Error e -> return! writeError ctx (statusOf e) (describe e) None with :? TaskCanceledException when not ctx.RequestAborted.IsCancellationRequested -> return! writeError ctx Status503ServiceUnavailable "Server is shutting down" None } @@ -69,8 +75,10 @@ let createDeck (createDeck: CreateDeckDto) : EndpointHandler = } let addNote (addNote: AddNoteDto) : EndpointHandler = + let note = toAddNoteRequest addNote fun (ctx: HttpContext) -> - task { - let note = toAddNoteRequest addNote - return! runCommand ctx (AddNote note) - } + runCommand ctx (AddNote note) + +let fullSync (fullSync: FullSyncDto) : EndpointHandler = + fun (ctx: HttpContext) -> + runCommand ctx (FullSync fullSync.Direction) diff --git a/src/Ankit.Server/Program.fs b/src/Ankit.Server/Program.fs index 1350d9f..b234e25 100644 --- a/src/Ankit.Server/Program.fs +++ b/src/Ankit.Server/Program.fs @@ -23,6 +23,7 @@ let endpoints = [ POST [ route "/decks" <| bindOrDeclineJson Handlers.createDeck route "/notes" <| bindOrDeclineJson Handlers.addNote + route "/sync/full" <| bindOrDeclineJson Handlers.fullSync ] ] diff --git a/src/Ankit.Server/Services.fs b/src/Ankit.Server/Services.fs index 43b0c6d..14168cf 100644 --- a/src/Ankit.Server/Services.fs +++ b/src/Ankit.Server/Services.fs @@ -26,9 +26,14 @@ type SyncFailed = type SyncResult<'a> = Result<'a, SyncFailed> type BackendErrorKind = BackendError.Types.Kind +type SyncDirection = + | Upload + | Download + [] type ISyncService = abstract TrySync: unit -> SyncResult + abstract FullSync: SyncDirection -> SyncResult type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor, logger: ILogger) = @@ -80,6 +85,14 @@ type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor< | required -> return! Error <| SyncFailed.FullSyncRequired required } + let fullSync (auth: SyncAuth) (direction: SyncDirection) : SyncResult = + result { + let request = FullUploadOrDownloadRequest(Auth = auth, Upload = (direction = Upload)) + do! call (fun () -> backend.FullUploadOrDownload request) + // rslib skips media without the server's media usn, so it's synced on its own + do! call (fun () -> backend.SyncMedia auth) + } + let getSyncStatus (auth: SyncAuth) : SyncResult = call (fun () -> backend.SyncStatus auth) @@ -105,3 +118,13 @@ type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor< logSyncStatus status } |> Result.teeError logFailure + + member this.FullSync(direction) = + result { + let! auth = getAuth() + logger.LogWarning("Starting full {Direction}, the other side's collection is replaced", direction) + do! fullSync auth direction + let! status = getSyncStatus auth + logSyncStatus status + } + |> Result.teeError logFailure From 7ab172183c9d7dd156db1cda21b32f17daca3c81 Mon Sep 17 00:00:00 2001 From: evgTSV Date: Sat, 24 Oct 2026 21:30:07 +0500 Subject: [PATCH 4/6] Test the sync service against a real sync server The server is built with Testcontainers from Anki's example Dockerfile at the Anki tag the rslib bindings use, with cargo limited to 4 jobs so the build doesn't run out of memory. The tests skip without Docker running Linux containers. Co-Authored-By: Claude Opus 5.5 --- paket.dependencies | 1 + paket.lock | 33 +++++++ tests/Ankit.Server.Tests/AnkiSyncServer.fs | 94 +++++++++++++++++++ .../Ankit.Server.Tests.fsproj | 4 +- .../Ankit.Server.Tests/SyncServer/Dockerfile | 33 +++++++ .../SyncServer/entrypoint.sh | 30 ++++++ tests/Ankit.Server.Tests/SyncServiceTests.fs | 84 +++++++++++++++++ tests/Ankit.Server.Tests/TempCollection.fs | 28 ++++++ tests/Ankit.Server.Tests/Tests.fs | 7 -- tests/Ankit.Server.Tests/paket.references | 1 + 10 files changed, 307 insertions(+), 8 deletions(-) create mode 100644 tests/Ankit.Server.Tests/AnkiSyncServer.fs create mode 100644 tests/Ankit.Server.Tests/SyncServer/Dockerfile create mode 100644 tests/Ankit.Server.Tests/SyncServer/entrypoint.sh create mode 100644 tests/Ankit.Server.Tests/SyncServiceTests.fs create mode 100644 tests/Ankit.Server.Tests/TempCollection.fs delete mode 100644 tests/Ankit.Server.Tests/Tests.fs diff --git a/paket.dependencies b/paket.dependencies index 48ff1ea..2145bf5 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -27,3 +27,4 @@ group Test nuget xunit nuget xunit.runner.visualstudio nuget coverlet.collector + nuget Testcontainers diff --git a/paket.lock b/paket.lock index b7ac3fe..d07fabe 100644 --- a/paket.lock +++ b/paket.lock @@ -16,15 +16,48 @@ GROUP Test RESTRICTION: == net10.0 NUGET remote: https://api.nuget.org/v3/index.json + BouncyCastle.Cryptography (2.7) coverlet.collector (10.1) + Docker.DotNet.Enhanced (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.LegacyHttp (>= 4.3.3) + Docker.DotNet.Enhanced.NativeHttp (>= 4.3.3) + Docker.DotNet.Enhanced.NPipe (>= 4.3.3) + Docker.DotNet.Enhanced.Unix (>= 4.3.3) + Microsoft.Extensions.Logging.Abstractions (>= 8.0.3) + Docker.DotNet.Enhanced.Handler.Abstractions (4.3.3) + Microsoft.Extensions.Logging.Abstractions (>= 8.0.3) + Docker.DotNet.Enhanced.LegacyHttp (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.NativeHttp (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.NPipe (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.Unix (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.X509 (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) FSharp.Core (10.1.402) Microsoft.CodeCoverage (18.11) + Microsoft.Extensions.DependencyInjection.Abstractions (10.0.13) + Microsoft.Extensions.Logging.Abstractions (10.0.13) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 10.0.13) Microsoft.NET.Test.Sdk (18.11) Microsoft.CodeCoverage (>= 18.11) Microsoft.TestPlatform.TestHost (>= 18.11) Microsoft.TestPlatform.ObjectModel (18.11) Microsoft.TestPlatform.TestHost (18.11) Microsoft.TestPlatform.ObjectModel (>= 18.11) + SharpZipLib (1.4.2) + SSH.NET (2026.0) + BouncyCastle.Cryptography (>= 2.7) + Microsoft.Extensions.Logging.Abstractions (>= 8.0.3) + Testcontainers (4.16) + Docker.DotNet.Enhanced (>= 4.3.3) + Docker.DotNet.Enhanced.X509 (>= 4.3.3) + Microsoft.Extensions.Logging.Abstractions (>= 8.0.3) + SharpZipLib (>= 1.4.2) + SSH.NET (>= 2026.0) xunit (2.9.3) xunit.analyzers (>= 1.18) xunit.assert (>= 2.9.3) diff --git a/tests/Ankit.Server.Tests/AnkiSyncServer.fs b/tests/Ankit.Server.Tests/AnkiSyncServer.fs new file mode 100644 index 0000000..925feb8 --- /dev/null +++ b/tests/Ankit.Server.Tests/AnkiSyncServer.fs @@ -0,0 +1,94 @@ +module Ankit.Server.Tests.AnkiSyncServer + +open System +open System.Diagnostics +open System.IO +open System.Text.RegularExpressions +open System.Threading.Tasks +open Ankit.Server.Env +open DotNet.Testcontainers.Builders +open DotNet.Testcontainers.Containers +open Xunit + +let private repoRoot = + let rec find (dir: DirectoryInfo) = + if isNull dir then failwith "paket.dependencies not found above the test output directory" + elif File.Exists(Path.Combine(dir.FullName, "paket.dependencies")) then dir.FullName + else find dir.Parent + find (DirectoryInfo AppContext.BaseDirectory) + +/// The sync server is built from the same Anki tag as the rslib we link against, so the protocol versions match +let private ankiVersion = + let dependencies = File.ReadAllText(Path.Combine(repoRoot, "paket.dependencies")) + Regex.Match(dependencies, @"github ankitects/anki:(\S+)").Groups[1].Value + +let private linuxDockerAvailable = + lazy + try + use docker = Process.Start(ProcessStartInfo("docker", "info --format {{.OSType}}", RedirectStandardOutput = true, RedirectStandardError = true)) + let osType = docker.StandardOutput.ReadToEnd().Trim() + docker.WaitForExit(10_000) && docker.ExitCode = 0 && osType = "linux" + with _ -> + false + +type LinuxDockerFactAttribute() as this = + inherit FactAttribute() + do + if not linuxDockerAvailable.Value then + this.Skip <- "Needs Docker with Linux containers" + +/// Parallel rustc jobs for compiling the sync server, more jobs need more memory in the Docker VM +let private cargoBuildJobs = Environment.GetEnvironmentVariable "ANKIT_TESTS_CARGO_JOBS" |> Option.ofObj |> Option.defaultValue "4" + +[] +let private SyncPort = 8080 + +[] +let Password = "password" + +/// Accounts of the sync server, give every test its own so they don't see each other's collections +let Users = [ for i in 1..5 -> $"user{i}" ] + +type AnkiSyncServer() = + let mutable container: IContainer = null + + member _.Settings(username: string) : SyncServerSettings = { + Host = container.Hostname + Port = int (container.GetMappedPublicPort SyncPort) + UseHttps = false + Username = username + Password = Password + } + + interface IAsyncLifetime with + member _.InitializeAsync() = + task { + if linuxDockerAvailable.Value then + // Compiling the server takes several minutes the first time, Docker caches the layers after that + let image = + ImageFromDockerfileBuilder() + .WithDockerfileDirectory(Path.Combine(repoRoot, "tests", "Ankit.Server.Tests", "SyncServer")) + .WithDockerfile("Dockerfile") + .WithBuildArgument("ANKI_VERSION", ankiVersion) + .WithBuildArgument("CARGO_BUILD_JOBS", cargoBuildJobs) + .WithName($"ankit-tests/anki-sync-server:{ankiVersion}") + .WithCleanUp(false) + .Build() + do! image.CreateAsync() + + let builder = + Users + |> List.indexed + |> List.fold + (fun (b: ContainerBuilder) (i, user) -> b.WithEnvironment($"SYNC_USER{i + 1}", $"{user}:{Password}")) + (ContainerBuilder(image.FullName)) + container <- + builder + .WithPortBinding(SyncPort, true) + .WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded(fun r -> r.ForPort(uint16 SyncPort).ForPath("/health"))) + .Build() + do! container.StartAsync() + } + + member _.DisposeAsync() = + if isNull container then Task.CompletedTask else container.DisposeAsync().AsTask() diff --git a/tests/Ankit.Server.Tests/Ankit.Server.Tests.fsproj b/tests/Ankit.Server.Tests/Ankit.Server.Tests.fsproj index bb440c0..c81dff8 100644 --- a/tests/Ankit.Server.Tests/Ankit.Server.Tests.fsproj +++ b/tests/Ankit.Server.Tests/Ankit.Server.Tests.fsproj @@ -6,7 +6,9 @@ - + + + diff --git a/tests/Ankit.Server.Tests/SyncServer/Dockerfile b/tests/Ankit.Server.Tests/SyncServer/Dockerfile new file mode 100644 index 0000000..085769a --- /dev/null +++ b/tests/Ankit.Server.Tests/SyncServer/Dockerfile @@ -0,0 +1,33 @@ +# Anki's example sync server image (docs/syncserver in the Anki repo), with a limit on parallel +# compile jobs: by default cargo runs one rustc per core and runs out of memory on big machines +FROM rust:1.97.1-alpine3.23 AS builder + +ARG ANKI_VERSION +ARG CARGO_BUILD_JOBS=4 + +RUN apk update && apk add --no-cache build-base protobuf && rm -rf /var/cache/apk/* + +RUN CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} cargo install --git https://github.com/ankitects/anki.git \ +--tag ${ANKI_VERSION} \ +--root /anki-server \ +--locked \ +anki-sync-server + +FROM alpine:3.23 + +ENV PUID=1000 +ENV PGID=1000 + +COPY --from=builder /anki-server/bin/anki-sync-server /usr/local/bin/anki-sync-server + +RUN apk update && apk add --no-cache bash su-exec && rm -rf /var/cache/apk/* + +EXPOSE 8080 + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["anki-sync-server"] + +VOLUME /anki_data diff --git a/tests/Ankit.Server.Tests/SyncServer/entrypoint.sh b/tests/Ankit.Server.Tests/SyncServer/entrypoint.sh new file mode 100644 index 0000000..072ae15 --- /dev/null +++ b/tests/Ankit.Server.Tests/SyncServer/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -o errexit +set -o nounset +set -o pipefail + +# Default PUID and PGID if not provided +export PUID=${PUID:-1000} +export PGID=${PGID:-1000} + +# These values are fixed and cannot be overwritten from the outside for +# convenience and safety reasons +export SYNC_PORT=8080 +export SYNC_BASE=/anki_data + +# Check if group exists, create if not +if ! getent group anki-group > /dev/null 2>&1; then + addgroup -g "$PGID" anki-group +fi + +# Check if user exists, create if not +if ! id -u anki > /dev/null 2>&1; then + adduser -D -H -u "$PUID" -G anki-group anki +fi + +# Fix ownership of mounted volumes +mkdir -p /anki_data +chown anki:anki-group /anki_data + +# Run the provided command as the `anki` user +exec su-exec anki "$@" diff --git a/tests/Ankit.Server.Tests/SyncServiceTests.fs b/tests/Ankit.Server.Tests/SyncServiceTests.fs new file mode 100644 index 0000000..ac9c31a --- /dev/null +++ b/tests/Ankit.Server.Tests/SyncServiceTests.fs @@ -0,0 +1,84 @@ +namespace Ankit.Server.Tests + +open System.Linq +open Anki.Notes +open Anki.Search +open Anki.Sync +open Ankit.Native +open Ankit.Server.Env +open Ankit.Server.Services +open Ankit.Server.Tests.AnkiSyncServer +open Microsoft.Extensions.Logging.Abstractions +open Microsoft.Extensions.Options +open Xunit + +type SyncServiceTests(server: AnkiSyncServer) = + + /// xUnit can't print F# unions, compare the printed values so a failure shows what came back + let assertResult (expected: SyncResult) (actual: SyncResult) = + Assert.Equal(sprintf "%A" expected, sprintf "%A" actual) + + let syncService (backend: RslibBackend) (settings: SyncServerSettings) : ISyncService = + let monitor = + { new IOptionsMonitor with + member _.CurrentValue = settings + member _.Get _ = settings + member _.OnChange _ = null } + AnkiSyncService(backend, monitor, NullLogger.Instance) + + let addBasicNote (backend: RslibBackend) (front: string) (back: string) = + let basic = backend.GetNotetypeNames().Entries.Single(fun e -> e.Name = "Basic") + let note = backend.NewNote(Anki.Notetypes.NotetypeId(Ntid = basic.Id)) + note.Fields[0] <- front + note.Fields[1] <- back + backend.AddNote(AddNoteRequest(Note = note, DeckId = 1L)) |> ignore + + [] + member _.``Wrong password is reported as invalid credentials``() = + use collection = new TempCollection() + let settings = { server.Settings Users[0] with Password = "wrong" } + + assertResult (Error SyncFailed.InvalidCredentials) ((syncService collection.Backend settings).TrySync()) + + [] + member _.``Untouched collection has nothing to sync with a fresh account``() = + use collection = new TempCollection() + + assertResult (Ok()) ((syncService collection.Backend (server.Settings Users[1])).TrySync()) + + [] + member _.``First sync of a collection with notes needs a full upload``() = + use collection = new TempCollection() + addBasicNote collection.Backend "犬" "dog" + + let result = (syncService collection.Backend (server.Settings Users[2])).TrySync() + + assertResult (Error(SyncFailed.FullSyncRequired SyncCollectionResponse.Types.ChangesRequired.FullUpload)) result + + [] + member _.``Changes reach other clients after the initial full sync``() = + let settings = server.Settings Users[3] + use local = new TempCollection() + addBasicNote local.Backend "犬" "dog" + let service = syncService local.Backend settings + assertResult (Ok()) (service.FullSync Upload) + + addBasicNote local.Backend "猫" "cat" + assertResult (Ok()) (service.TrySync()) + assertResult (Ok()) (service.TrySync()) + + use other = new TempCollection() + let otherService = syncService other.Backend settings + assertResult (Error(SyncFailed.FullSyncRequired SyncCollectionResponse.Types.ChangesRequired.FullDownload)) (otherService.TrySync()) + assertResult (Ok()) (otherService.FullSync Download) + assertResult (Ok()) (otherService.TrySync()) + Assert.Equal(2, other.Backend.SearchNotes(SearchRequest(Search = "")).Ids.Count) + + [] + member _.``Full sync with a wrong password is reported as invalid credentials``() = + use collection = new TempCollection() + let settings = { server.Settings Users[4] with Password = "wrong" } + + assertResult (Error SyncFailed.InvalidCredentials) ((syncService collection.Backend settings).FullSync Upload) + + interface IClassFixture diff --git a/tests/Ankit.Server.Tests/TempCollection.fs b/tests/Ankit.Server.Tests/TempCollection.fs new file mode 100644 index 0000000..a0c2f02 --- /dev/null +++ b/tests/Ankit.Server.Tests/TempCollection.fs @@ -0,0 +1,28 @@ +namespace Ankit.Server.Tests + +open System +open System.IO +open Anki.Collection +open Ankit.Native + +/// A backend with a fresh collection in a temporary directory, removed on dispose +type TempCollection() = + let directory = Directory.CreateTempSubdirectory("ankit-server-tests-") + let backend = RslibBackend.Open [ "en" ] + + do + backend.OpenCollection( + OpenCollectionRequest( + CollectionPath = Path.Combine(directory.FullName, "collection.anki2"), + MediaFolderPath = Path.Combine(directory.FullName, "collection.media"), + MediaDbPath = Path.Combine(directory.FullName, "collection.media.db") + ) + ) + + member _.Backend = backend + + interface IDisposable with + member _.Dispose() = + backend.CloseCollection(CloseCollectionRequest()) + backend.Dispose() + directory.Delete(recursive = true) diff --git a/tests/Ankit.Server.Tests/Tests.fs b/tests/Ankit.Server.Tests/Tests.fs deleted file mode 100644 index ae63213..0000000 --- a/tests/Ankit.Server.Tests/Tests.fs +++ /dev/null @@ -1,7 +0,0 @@ -module Tests - -open System -open Xunit - -[] -let ``My test`` () = Assert.True(true) diff --git a/tests/Ankit.Server.Tests/paket.references b/tests/Ankit.Server.Tests/paket.references index fee5b69..8c5d90c 100644 --- a/tests/Ankit.Server.Tests/paket.references +++ b/tests/Ankit.Server.Tests/paket.references @@ -4,3 +4,4 @@ group Test xunit xunit.runner.visualstudio coverlet.collector + Testcontainers From b6857561b7d5f751501408a778d317668f23f82f Mon Sep 17 00:00:00 2001 From: evgTSV Date: Sat, 24 Oct 2026 21:45:36 +0500 Subject: [PATCH 5/6] Apply formatting, add "Verify-Sources" target --- .github/workflows/main.yml | 6 +-- Ankit.sln | 1 + Scripts/build.fsx | 13 +++++- Scripts/gen-workflow.fsx | 3 +- src/Ankit.Server/AnkiSyncWorker.fs | 41 +++++++++++------ src/Ankit.Server/Dto.fs | 19 +++++--- src/Ankit.Server/Env.fs | 17 ++----- src/Ankit.Server/ErrorHandler.fs | 11 ++--- src/Ankit.Server/Handlers.fs | 18 ++++---- src/Ankit.Server/Program.fs | 32 +++++++------ src/Ankit.Server/Services.fs | 46 +++++++++++-------- tests/Ankit.Server.Tests/AnkiSyncServer.fs | 48 ++++++++++++++++---- tests/Ankit.Server.Tests/SyncServiceTests.fs | 14 ++++-- 13 files changed, 167 insertions(+), 102 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6875c1c..50a3485 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,10 +24,8 @@ jobs: uses: actions/setup-dotnet@v6 with: dotnet-version: 10.0.x - - name: Verify encoding - run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Encoding - - name: Verify workflow - run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Workflow + - name: Verify sources + run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Sources build: name: Build and test (${{ matrix.image }}) strategy: diff --git a/Ankit.sln b/Ankit.sln index 978182f..eb5f074 100644 --- a/Ankit.sln +++ b/Ankit.sln @@ -9,6 +9,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution paket.dependencies = paket.dependencies README.md = README.md build.fsx = Scripts/build.fsx + gen-workflow.fsx = Scripts/gen-workflow.fsx EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ankit.Native", "src\Ankit.Native\Ankit.Native.csproj", "{452916EC-79B6-4C8B-8528-369D929FD39B}" diff --git a/Scripts/build.fsx b/Scripts/build.fsx index 37db86e..088de88 100644 --- a/Scripts/build.fsx +++ b/Scripts/build.fsx @@ -177,6 +177,9 @@ Target.create "Restore" (fun _ -> Target.create "Format" (fun _ -> dotnet "fantomas" "." |> handleResult "Format") +Target.create "Verify-Format" (fun _ -> + dotnet "fantomas" "--check ." |> handleResult "Verify-Format") + let utf8Bom = Encoding.UTF8.GetPreamble() let hasBom (bytes: ReadOnlySpan) = @@ -410,6 +413,7 @@ Target.create "Prepare" doNothing Target.create "Build" doNothing Target.create "All" doNothing Target.create "Prepare-Sources" doNothing +Target.create "Verify-Sources" doNothing Target.create "Before-Push" doNothing // --- Prepare: restore packages and tools, generate sources --- @@ -461,11 +465,16 @@ Target.create "Before-Push" doNothing "Format" ==> "Prepare-Sources" -"Before-Push" <== [ - "Prepare-Sources" +"Verify-Sources" <== [ "Verify-Encoding" "Verify-Workflow" + "Verify-Format" +] + +"Before-Push" <== [ "Test" + "Prepare-Sources" + "Verify-Sources" ] if DocoptResult.hasFlag "--list" cli then diff --git a/Scripts/gen-workflow.fsx b/Scripts/gen-workflow.fsx index b020af9..e387d01 100644 --- a/Scripts/gen-workflow.fsx +++ b/Scripts/gen-workflow.fsx @@ -86,8 +86,7 @@ let workflow = [ yield! env checkoutStep setUpDotNetSdkStep - buildCliStep "Verify encoding" "Verify-Encoding" - buildCliStep "Verify workflow" "Verify-Workflow" + buildCliStep "Verify sources" "Verify-Sources" ] job "build" [ diff --git a/src/Ankit.Server/AnkiSyncWorker.fs b/src/Ankit.Server/AnkiSyncWorker.fs index 284ee28..d46596e 100644 --- a/src/Ankit.Server/AnkiSyncWorker.fs +++ b/src/Ankit.Server/AnkiSyncWorker.fs @@ -22,19 +22,24 @@ type AnkiCommand = | AddNote of Anki.Notes.AddNoteRequest | FullSync of SyncDirection -type AnkiSyncMsg = - | AnkiSyncMsg of cmd: AnkiCommand * reply: TaskCompletionSource> +type AnkiSyncMsg = AnkiSyncMsg of cmd: AnkiCommand * reply: TaskCompletionSource> -type AnkiSyncWorker(backend: RslibBackend, syncService: ISyncService, ch: Channel, logger: ILogger, settings: IOptionsMonitor) = +type AnkiSyncWorker + ( + backend: RslibBackend, + syncService: ISyncService, + ch: Channel, + logger: ILogger, + settings: IOptionsMonitor + ) = inherit BackgroundService() - let getDebounceTime() = + let getDebounceTime () = TimeSpan.FromMilliseconds(float settings.CurrentValue.SyncDebounceTime) let applyCommand (cmd: AnkiCommand) : Result = match cmd with - | FullSync direction -> - syncService.FullSync direction |> Result.mapError AnkiError.FullSyncError + | FullSync direction -> syncService.FullSync direction |> Result.mapError AnkiError.FullSyncError | CreateDeck _ | AddNote _ -> try @@ -42,9 +47,11 @@ type AnkiSyncWorker(backend: RslibBackend, syncService: ISyncService, ch: Channe | CreateDeck deck -> backend.AddDeck deck |> ignore | AddNote noteRequest -> backend.AddNote noteRequest |> ignore | FullSync _ -> () - Ok () + + Ok() with :? RslibException as e -> logger.LogError(e, "Failed to apply {Command}", cmd) + match cmd with | CreateDeck _ -> Error AnkiError.CreateDeckError | _ -> Error AnkiError.AddNoteError @@ -56,32 +63,39 @@ type AnkiSyncWorker(backend: RslibBackend, syncService: ISyncService, ch: Channe let rd = ch.Reader let requiresSync = ref false - let applyPending() = + let applyPending () = let mutable msg = Unchecked.defaultof + while rd.TryRead(&msg) do let (AnkiSyncMsg(cmd, reply)) = msg let result = applyCommand cmd + match cmd, result with - | FullSync _, Ok () -> requiresSync.Value <- false - | _, Ok () -> requiresSync.Value <- true + | FullSync _, Ok() -> requiresSync.Value <- false + | _, Ok() -> requiresSync.Value <- true | _, Error _ -> () + reply.TrySetResult result |> ignore try let mutable running = true + while running do let! canRead = rd.WaitToReadAsync(stoppingToken) + if not canRead then running <- false else - applyPending() + applyPending () use window = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken) - window.CancelAfter(getDebounceTime()) + window.CancelAfter(getDebounceTime ()) + try let mutable windowOpen = true + while windowOpen do let! more = rd.WaitToReadAsync(window.Token) - if more then applyPending() else windowOpen <- false + if more then applyPending () else windowOpen <- false with :? OperationCanceledException when not stoppingToken.IsCancellationRequested -> () @@ -92,6 +106,7 @@ type AnkiSyncWorker(backend: RslibBackend, syncService: ISyncService, ch: Channe () let mutable msg = Unchecked.defaultof + while rd.TryRead(&msg) do let (AnkiSyncMsg(_, reply)) = msg reply.TrySetCanceled(stoppingToken) |> ignore diff --git a/src/Ankit.Server/Dto.fs b/src/Ankit.Server/Dto.fs index c58fbcd..09b50bf 100644 --- a/src/Ankit.Server/Dto.fs +++ b/src/Ankit.Server/Dto.fs @@ -39,11 +39,7 @@ type FullSyncDto = { } [] -type FieldError = { - field: string - issue: string - rejectedValue: obj option -} +type FieldError = { field: string; issue: string; rejectedValue: obj option } [] type ValidationError = { @@ -72,6 +68,7 @@ let fetchInvalidFieldsFromModel (model: 'T) (errors: ValidationResult seq) = None else let ty = value.GetType() + if ty.IsGenericType && ty.GetGenericTypeDefinition() = typedefof> then FSharpValue.GetUnionFields(value, ty) |> snd |> Array.tryHead else @@ -79,6 +76,7 @@ let fetchInvalidFieldsFromModel (model: 'T) (errors: ValidationResult seq) = let lookUpValue (name: string) : obj option = let m = box model + if isNull m || not (FSharpType.IsRecord(m.GetType())) then None else @@ -90,7 +88,13 @@ let fetchInvalidFieldsFromModel (model: 'T) (errors: ValidationResult seq) = |> Seq.collect (fun err -> // A rule may cover several members, each of them gets its own entry match err.MemberNames |> Seq.map _.Trim() |> List.ofSeq with - | [] -> [ { field = ""; issue = err.ErrorMessage; rejectedValue = None } ] + | [] -> [ + { + field = "" + issue = err.ErrorMessage + rejectedValue = None + } + ] | names -> names |> List.map (fun name -> { @@ -128,7 +132,8 @@ let toDeck (dto: CreateDeckDto) : Anki.Decks.Deck = Anki.Decks.Deck( Name = dto.Name.Trim(), Common = Anki.Decks.Deck.Types.Common(StudyCollapsed = true, BrowserCollapsed = true), - Normal = Anki.Decks.Deck.Types.Normal(ConfigId = 1L)) + Normal = Anki.Decks.Deck.Types.Normal(ConfigId = 1L) + ) let toAddNoteRequest (dto: AddNoteDto) : Anki.Notes.AddNoteRequest = let note = Anki.Notes.Note(NotetypeId = dto.NoteTypeId) diff --git a/src/Ankit.Server/Env.fs b/src/Ankit.Server/Env.fs index a3e8d2d..3a914ef 100644 --- a/src/Ankit.Server/Env.fs +++ b/src/Ankit.Server/Env.fs @@ -7,17 +7,13 @@ let tryGetEnv (key: string) : string option = | null -> None | value -> Some value -let tryGetEnvOrDefault (key: string) (defaultValue: string) : string = - tryGetEnv key |> Option.defaultValue defaultValue +let tryGetEnvOrDefault (key: string) (defaultValue: string) : string = tryGetEnv key |> Option.defaultValue defaultValue [] let AnkitServerSettingsSection = "Ankit" [] -type AnkitServerSettings = { - SyncDebounceTime: int64 - CollectionPath: string -} +type AnkitServerSettings = { SyncDebounceTime: int64; CollectionPath: string } [] let SyncServerSettingsSection = "SyncServer" @@ -30,11 +26,8 @@ type SyncServerSettings = { Username: string Password: string } with + member this.Endpoint = - UriBuilder( - Host = this.Host, - Port = this.Port, - Scheme = if this.UseHttps then "https" else "http" - ).ToString() + UriBuilder(Host = this.Host, Port = this.Port, Scheme = if this.UseHttps then "https" else "http").ToString() -let defaultLangs = ["en"] +let defaultLangs = [ "en" ] diff --git a/src/Ankit.Server/ErrorHandler.fs b/src/Ankit.Server/ErrorHandler.fs index 6aec603..9b4b168 100644 --- a/src/Ankit.Server/ErrorHandler.fs +++ b/src/Ankit.Server/ErrorHandler.fs @@ -12,19 +12,14 @@ open type Microsoft.AspNetCore.Http.TypedResults open type Microsoft.AspNetCore.Http.StatusCodes let notFoundHandler: EndpointHandler = - fun ctx -> - ctx.Write - <| NotFound {| - message = "Requested method doesn't exist" - |} + fun ctx -> ctx.Write <| NotFound {| message = "Requested method doesn't exist" |} let private (|RejectedJsonField|_|) (ex: exn) : string option = match ex with - | :? JsonException as ex when ex.Message.StartsWith("Missing") -> - Some (ex.Message.Split(' ') |> Array.last) + | :? JsonException as ex when ex.Message.StartsWith("Missing") -> Some(ex.Message.Split(' ') |> Array.last) | :? JsonException as ex when ex.Message.EndsWith("but was null.") -> let path = ex.Message.Split(' ') |> Array.head - Some (path.Substring(path.LastIndexOf('.') + 1)) + Some(path.Substring(path.LastIndexOf('.') + 1)) | _ -> None let errorHandler (ctx: HttpContext) (next: RequestDelegate) = diff --git a/src/Ankit.Server/Handlers.fs b/src/Ankit.Server/Handlers.fs index 8b5c8f5..61b87e9 100644 --- a/src/Ankit.Server/Handlers.fs +++ b/src/Ankit.Server/Handlers.fs @@ -10,10 +10,9 @@ open Microsoft.Extensions.Logging open Oxpecker open type Microsoft.AspNetCore.Http.StatusCodes -let ping = - (setStatusCode Status200OK) >=> (text "PONG!") +let ping = (setStatusCode Status200OK) >=> (text "PONG!") -let writeError (ctx: HttpContext) (code: int) (msg: string) (details: obj option) = +let writeError (ctx: HttpContext) (code: int) (msg: string) (details: obj option) = let logger = ctx.GetLogger() logger.LogWarning("Request failed with {StatusCode}: {Message}", code, msg) @@ -43,12 +42,15 @@ let private statusOf (error: AnkiError) = let private runCommand (ctx: HttpContext) (cmd: AnkiCommand) : Task = task { - let reply = TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously) + let reply = + TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously) + let ch = ctx.GetService>() do! ch.Writer.WriteAsync(AnkiSyncMsg(cmd, reply), ctx.RequestAborted) + try match! reply.Task.WaitAsync(ctx.RequestAborted) with - | Ok () -> ctx.SetStatusCode Status204NoContent + | Ok() -> ctx.SetStatusCode Status204NoContent | Error e -> return! writeError ctx (statusOf e) (describe e) None with :? TaskCanceledException when not ctx.RequestAborted.IsCancellationRequested -> return! writeError ctx Status503ServiceUnavailable "Server is shutting down" None @@ -76,9 +78,7 @@ let createDeck (createDeck: CreateDeckDto) : EndpointHandler = let addNote (addNote: AddNoteDto) : EndpointHandler = let note = toAddNoteRequest addNote - fun (ctx: HttpContext) -> - runCommand ctx (AddNote note) + fun (ctx: HttpContext) -> runCommand ctx (AddNote note) let fullSync (fullSync: FullSyncDto) : EndpointHandler = - fun (ctx: HttpContext) -> - runCommand ctx (FullSync fullSync.Direction) + fun (ctx: HttpContext) -> runCommand ctx (FullSync fullSync.Direction) diff --git a/src/Ankit.Server/Program.fs b/src/Ankit.Server/Program.fs index b234e25..3744fcd 100644 --- a/src/Ankit.Server/Program.fs +++ b/src/Ankit.Server/Program.fs @@ -17,9 +17,7 @@ open Microsoft.Extensions.DependencyInjection open Microsoft.Extensions.Options let endpoints = [ - GET [ - route "/ping" Handlers.ping - ] + GET [ route "/ping" Handlers.ping ] POST [ route "/decks" <| bindOrDeclineJson Handlers.createDeck route "/notes" <| bindOrDeclineJson Handlers.addNote @@ -28,14 +26,22 @@ let endpoints = [ ] let openBackend (sp: IServiceProvider) : RslibBackend = - let collectionPath = sp.GetRequiredService>().Value.CollectionPath |> Path.GetFullPath + let collectionPath = + sp.GetRequiredService>().Value.CollectionPath + |> Path.GetFullPath + Directory.CreateDirectory(Path.GetDirectoryName collectionPath) |> ignore let backend = RslibBackend.Open defaultLangs - backend.OpenCollection(OpenCollectionRequest( - CollectionPath = collectionPath, - MediaFolderPath = Path.ChangeExtension(collectionPath, ".media"), - MediaDbPath = Path.ChangeExtension(collectionPath, ".media.db2"))) + + backend.OpenCollection( + OpenCollectionRequest( + CollectionPath = collectionPath, + MediaFolderPath = Path.ChangeExtension(collectionPath, ".media"), + MediaDbPath = Path.ChangeExtension(collectionPath, ".media.db2") + ) + ) + backend let configureServices (configuration: IConfiguration) (services: IServiceCollection) = @@ -46,17 +52,15 @@ let configureServices (configuration: IConfiguration) (services: IServiceCollect .AddOxpecker() .AddSingleton(SystemTextJsonSerializer(jsonOptions)) .AddSingleton(openBackend) - .AddSingleton>(Channel.CreateUnbounded(UnboundedChannelOptions(SingleReader = true))) + .AddSingleton>( + Channel.CreateUnbounded(UnboundedChannelOptions(SingleReader = true)) + ) .AddSingleton() .AddHostedService() |> ignore let configureApp (app: IApplicationBuilder) = - app - .UseRouting() - .Use(ErrorHandler.errorHandler) - .UseOxpecker(endpoints) - .Run(ErrorHandler.notFoundHandler) + app.UseRouting().Use(ErrorHandler.errorHandler).UseOxpecker(endpoints).Run(ErrorHandler.notFoundHandler) let builder = WebApplication.CreateBuilder() configureServices builder.Configuration builder.Services diff --git a/src/Ankit.Server/Services.fs b/src/Ankit.Server/Services.fs index 14168cf..94cc50b 100644 --- a/src/Ankit.Server/Services.fs +++ b/src/Ankit.Server/Services.fs @@ -35,7 +35,8 @@ type ISyncService = abstract TrySync: unit -> SyncResult abstract FullSync: SyncDirection -> SyncResult -type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor, logger: ILogger) = +type AnkiSyncService + (backend: RslibBackend, syncServerSettings: IOptionsMonitor, logger: ILogger) = let classify (error: BackendError) : SyncFailed = match error.Kind with @@ -47,7 +48,7 @@ type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor< let call (f: unit -> 'a) : SyncResult<'a> = try - Ok <| f() + Ok <| f () with | :? BackendException as e -> Error <| classify e.Error | :? RslibException as e -> Error <| SyncFailed.NativeFailed e.Message @@ -62,11 +63,14 @@ type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor< redirected | _ -> auth - let getAuth() : SyncResult = - let loginRequest = SyncLoginRequest( - Username = syncServerSettings.CurrentValue.Username, - Password = syncServerSettings.CurrentValue.Password, - Endpoint = syncServerSettings.CurrentValue.Endpoint) + let getAuth () : SyncResult = + let loginRequest = + SyncLoginRequest( + Username = syncServerSettings.CurrentValue.Username, + Password = syncServerSettings.CurrentValue.Password, + Endpoint = syncServerSettings.CurrentValue.Endpoint + ) + call (fun () -> backend.SyncLogin loginRequest) let syncAll (auth: SyncAuth) : SyncResult = @@ -77,7 +81,13 @@ type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor< if not (System.String.IsNullOrEmpty response.ServerMessage) then logger.LogWarning("Sync server says: {ServerMessage}", response.ServerMessage) - let auth = followRedirect auth (if response.HasNewEndpoint then Some response.NewEndpoint else None) + let auth = + followRedirect + auth + (if response.HasNewEndpoint then + Some response.NewEndpoint + else + None) match response.Required with | ChangesRequired.NoChanges @@ -87,32 +97,32 @@ type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor< let fullSync (auth: SyncAuth) (direction: SyncDirection) : SyncResult = result { - let request = FullUploadOrDownloadRequest(Auth = auth, Upload = (direction = Upload)) + let request = + FullUploadOrDownloadRequest(Auth = auth, Upload = (direction = Upload)) + do! call (fun () -> backend.FullUploadOrDownload request) // rslib skips media without the server's media usn, so it's synced on its own do! call (fun () -> backend.SyncMedia auth) } - let getSyncStatus (auth: SyncAuth) : SyncResult = - call (fun () -> backend.SyncStatus auth) + let getSyncStatus (auth: SyncAuth) : SyncResult = call (fun () -> backend.SyncStatus auth) let logSyncStatus (status: SyncStatusResponse) = match status.Required with - | Required.NoChanges -> - logger.LogInformation("Sync finished, collection is up to date") - | required -> - logger.LogWarning("Sync finished, but the collection still requires {Required}", required) + | Required.NoChanges -> logger.LogInformation("Sync finished, collection is up to date") + | required -> logger.LogWarning("Sync finished, but the collection still requires {Required}", required) let logFailure (failure: SyncFailed) = match failure with | SyncFailed.Aborted -> logger.LogInformation("Sync was aborted") - | SyncFailed.NetworkError message -> logger.LogWarning("Sync failed, sync server is unreachable: {Message}", message) + | SyncFailed.NetworkError message -> + logger.LogWarning("Sync failed, sync server is unreachable: {Message}", message) | failure -> logger.LogError("Sync failed: {Failure}", failure) interface ISyncService with member this.TrySync() = result { - let! auth = getAuth() + let! auth = getAuth () let! auth = syncAll auth let! status = getSyncStatus auth logSyncStatus status @@ -121,7 +131,7 @@ type AnkiSyncService(backend: RslibBackend, syncServerSettings: IOptionsMonitor< member this.FullSync(direction) = result { - let! auth = getAuth() + let! auth = getAuth () logger.LogWarning("Starting full {Direction}, the other side's collection is replaced", direction) do! fullSync auth direction let! status = getSyncStatus auth diff --git a/tests/Ankit.Server.Tests/AnkiSyncServer.fs b/tests/Ankit.Server.Tests/AnkiSyncServer.fs index 925feb8..51c18d1 100644 --- a/tests/Ankit.Server.Tests/AnkiSyncServer.fs +++ b/tests/Ankit.Server.Tests/AnkiSyncServer.fs @@ -12,9 +12,13 @@ open Xunit let private repoRoot = let rec find (dir: DirectoryInfo) = - if isNull dir then failwith "paket.dependencies not found above the test output directory" - elif File.Exists(Path.Combine(dir.FullName, "paket.dependencies")) then dir.FullName - else find dir.Parent + if isNull dir then + failwith "paket.dependencies not found above the test output directory" + elif File.Exists(Path.Combine(dir.FullName, "paket.dependencies")) then + dir.FullName + else + find dir.Parent + find (DirectoryInfo AppContext.BaseDirectory) /// The sync server is built from the same Anki tag as the rslib we link against, so the protocol versions match @@ -25,7 +29,16 @@ let private ankiVersion = let private linuxDockerAvailable = lazy try - use docker = Process.Start(ProcessStartInfo("docker", "info --format {{.OSType}}", RedirectStandardOutput = true, RedirectStandardError = true)) + use docker = + Process.Start( + ProcessStartInfo( + "docker", + "info --format {{.OSType}}", + RedirectStandardOutput = true, + RedirectStandardError = true + ) + ) + let osType = docker.StandardOutput.ReadToEnd().Trim() docker.WaitForExit(10_000) && docker.ExitCode = 0 && osType = "linux" with _ -> @@ -33,12 +46,16 @@ let private linuxDockerAvailable = type LinuxDockerFactAttribute() as this = inherit FactAttribute() + do if not linuxDockerAvailable.Value then this.Skip <- "Needs Docker with Linux containers" /// Parallel rustc jobs for compiling the sync server, more jobs need more memory in the Docker VM -let private cargoBuildJobs = Environment.GetEnvironmentVariable "ANKIT_TESTS_CARGO_JOBS" |> Option.ofObj |> Option.defaultValue "4" +let private cargoBuildJobs = + Environment.GetEnvironmentVariable "ANKIT_TESTS_CARGO_JOBS" + |> Option.ofObj + |> Option.defaultValue "4" [] let private SyncPort = 8080 @@ -67,28 +84,41 @@ type AnkiSyncServer() = // Compiling the server takes several minutes the first time, Docker caches the layers after that let image = ImageFromDockerfileBuilder() - .WithDockerfileDirectory(Path.Combine(repoRoot, "tests", "Ankit.Server.Tests", "SyncServer")) + .WithDockerfileDirectory( + Path.Combine(repoRoot, "tests", "Ankit.Server.Tests", "SyncServer") + ) .WithDockerfile("Dockerfile") .WithBuildArgument("ANKI_VERSION", ankiVersion) .WithBuildArgument("CARGO_BUILD_JOBS", cargoBuildJobs) .WithName($"ankit-tests/anki-sync-server:{ankiVersion}") .WithCleanUp(false) .Build() + do! image.CreateAsync() let builder = Users |> List.indexed |> List.fold - (fun (b: ContainerBuilder) (i, user) -> b.WithEnvironment($"SYNC_USER{i + 1}", $"{user}:{Password}")) + (fun (b: ContainerBuilder) (i, user) -> + b.WithEnvironment($"SYNC_USER{i + 1}", $"{user}:{Password}")) (ContainerBuilder(image.FullName)) + container <- builder .WithPortBinding(SyncPort, true) - .WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded(fun r -> r.ForPort(uint16 SyncPort).ForPath("/health"))) + .WithWaitStrategy( + Wait + .ForUnixContainer() + .UntilHttpRequestIsSucceeded(fun r -> r.ForPort(uint16 SyncPort).ForPath("/health")) + ) .Build() + do! container.StartAsync() } member _.DisposeAsync() = - if isNull container then Task.CompletedTask else container.DisposeAsync().AsTask() + if isNull container then + Task.CompletedTask + else + container.DisposeAsync().AsTask() diff --git a/tests/Ankit.Server.Tests/SyncServiceTests.fs b/tests/Ankit.Server.Tests/SyncServiceTests.fs index ac9c31a..dac4df1 100644 --- a/tests/Ankit.Server.Tests/SyncServiceTests.fs +++ b/tests/Ankit.Server.Tests/SyncServiceTests.fs @@ -19,11 +19,13 @@ type SyncServiceTests(server: AnkiSyncServer) = Assert.Equal(sprintf "%A" expected, sprintf "%A" actual) let syncService (backend: RslibBackend) (settings: SyncServerSettings) : ISyncService = - let monitor = - { new IOptionsMonitor with + let monitor = { + new IOptionsMonitor with member _.CurrentValue = settings member _.Get _ = settings - member _.OnChange _ = null } + member _.OnChange _ = null + } + AnkiSyncService(backend, monitor, NullLogger.Instance) let addBasicNote (backend: RslibBackend) (front: string) (back: string) = @@ -69,7 +71,11 @@ type SyncServiceTests(server: AnkiSyncServer) = use other = new TempCollection() let otherService = syncService other.Backend settings - assertResult (Error(SyncFailed.FullSyncRequired SyncCollectionResponse.Types.ChangesRequired.FullDownload)) (otherService.TrySync()) + + assertResult + (Error(SyncFailed.FullSyncRequired SyncCollectionResponse.Types.ChangesRequired.FullDownload)) + (otherService.TrySync()) + assertResult (Ok()) (otherService.FullSync Download) assertResult (Ok()) (otherService.TrySync()) Assert.Equal(2, other.Backend.SearchNotes(SearchRequest(Search = "")).Ids.Count) From c94674a76e3a1158aa9b8beea93726b3624e73d4 Mon Sep 17 00:00:00 2001 From: evgTSV Date: Sat, 24 Oct 2026 21:55:36 +0500 Subject: [PATCH 6/6] Restore tools before verifying the format Verify-Format runs Fantomas, a local tool, but the verify job never ran "dotnet tool restore", so the check failed before Fantomas started. Co-Authored-By: Claude Opus 5.5 --- Scripts/build.fsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Scripts/build.fsx b/Scripts/build.fsx index 088de88..156ff64 100644 --- a/Scripts/build.fsx +++ b/Scripts/build.fsx @@ -459,6 +459,9 @@ Target.create "Before-Push" doNothing "Restore" ==> "Format" +"Restore" + ==> "Verify-Format" + "Remove-BOM" =?> ("Prepare-Sources", verifyEncoding() |> Result.isError)