diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6875c1c..50a3485 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,10 +24,8 @@ jobs: uses: actions/setup-dotnet@v6 with: dotnet-version: 10.0.x - - name: Verify encoding - run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Encoding - - name: Verify workflow - run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Workflow + - name: Verify sources + run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Sources build: name: Build and test (${{ matrix.image }}) strategy: diff --git a/Ankit.sln b/Ankit.sln index 718ffc3..eb5f074 100644 --- a/Ankit.sln +++ b/Ankit.sln @@ -8,7 +8,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution global.json = global.json paket.dependencies = paket.dependencies README.md = README.md - Scriptsuild.fsx = Scriptsuild.fsx + build.fsx = Scripts/build.fsx + gen-workflow.fsx = Scripts/gen-workflow.fsx EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ankit.Native", "src\Ankit.Native\Ankit.Native.csproj", "{452916EC-79B6-4C8B-8528-369D929FD39B}" diff --git a/Scripts/build.fsx b/Scripts/build.fsx index 37db86e..156ff64 100644 --- a/Scripts/build.fsx +++ b/Scripts/build.fsx @@ -177,6 +177,9 @@ Target.create "Restore" (fun _ -> Target.create "Format" (fun _ -> dotnet "fantomas" "." |> handleResult "Format") +Target.create "Verify-Format" (fun _ -> + dotnet "fantomas" "--check ." |> handleResult "Verify-Format") + let utf8Bom = Encoding.UTF8.GetPreamble() let hasBom (bytes: ReadOnlySpan) = @@ -410,6 +413,7 @@ Target.create "Prepare" doNothing Target.create "Build" doNothing Target.create "All" doNothing Target.create "Prepare-Sources" doNothing +Target.create "Verify-Sources" doNothing Target.create "Before-Push" doNothing // --- Prepare: restore packages and tools, generate sources --- @@ -455,17 +459,25 @@ Target.create "Before-Push" doNothing "Restore" ==> "Format" +"Restore" + ==> "Verify-Format" + "Remove-BOM" =?> ("Prepare-Sources", verifyEncoding() |> Result.isError) "Format" ==> "Prepare-Sources" -"Before-Push" <== [ - "Prepare-Sources" +"Verify-Sources" <== [ "Verify-Encoding" "Verify-Workflow" + "Verify-Format" +] + +"Before-Push" <== [ "Test" + "Prepare-Sources" + "Verify-Sources" ] if DocoptResult.hasFlag "--list" cli then diff --git a/Scripts/gen-workflow.fsx b/Scripts/gen-workflow.fsx index b020af9..e387d01 100644 --- a/Scripts/gen-workflow.fsx +++ b/Scripts/gen-workflow.fsx @@ -86,8 +86,7 @@ let workflow = [ yield! env checkoutStep setUpDotNetSdkStep - buildCliStep "Verify encoding" "Verify-Encoding" - buildCliStep "Verify workflow" "Verify-Workflow" + buildCliStep "Verify sources" "Verify-Sources" ] job "build" [ diff --git a/paket.dependencies b/paket.dependencies index 540bf45..2145bf5 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -5,6 +5,8 @@ group WebServer framework: net10.0 nuget Oxpecker 2.1.1 + nuget FsToolkit.ErrorHandling 5.2.0 + nuget FSharp.SystemTextJson 1.4.36 group Native source https://api.nuget.org/v3/index.json @@ -25,3 +27,4 @@ group Test nuget xunit nuget xunit.runner.visualstudio nuget coverlet.collector + nuget Testcontainers diff --git a/paket.lock b/paket.lock index 93e1a1b..d07fabe 100644 --- a/paket.lock +++ b/paket.lock @@ -16,15 +16,48 @@ GROUP Test RESTRICTION: == net10.0 NUGET remote: https://api.nuget.org/v3/index.json + BouncyCastle.Cryptography (2.7) coverlet.collector (10.1) + Docker.DotNet.Enhanced (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.LegacyHttp (>= 4.3.3) + Docker.DotNet.Enhanced.NativeHttp (>= 4.3.3) + Docker.DotNet.Enhanced.NPipe (>= 4.3.3) + Docker.DotNet.Enhanced.Unix (>= 4.3.3) + Microsoft.Extensions.Logging.Abstractions (>= 8.0.3) + Docker.DotNet.Enhanced.Handler.Abstractions (4.3.3) + Microsoft.Extensions.Logging.Abstractions (>= 8.0.3) + Docker.DotNet.Enhanced.LegacyHttp (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.NativeHttp (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.NPipe (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.Unix (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) + Docker.DotNet.Enhanced.X509 (4.3.3) + Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3) FSharp.Core (10.1.402) Microsoft.CodeCoverage (18.11) + Microsoft.Extensions.DependencyInjection.Abstractions (10.0.13) + Microsoft.Extensions.Logging.Abstractions (10.0.13) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 10.0.13) Microsoft.NET.Test.Sdk (18.11) Microsoft.CodeCoverage (>= 18.11) Microsoft.TestPlatform.TestHost (>= 18.11) Microsoft.TestPlatform.ObjectModel (18.11) Microsoft.TestPlatform.TestHost (18.11) Microsoft.TestPlatform.ObjectModel (>= 18.11) + SharpZipLib (1.4.2) + SSH.NET (2026.0) + BouncyCastle.Cryptography (>= 2.7) + Microsoft.Extensions.Logging.Abstractions (>= 8.0.3) + Testcontainers (4.16) + Docker.DotNet.Enhanced (>= 4.3.3) + Docker.DotNet.Enhanced.X509 (>= 4.3.3) + Microsoft.Extensions.Logging.Abstractions (>= 8.0.3) + SharpZipLib (>= 1.4.2) + SSH.NET (>= 2026.0) xunit (2.9.3) xunit.analyzers (>= 1.18) xunit.assert (>= 2.9.3) @@ -46,6 +79,11 @@ RESTRICTION: == net10.0 NUGET remote: https://api.nuget.org/v3/index.json FSharp.Core (10.1.402) + FSharp.SystemTextJson (1.4.36) + FSharp.Core (>= 4.7) + System.Text.Json (>= 6.0.10) + FsToolkit.ErrorHandling (5.2) + FSharp.Core (>= 9.0.300) Microsoft.Extensions.ObjectPool (10.0.13) Microsoft.IO.RecyclableMemoryStream (3.0.1) Oxpecker (2.1.1) @@ -55,5 +93,6 @@ NUGET Oxpecker.ViewEngine (2.1) FSharp.Core (>= 10.0.100) Microsoft.Extensions.ObjectPool (>= 10.0) + System.Text.Json (10.0.13) TypeShape (10.0) FSharp.Core (>= 4.5.4) diff --git a/src/Ankit.Server/AnkiSyncWorker.fs b/src/Ankit.Server/AnkiSyncWorker.fs new file mode 100644 index 0000000..d46596e --- /dev/null +++ b/src/Ankit.Server/AnkiSyncWorker.fs @@ -0,0 +1,115 @@ +namespace Ankit.Server + +open System +open System.Threading +open System.Threading.Tasks +open Ankit.Native +open Ankit.Server.Env +open Ankit.Server.Services +open Microsoft.Extensions.Hosting +open System.Threading.Channels +open Microsoft.Extensions.Logging +open Microsoft.Extensions.Options + +[] +type AnkiError = + | CreateDeckError + | AddNoteError + | FullSyncError of SyncFailed + +type AnkiCommand = + | CreateDeck of Anki.Decks.Deck + | AddNote of Anki.Notes.AddNoteRequest + | FullSync of SyncDirection + +type AnkiSyncMsg = AnkiSyncMsg of cmd: AnkiCommand * reply: TaskCompletionSource> + +type AnkiSyncWorker + ( + backend: RslibBackend, + syncService: ISyncService, + ch: Channel, + logger: ILogger, + settings: IOptionsMonitor + ) = + inherit BackgroundService() + + let getDebounceTime () = + TimeSpan.FromMilliseconds(float settings.CurrentValue.SyncDebounceTime) + + let applyCommand (cmd: AnkiCommand) : Result = + match cmd with + | FullSync direction -> syncService.FullSync direction |> Result.mapError AnkiError.FullSyncError + | CreateDeck _ + | AddNote _ -> + try + match cmd with + | CreateDeck deck -> backend.AddDeck deck |> ignore + | AddNote noteRequest -> backend.AddNote noteRequest |> ignore + | FullSync _ -> () + + Ok() + with :? RslibException as e -> + logger.LogError(e, "Failed to apply {Command}", cmd) + + match cmd with + | CreateDeck _ -> Error AnkiError.CreateDeckError + | _ -> Error AnkiError.AddNoteError + + override this.ExecuteAsync(stoppingToken) = + task { + logger.LogInformation("AnkiSyncWorker is starting.") + + let rd = ch.Reader + let requiresSync = ref false + + let applyPending () = + let mutable msg = Unchecked.defaultof + + while rd.TryRead(&msg) do + let (AnkiSyncMsg(cmd, reply)) = msg + let result = applyCommand cmd + + match cmd, result with + | FullSync _, Ok() -> requiresSync.Value <- false + | _, Ok() -> requiresSync.Value <- true + | _, Error _ -> () + + reply.TrySetResult result |> ignore + + try + let mutable running = true + + while running do + let! canRead = rd.WaitToReadAsync(stoppingToken) + + if not canRead then + running <- false + else + applyPending () + use window = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken) + window.CancelAfter(getDebounceTime ()) + + try + let mutable windowOpen = true + + while windowOpen do + let! more = rd.WaitToReadAsync(window.Token) + if more then applyPending () else windowOpen <- false + with :? OperationCanceledException when not stoppingToken.IsCancellationRequested -> + () + + if requiresSync.Value then + requiresSync.Value <- false + syncService.TrySync() |> ignore + with :? OperationCanceledException when stoppingToken.IsCancellationRequested -> + () + + let mutable msg = Unchecked.defaultof + + while rd.TryRead(&msg) do + let (AnkiSyncMsg(_, reply)) = msg + reply.TrySetCanceled(stoppingToken) |> ignore + + logger.LogInformation("AnkiSyncWorker is stopping due to cancellation.") + } diff --git a/src/Ankit.Server/Ankit.Server.fsproj b/src/Ankit.Server/Ankit.Server.fsproj index c5cbd04..2d6ea22 100644 --- a/src/Ankit.Server/Ankit.Server.fsproj +++ b/src/Ankit.Server/Ankit.Server.fsproj @@ -7,6 +7,13 @@ + + + + + + + @@ -14,9 +21,18 @@ .dockerignore + + + + + + + + + diff --git a/src/Ankit.Server/Dto.fs b/src/Ankit.Server/Dto.fs new file mode 100644 index 0000000..09b50bf --- /dev/null +++ b/src/Ankit.Server/Dto.fs @@ -0,0 +1,142 @@ +module Ankit.Server.Dto + +open System +open System.ComponentModel.DataAnnotations +open System.Threading.Tasks +open Ankit.Server.Services +open Ankit.Server.Utils +open Microsoft.AspNetCore.Http +open Microsoft.FSharp.Reflection +open Oxpecker + +open type Microsoft.AspNetCore.Http.StatusCodes + +[] +type CreateDeckDto = { + [] + Name: string +} + +[] +type AddNoteDto = { + [] + DeckId: int64 + + [] + NoteTypeId: int64 + + [] + Fields: string array + + Tags: string array option +} + +[] +type FullSyncDto = { + /// "upload" replaces the server's collection with ours, "download" replaces ours with the server's + [] + Direction: SyncDirection +} + +[] +type FieldError = { field: string; issue: string; rejectedValue: obj option } + +[] +type ValidationError = { + code: int + message: string + traceId: string + timestamp: string + path: string + fieldErrors: FieldError[] +} + +[] +type ApiError = { + code: int + message: string + traceId: string + timestamp: string + path: string + details: obj option +} + +let fetchInvalidFieldsFromModel (model: 'T) (errors: ValidationResult seq) = + // Option fields report the wrapped value, None and null report nothing + let unwrap (value: obj) : obj option = + if isNull value then + None + else + let ty = value.GetType() + + if ty.IsGenericType && ty.GetGenericTypeDefinition() = typedefof> then + FSharpValue.GetUnionFields(value, ty) |> snd |> Array.tryHead + else + Some value + + let lookUpValue (name: string) : obj option = + let m = box model + + if isNull m || not (FSharpType.IsRecord(m.GetType())) then + None + else + FSharpType.GetRecordFields(m.GetType()) + |> Array.tryFind (fun p -> p.Name = name) + |> Option.bind (fun p -> FSharpValue.GetRecordField(m, p) |> unwrap) + + errors + |> Seq.collect (fun err -> + // A rule may cover several members, each of them gets its own entry + match err.MemberNames |> Seq.map _.Trim() |> List.ofSeq with + | [] -> [ + { + field = "" + issue = err.ErrorMessage + rejectedValue = None + } + ] + | names -> + names + |> List.map (fun name -> { + // Same naming as the JSON body, so it matches the fields the client sent + field = jsonOptions.PropertyNamingPolicy.ConvertName name + issue = err.ErrorMessage + rejectedValue = lookUpValue name + })) + |> Seq.toArray + +let buildValidationError (invalidFields: FieldError[]) (ctx: HttpContext) = { + ValidationError.code = Status422UnprocessableEntity + message = "Validation failed" + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + fieldErrors = invalidFields +} + +let validateModel (f: 'T -> EndpointHandler) : ModelValidationResult<'T> -> HttpContext -> Task = + (fun res ctx -> + match res with + | ModelValidationResult.Valid dto -> f dto ctx + | ModelValidationResult.Invalid(model, errors) -> + task { + let invalidFields = fetchInvalidFieldsFromModel model errors.All + let valError = buildValidationError invalidFields ctx + + ctx.SetStatusCode(Status422UnprocessableEntity) + return! json valError ctx + }) + +let toDeck (dto: CreateDeckDto) : Anki.Decks.Deck = + // Mirrors Deck::new_normal in rslib + Anki.Decks.Deck( + Name = dto.Name.Trim(), + Common = Anki.Decks.Deck.Types.Common(StudyCollapsed = true, BrowserCollapsed = true), + Normal = Anki.Decks.Deck.Types.Normal(ConfigId = 1L) + ) + +let toAddNoteRequest (dto: AddNoteDto) : Anki.Notes.AddNoteRequest = + let note = Anki.Notes.Note(NotetypeId = dto.NoteTypeId) + note.Fields.AddRange dto.Fields + dto.Tags |> Option.iter note.Tags.AddRange + Anki.Notes.AddNoteRequest(Note = note, DeckId = dto.DeckId) diff --git a/src/Ankit.Server/Env.fs b/src/Ankit.Server/Env.fs new file mode 100644 index 0000000..3a914ef --- /dev/null +++ b/src/Ankit.Server/Env.fs @@ -0,0 +1,33 @@ +module Ankit.Server.Env + +open System + +let tryGetEnv (key: string) : string option = + match Environment.GetEnvironmentVariable(key) with + | null -> None + | value -> Some value + +let tryGetEnvOrDefault (key: string) (defaultValue: string) : string = tryGetEnv key |> Option.defaultValue defaultValue + +[] +let AnkitServerSettingsSection = "Ankit" + +[] +type AnkitServerSettings = { SyncDebounceTime: int64; CollectionPath: string } + +[] +let SyncServerSettingsSection = "SyncServer" + +[] +type SyncServerSettings = { + Host: string + Port: int + UseHttps: bool + Username: string + Password: string +} with + + member this.Endpoint = + UriBuilder(Host = this.Host, Port = this.Port, Scheme = if this.UseHttps then "https" else "http").ToString() + +let defaultLangs = [ "en" ] diff --git a/src/Ankit.Server/ErrorHandler.fs b/src/Ankit.Server/ErrorHandler.fs new file mode 100644 index 0000000..9b4b168 --- /dev/null +++ b/src/Ankit.Server/ErrorHandler.fs @@ -0,0 +1,108 @@ +module Ankit.Server.ErrorHandler + +open System +open System.Text.Json +open System.Threading.Tasks +open Ankit.Server.Dto +open Microsoft.AspNetCore.Http +open Microsoft.Extensions.Logging +open Oxpecker + +open type Microsoft.AspNetCore.Http.TypedResults +open type Microsoft.AspNetCore.Http.StatusCodes + +let notFoundHandler: EndpointHandler = + fun ctx -> ctx.Write <| NotFound {| message = "Requested method doesn't exist" |} + +let private (|RejectedJsonField|_|) (ex: exn) : string option = + match ex with + | :? JsonException as ex when ex.Message.StartsWith("Missing") -> Some(ex.Message.Split(' ') |> Array.last) + | :? JsonException as ex when ex.Message.EndsWith("but was null.") -> + let path = ex.Message.Split(' ') |> Array.head + Some(path.Substring(path.LastIndexOf('.') + 1)) + | _ -> None + +let errorHandler (ctx: HttpContext) (next: RequestDelegate) = + task { + let logger = ctx.GetLogger("Ankit.Server.ErrorHandler") + + try + return! next.Invoke(ctx) + with + | :? OperationCanceledException when ctx.RequestAborted.IsCancellationRequested -> + logger.LogDebug("Request was aborted by the client") + | :? ModelBindException as ex -> + logger.LogWarning(ex, "Reading request body for validation errors") + + match ex.InnerException with + | RejectedJsonField field -> + let fieldErr = { + field = field + issue = ex.InnerException.Message + rejectedValue = None + } + + let code = Status422UnprocessableEntity + + let valError = { + ValidationError.code = code + message = "Validation failed" + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + fieldErrors = [| fieldErr |] + } + + ctx.SetStatusCode(code) + return! json valError ctx + | ex -> + logger.LogWarning(ex, "Model bind error") + + let code = Status400BadRequest + + let error = { + code = code + message = "Model binding failed: " + ex.Message + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + details = Some {| errors = ex.Message |} + } + + ctx.SetStatusCode(code) + return! json error ctx + + | :? RouteParseException as ex -> + logger.LogWarning(ex, "Route parse error") + + let code = Status400BadRequest + + let error = { + code = code + message = "Invalid route:" + ex.Message + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + details = None + } + + ctx.SetStatusCode(code) + return! json error ctx + | ex -> + logger.LogError(ex, "Unhandled error") + + let code = Status500InternalServerError + + let error = { + code = code + message = "Internal server error: an unexpected error occurred" + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + details = None + } + + ctx.SetStatusCode(code) + return! json error ctx + } + :> Task diff --git a/src/Ankit.Server/Handlers.fs b/src/Ankit.Server/Handlers.fs new file mode 100644 index 0000000..61b87e9 --- /dev/null +++ b/src/Ankit.Server/Handlers.fs @@ -0,0 +1,84 @@ +module Ankit.Server.Handlers + +open System +open System.Threading.Channels +open System.Threading.Tasks +open Ankit.Server +open Ankit.Server.Dto +open Microsoft.AspNetCore.Http +open Microsoft.Extensions.Logging +open Oxpecker +open type Microsoft.AspNetCore.Http.StatusCodes + +let ping = (setStatusCode Status200OK) >=> (text "PONG!") + +let writeError (ctx: HttpContext) (code: int) (msg: string) (details: obj option) = + let logger = ctx.GetLogger() + logger.LogWarning("Request failed with {StatusCode}: {Message}", code, msg) + + let error = { + code = code + message = msg + traceId = ctx.TraceIdentifier + timestamp = DateTime.UtcNow.ToString("O") + path = ctx.Request.Path.Value + details = details + } + + ctx.SetStatusCode(code) + + ctx.WriteJson <| error + +let private describe (error: AnkiError) = + match error with + | AnkiError.CreateDeckError -> "Failed to create the deck" + | AnkiError.AddNoteError -> "Failed to add the note, check the deck, the notetype and the number of fields" + | AnkiError.FullSyncError failure -> $"Full sync failed: %A{failure}" + +let private statusOf (error: AnkiError) = + match error with + | AnkiError.FullSyncError _ -> Status502BadGateway + | _ -> Status422UnprocessableEntity + +let private runCommand (ctx: HttpContext) (cmd: AnkiCommand) : Task = + task { + let reply = + TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously) + + let ch = ctx.GetService>() + do! ch.Writer.WriteAsync(AnkiSyncMsg(cmd, reply), ctx.RequestAborted) + + try + match! reply.Task.WaitAsync(ctx.RequestAborted) with + | Ok() -> ctx.SetStatusCode Status204NoContent + | Error e -> return! writeError ctx (statusOf e) (describe e) None + with :? TaskCanceledException when not ctx.RequestAborted.IsCancellationRequested -> + return! writeError ctx Status503ServiceUnavailable "Server is shutting down" None + } + +module ModelBinder = + let bindOrDeclineJson<'T when 'T: not null> (f: 'T -> EndpointHandler) : EndpointHandler = + bindAndValidateJson<'T> <| Dto.validateModel f + + let bindOrDeclineJson1 (f: 'T0 -> 'T1 -> EndpointHandler) : 'T1 -> EndpointHandler = + fun x -> bindOrDeclineJson (f >> (fun g -> g x)) + + let bindOrDeclineQuery<'T when 'T: not null> (f: 'T -> EndpointHandler) : EndpointHandler = + bindAndValidateQuery<'T> <| Dto.validateModel f + + let bindOrDeclineQuery1 (f: 'T0 -> 'T1 -> EndpointHandler) : 'T1 -> EndpointHandler = + fun x -> bindOrDeclineQuery (f >> (fun g -> g x)) + +let createDeck (createDeck: CreateDeckDto) : EndpointHandler = + fun (ctx: HttpContext) -> + task { + let deck = toDeck createDeck + return! runCommand ctx (CreateDeck deck) + } + +let addNote (addNote: AddNoteDto) : EndpointHandler = + let note = toAddNoteRequest addNote + fun (ctx: HttpContext) -> runCommand ctx (AddNote note) + +let fullSync (fullSync: FullSyncDto) : EndpointHandler = + fun (ctx: HttpContext) -> runCommand ctx (FullSync fullSync.Direction) diff --git a/src/Ankit.Server/Program.fs b/src/Ankit.Server/Program.fs index e0827a9..3744fcd 100644 --- a/src/Ankit.Server/Program.fs +++ b/src/Ankit.Server/Program.fs @@ -1,2 +1,71 @@ -// For more information see https://aka.ms/fsharp-console-apps -printfn "Hello from F#" +module Ankit.Server.Main + +open System +open System.IO +open System.Threading.Channels +open Anki.Collection +open Ankit.Server +open Ankit.Server.Handlers.ModelBinder +open Ankit.Server.Env +open Ankit.Server.Services +open Ankit.Native +open Ankit.Server.Utils +open Oxpecker +open Microsoft.AspNetCore.Builder +open Microsoft.Extensions.Configuration +open Microsoft.Extensions.DependencyInjection +open Microsoft.Extensions.Options + +let endpoints = [ + GET [ route "/ping" Handlers.ping ] + POST [ + route "/decks" <| bindOrDeclineJson Handlers.createDeck + route "/notes" <| bindOrDeclineJson Handlers.addNote + route "/sync/full" <| bindOrDeclineJson Handlers.fullSync + ] +] + +let openBackend (sp: IServiceProvider) : RslibBackend = + let collectionPath = + sp.GetRequiredService>().Value.CollectionPath + |> Path.GetFullPath + + Directory.CreateDirectory(Path.GetDirectoryName collectionPath) |> ignore + + let backend = RslibBackend.Open defaultLangs + + backend.OpenCollection( + OpenCollectionRequest( + CollectionPath = collectionPath, + MediaFolderPath = Path.ChangeExtension(collectionPath, ".media"), + MediaDbPath = Path.ChangeExtension(collectionPath, ".media.db2") + ) + ) + + backend + +let configureServices (configuration: IConfiguration) (services: IServiceCollection) = + services + .Configure(configuration.GetSection AnkitServerSettingsSection) + .Configure(configuration.GetSection SyncServerSettingsSection) + .AddRouting() + .AddOxpecker() + .AddSingleton(SystemTextJsonSerializer(jsonOptions)) + .AddSingleton(openBackend) + .AddSingleton>( + Channel.CreateUnbounded(UnboundedChannelOptions(SingleReader = true)) + ) + .AddSingleton() + .AddHostedService() + |> ignore + +let configureApp (app: IApplicationBuilder) = + app.UseRouting().Use(ErrorHandler.errorHandler).UseOxpecker(endpoints).Run(ErrorHandler.notFoundHandler) + +let builder = WebApplication.CreateBuilder() +configureServices builder.Configuration builder.Services + +let app = builder.Build() +configureApp app + +app.Run() diff --git a/src/Ankit.Server/Services.fs b/src/Ankit.Server/Services.fs new file mode 100644 index 0000000..94cc50b --- /dev/null +++ b/src/Ankit.Server/Services.fs @@ -0,0 +1,140 @@ +module Ankit.Server.Services + +open Anki.Backend +open Anki.Sync +open Ankit.Native +open Ankit.Server.Env +open Microsoft.Extensions.Logging +open Microsoft.Extensions.Options +open FsToolkit.ErrorHandling + +open type SyncCollectionResponse.Types +open type SyncStatusResponse.Types + +[] +type SyncFailed = + | InvalidCredentials + | Aborted + | NetworkError of message: string + /// The server refused the sync and explained why, e.g. the account is locked + | ServerRejected of message: string + /// The collections diverged and one side has to overwrite the other, which needs a human decision + | FullSyncRequired of SyncCollectionResponse.Types.ChangesRequired + | BackendFailed of kind: BackendError.Types.Kind * message: string + | NativeFailed of message: string + +type SyncResult<'a> = Result<'a, SyncFailed> +type BackendErrorKind = BackendError.Types.Kind + +type SyncDirection = + | Upload + | Download + +[] +type ISyncService = + abstract TrySync: unit -> SyncResult + abstract FullSync: SyncDirection -> SyncResult + +type AnkiSyncService + (backend: RslibBackend, syncServerSettings: IOptionsMonitor, logger: ILogger) = + + let classify (error: BackendError) : SyncFailed = + match error.Kind with + | BackendErrorKind.SyncAuthError -> SyncFailed.InvalidCredentials + | BackendErrorKind.Interrupted -> SyncFailed.Aborted + | BackendErrorKind.NetworkError -> SyncFailed.NetworkError error.Message + | BackendErrorKind.SyncServerMessage -> SyncFailed.ServerRejected error.Message + | kind -> SyncFailed.BackendFailed(kind, error.Message) + + let call (f: unit -> 'a) : SyncResult<'a> = + try + Ok <| f () + with + | :? BackendException as e -> Error <| classify e.Error + | :? RslibException as e -> Error <| SyncFailed.NativeFailed e.Message + + /// The server may redirect us to another shard, subsequent requests have to go there + let followRedirect (auth: SyncAuth) (newEndpoint: string option) : SyncAuth = + match newEndpoint with + | Some endpoint when endpoint <> auth.Endpoint -> + logger.LogInformation("Sync server moved the collection to {Endpoint}", endpoint) + let redirected = auth.Clone() + redirected.Endpoint <- endpoint + redirected + | _ -> auth + + let getAuth () : SyncResult = + let loginRequest = + SyncLoginRequest( + Username = syncServerSettings.CurrentValue.Username, + Password = syncServerSettings.CurrentValue.Password, + Endpoint = syncServerSettings.CurrentValue.Endpoint + ) + + call (fun () -> backend.SyncLogin loginRequest) + + let syncAll (auth: SyncAuth) : SyncResult = + result { + let request = SyncCollectionRequest(Auth = auth, SyncMedia = true) + let! response = call (fun () -> backend.SyncCollection request) + + if not (System.String.IsNullOrEmpty response.ServerMessage) then + logger.LogWarning("Sync server says: {ServerMessage}", response.ServerMessage) + + let auth = + followRedirect + auth + (if response.HasNewEndpoint then + Some response.NewEndpoint + else + None) + + match response.Required with + | ChangesRequired.NoChanges + | ChangesRequired.NormalSync -> return auth + | required -> return! Error <| SyncFailed.FullSyncRequired required + } + + let fullSync (auth: SyncAuth) (direction: SyncDirection) : SyncResult = + result { + let request = + FullUploadOrDownloadRequest(Auth = auth, Upload = (direction = Upload)) + + do! call (fun () -> backend.FullUploadOrDownload request) + // rslib skips media without the server's media usn, so it's synced on its own + do! call (fun () -> backend.SyncMedia auth) + } + + let getSyncStatus (auth: SyncAuth) : SyncResult = call (fun () -> backend.SyncStatus auth) + + let logSyncStatus (status: SyncStatusResponse) = + match status.Required with + | Required.NoChanges -> logger.LogInformation("Sync finished, collection is up to date") + | required -> logger.LogWarning("Sync finished, but the collection still requires {Required}", required) + + let logFailure (failure: SyncFailed) = + match failure with + | SyncFailed.Aborted -> logger.LogInformation("Sync was aborted") + | SyncFailed.NetworkError message -> + logger.LogWarning("Sync failed, sync server is unreachable: {Message}", message) + | failure -> logger.LogError("Sync failed: {Failure}", failure) + + interface ISyncService with + member this.TrySync() = + result { + let! auth = getAuth () + let! auth = syncAll auth + let! status = getSyncStatus auth + logSyncStatus status + } + |> Result.teeError logFailure + + member this.FullSync(direction) = + result { + let! auth = getAuth () + logger.LogWarning("Starting full {Direction}, the other side's collection is replaced", direction) + do! fullSync auth direction + let! status = getSyncStatus auth + logSyncStatus status + } + |> Result.teeError logFailure diff --git a/src/Ankit.Server/Utils.fs b/src/Ankit.Server/Utils.fs new file mode 100644 index 0000000..bdf1435 --- /dev/null +++ b/src/Ankit.Server/Utils.fs @@ -0,0 +1,35 @@ +module Ankit.Server.Utils + +open System.ComponentModel.DataAnnotations +open System.Text.Json +open System.Text.Json.Serialization + +let jsonOptions = + let opts = JsonSerializerOptions() + opts.PropertyNamingPolicy <- JsonNamingPolicy.CamelCase + opts.DefaultIgnoreCondition <- JsonIgnoreCondition.Never + opts.AllowDuplicateProperties <- false + opts.AllowTrailingCommas <- false + opts.PropertyNameCaseInsensitive <- true + opts.WriteIndented <- true + + JsonFSharpOptions + .Default() + .WithUnionUnwrapFieldlessTags() + .WithUnionTagNamingPolicy(JsonNamingPolicy.SnakeCaseUpper) + .WithUnionNamedFields() + .WithUnionTagName("code") + .WithUnionInternalTag() + .WithSkippableOptionFields() + .AddToJsonSerializerOptions(opts) + + opts + +type NumericIdAttribute() = + inherit ValidationAttribute("Must be a positive id") + + override this.IsValid(value) = + match value with + | :? int64 as i -> i >= 1L + | :? int32 as i -> i >= 1 + | _ -> true diff --git a/src/Ankit.Server/appsettings.json b/src/Ankit.Server/appsettings.json new file mode 100644 index 0000000..e244c13 --- /dev/null +++ b/src/Ankit.Server/appsettings.json @@ -0,0 +1,20 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "Urls": "http://0.0.0.0:5000", + "Ankit": { + "SyncDebounceTime": 5000, + "CollectionPath": "data/collection.anki2" + }, + "SyncServer": { + "Host": "localhost", + "Port": 8080, + "UseHttps": false, + "Username": "", + "Password": "" + } +} diff --git a/src/Ankit.Server/paket.references b/src/Ankit.Server/paket.references index db4f7f8..d955110 100644 --- a/src/Ankit.Server/paket.references +++ b/src/Ankit.Server/paket.references @@ -1,2 +1,4 @@ group WebServer - Oxpecker \ No newline at end of file + Oxpecker + FsToolkit.ErrorHandling + FSharp.SystemTextJson diff --git a/tests/Ankit.Server.Tests/AnkiSyncServer.fs b/tests/Ankit.Server.Tests/AnkiSyncServer.fs new file mode 100644 index 0000000..51c18d1 --- /dev/null +++ b/tests/Ankit.Server.Tests/AnkiSyncServer.fs @@ -0,0 +1,124 @@ +module Ankit.Server.Tests.AnkiSyncServer + +open System +open System.Diagnostics +open System.IO +open System.Text.RegularExpressions +open System.Threading.Tasks +open Ankit.Server.Env +open DotNet.Testcontainers.Builders +open DotNet.Testcontainers.Containers +open Xunit + +let private repoRoot = + let rec find (dir: DirectoryInfo) = + if isNull dir then + failwith "paket.dependencies not found above the test output directory" + elif File.Exists(Path.Combine(dir.FullName, "paket.dependencies")) then + dir.FullName + else + find dir.Parent + + find (DirectoryInfo AppContext.BaseDirectory) + +/// The sync server is built from the same Anki tag as the rslib we link against, so the protocol versions match +let private ankiVersion = + let dependencies = File.ReadAllText(Path.Combine(repoRoot, "paket.dependencies")) + Regex.Match(dependencies, @"github ankitects/anki:(\S+)").Groups[1].Value + +let private linuxDockerAvailable = + lazy + try + use docker = + Process.Start( + ProcessStartInfo( + "docker", + "info --format {{.OSType}}", + RedirectStandardOutput = true, + RedirectStandardError = true + ) + ) + + let osType = docker.StandardOutput.ReadToEnd().Trim() + docker.WaitForExit(10_000) && docker.ExitCode = 0 && osType = "linux" + with _ -> + false + +type LinuxDockerFactAttribute() as this = + inherit FactAttribute() + + do + if not linuxDockerAvailable.Value then + this.Skip <- "Needs Docker with Linux containers" + +/// Parallel rustc jobs for compiling the sync server, more jobs need more memory in the Docker VM +let private cargoBuildJobs = + Environment.GetEnvironmentVariable "ANKIT_TESTS_CARGO_JOBS" + |> Option.ofObj + |> Option.defaultValue "4" + +[] +let private SyncPort = 8080 + +[] +let Password = "password" + +/// Accounts of the sync server, give every test its own so they don't see each other's collections +let Users = [ for i in 1..5 -> $"user{i}" ] + +type AnkiSyncServer() = + let mutable container: IContainer = null + + member _.Settings(username: string) : SyncServerSettings = { + Host = container.Hostname + Port = int (container.GetMappedPublicPort SyncPort) + UseHttps = false + Username = username + Password = Password + } + + interface IAsyncLifetime with + member _.InitializeAsync() = + task { + if linuxDockerAvailable.Value then + // Compiling the server takes several minutes the first time, Docker caches the layers after that + let image = + ImageFromDockerfileBuilder() + .WithDockerfileDirectory( + Path.Combine(repoRoot, "tests", "Ankit.Server.Tests", "SyncServer") + ) + .WithDockerfile("Dockerfile") + .WithBuildArgument("ANKI_VERSION", ankiVersion) + .WithBuildArgument("CARGO_BUILD_JOBS", cargoBuildJobs) + .WithName($"ankit-tests/anki-sync-server:{ankiVersion}") + .WithCleanUp(false) + .Build() + + do! image.CreateAsync() + + let builder = + Users + |> List.indexed + |> List.fold + (fun (b: ContainerBuilder) (i, user) -> + b.WithEnvironment($"SYNC_USER{i + 1}", $"{user}:{Password}")) + (ContainerBuilder(image.FullName)) + + container <- + builder + .WithPortBinding(SyncPort, true) + .WithWaitStrategy( + Wait + .ForUnixContainer() + .UntilHttpRequestIsSucceeded(fun r -> r.ForPort(uint16 SyncPort).ForPath("/health")) + ) + .Build() + + do! container.StartAsync() + } + + member _.DisposeAsync() = + if isNull container then + Task.CompletedTask + else + container.DisposeAsync().AsTask() diff --git a/tests/Ankit.Server.Tests/Ankit.Server.Tests.fsproj b/tests/Ankit.Server.Tests/Ankit.Server.Tests.fsproj index bb440c0..c81dff8 100644 --- a/tests/Ankit.Server.Tests/Ankit.Server.Tests.fsproj +++ b/tests/Ankit.Server.Tests/Ankit.Server.Tests.fsproj @@ -6,7 +6,9 @@ - + + + diff --git a/tests/Ankit.Server.Tests/SyncServer/Dockerfile b/tests/Ankit.Server.Tests/SyncServer/Dockerfile new file mode 100644 index 0000000..085769a --- /dev/null +++ b/tests/Ankit.Server.Tests/SyncServer/Dockerfile @@ -0,0 +1,33 @@ +# Anki's example sync server image (docs/syncserver in the Anki repo), with a limit on parallel +# compile jobs: by default cargo runs one rustc per core and runs out of memory on big machines +FROM rust:1.97.1-alpine3.23 AS builder + +ARG ANKI_VERSION +ARG CARGO_BUILD_JOBS=4 + +RUN apk update && apk add --no-cache build-base protobuf && rm -rf /var/cache/apk/* + +RUN CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} cargo install --git https://github.com/ankitects/anki.git \ +--tag ${ANKI_VERSION} \ +--root /anki-server \ +--locked \ +anki-sync-server + +FROM alpine:3.23 + +ENV PUID=1000 +ENV PGID=1000 + +COPY --from=builder /anki-server/bin/anki-sync-server /usr/local/bin/anki-sync-server + +RUN apk update && apk add --no-cache bash su-exec && rm -rf /var/cache/apk/* + +EXPOSE 8080 + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["anki-sync-server"] + +VOLUME /anki_data diff --git a/tests/Ankit.Server.Tests/SyncServer/entrypoint.sh b/tests/Ankit.Server.Tests/SyncServer/entrypoint.sh new file mode 100644 index 0000000..072ae15 --- /dev/null +++ b/tests/Ankit.Server.Tests/SyncServer/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -o errexit +set -o nounset +set -o pipefail + +# Default PUID and PGID if not provided +export PUID=${PUID:-1000} +export PGID=${PGID:-1000} + +# These values are fixed and cannot be overwritten from the outside for +# convenience and safety reasons +export SYNC_PORT=8080 +export SYNC_BASE=/anki_data + +# Check if group exists, create if not +if ! getent group anki-group > /dev/null 2>&1; then + addgroup -g "$PGID" anki-group +fi + +# Check if user exists, create if not +if ! id -u anki > /dev/null 2>&1; then + adduser -D -H -u "$PUID" -G anki-group anki +fi + +# Fix ownership of mounted volumes +mkdir -p /anki_data +chown anki:anki-group /anki_data + +# Run the provided command as the `anki` user +exec su-exec anki "$@" diff --git a/tests/Ankit.Server.Tests/SyncServiceTests.fs b/tests/Ankit.Server.Tests/SyncServiceTests.fs new file mode 100644 index 0000000..dac4df1 --- /dev/null +++ b/tests/Ankit.Server.Tests/SyncServiceTests.fs @@ -0,0 +1,90 @@ +namespace Ankit.Server.Tests + +open System.Linq +open Anki.Notes +open Anki.Search +open Anki.Sync +open Ankit.Native +open Ankit.Server.Env +open Ankit.Server.Services +open Ankit.Server.Tests.AnkiSyncServer +open Microsoft.Extensions.Logging.Abstractions +open Microsoft.Extensions.Options +open Xunit + +type SyncServiceTests(server: AnkiSyncServer) = + + /// xUnit can't print F# unions, compare the printed values so a failure shows what came back + let assertResult (expected: SyncResult) (actual: SyncResult) = + Assert.Equal(sprintf "%A" expected, sprintf "%A" actual) + + let syncService (backend: RslibBackend) (settings: SyncServerSettings) : ISyncService = + let monitor = { + new IOptionsMonitor with + member _.CurrentValue = settings + member _.Get _ = settings + member _.OnChange _ = null + } + + AnkiSyncService(backend, monitor, NullLogger.Instance) + + let addBasicNote (backend: RslibBackend) (front: string) (back: string) = + let basic = backend.GetNotetypeNames().Entries.Single(fun e -> e.Name = "Basic") + let note = backend.NewNote(Anki.Notetypes.NotetypeId(Ntid = basic.Id)) + note.Fields[0] <- front + note.Fields[1] <- back + backend.AddNote(AddNoteRequest(Note = note, DeckId = 1L)) |> ignore + + [] + member _.``Wrong password is reported as invalid credentials``() = + use collection = new TempCollection() + let settings = { server.Settings Users[0] with Password = "wrong" } + + assertResult (Error SyncFailed.InvalidCredentials) ((syncService collection.Backend settings).TrySync()) + + [] + member _.``Untouched collection has nothing to sync with a fresh account``() = + use collection = new TempCollection() + + assertResult (Ok()) ((syncService collection.Backend (server.Settings Users[1])).TrySync()) + + [] + member _.``First sync of a collection with notes needs a full upload``() = + use collection = new TempCollection() + addBasicNote collection.Backend "犬" "dog" + + let result = (syncService collection.Backend (server.Settings Users[2])).TrySync() + + assertResult (Error(SyncFailed.FullSyncRequired SyncCollectionResponse.Types.ChangesRequired.FullUpload)) result + + [] + member _.``Changes reach other clients after the initial full sync``() = + let settings = server.Settings Users[3] + use local = new TempCollection() + addBasicNote local.Backend "犬" "dog" + let service = syncService local.Backend settings + assertResult (Ok()) (service.FullSync Upload) + + addBasicNote local.Backend "猫" "cat" + assertResult (Ok()) (service.TrySync()) + assertResult (Ok()) (service.TrySync()) + + use other = new TempCollection() + let otherService = syncService other.Backend settings + + assertResult + (Error(SyncFailed.FullSyncRequired SyncCollectionResponse.Types.ChangesRequired.FullDownload)) + (otherService.TrySync()) + + assertResult (Ok()) (otherService.FullSync Download) + assertResult (Ok()) (otherService.TrySync()) + Assert.Equal(2, other.Backend.SearchNotes(SearchRequest(Search = "")).Ids.Count) + + [] + member _.``Full sync with a wrong password is reported as invalid credentials``() = + use collection = new TempCollection() + let settings = { server.Settings Users[4] with Password = "wrong" } + + assertResult (Error SyncFailed.InvalidCredentials) ((syncService collection.Backend settings).FullSync Upload) + + interface IClassFixture diff --git a/tests/Ankit.Server.Tests/TempCollection.fs b/tests/Ankit.Server.Tests/TempCollection.fs new file mode 100644 index 0000000..a0c2f02 --- /dev/null +++ b/tests/Ankit.Server.Tests/TempCollection.fs @@ -0,0 +1,28 @@ +namespace Ankit.Server.Tests + +open System +open System.IO +open Anki.Collection +open Ankit.Native + +/// A backend with a fresh collection in a temporary directory, removed on dispose +type TempCollection() = + let directory = Directory.CreateTempSubdirectory("ankit-server-tests-") + let backend = RslibBackend.Open [ "en" ] + + do + backend.OpenCollection( + OpenCollectionRequest( + CollectionPath = Path.Combine(directory.FullName, "collection.anki2"), + MediaFolderPath = Path.Combine(directory.FullName, "collection.media"), + MediaDbPath = Path.Combine(directory.FullName, "collection.media.db") + ) + ) + + member _.Backend = backend + + interface IDisposable with + member _.Dispose() = + backend.CloseCollection(CloseCollectionRequest()) + backend.Dispose() + directory.Delete(recursive = true) diff --git a/tests/Ankit.Server.Tests/Tests.fs b/tests/Ankit.Server.Tests/Tests.fs deleted file mode 100644 index ae63213..0000000 --- a/tests/Ankit.Server.Tests/Tests.fs +++ /dev/null @@ -1,7 +0,0 @@ -module Tests - -open System -open Xunit - -[] -let ``My test`` () = Assert.True(true) diff --git a/tests/Ankit.Server.Tests/paket.references b/tests/Ankit.Server.Tests/paket.references index fee5b69..8c5d90c 100644 --- a/tests/Ankit.Server.Tests/paket.references +++ b/tests/Ankit.Server.Tests/paket.references @@ -4,3 +4,4 @@ group Test xunit xunit.runner.visualstudio coverlet.collector + Testcontainers