Add sync worker, HTTP API and sync server tests

.github/workflows/main.yml

@@ -24,10 +24,8 @@ jobs:
       uses: actions/setup-dotnet@v6
       with:
         dotnet-version: 10.0.x
-    - name: Verify encoding
-      run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Encoding
-    - name: Verify workflow
-      run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Workflow
+    - name: Verify sources
+      run: dotnet fsi Scripts/build.fsx -- --no-logo Verify-Sources
   build:
     name: Build and test (${{ matrix.image }})
     strategy:

Ankit.sln

@@ -8,7 +8,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		global.json = global.json
 		paket.dependencies = paket.dependencies
 		README.md = README.md
-		Scriptsuild.fsx = Scriptsuild.fsx
+		build.fsx = Scripts/build.fsx
+		gen-workflow.fsx = Scripts/gen-workflow.fsx
 	EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ankit.Native", "src\Ankit.Native\Ankit.Native.csproj", "{452916EC-79B6-4C8B-8528-369D929FD39B}"

Scripts/build.fsx

@@ -177,6 +177,9 @@ Target.create "Restore" (fun _ ->
 Target.create "Format" (fun _ ->
     dotnet "fantomas" "." |> handleResult "Format")
 
+Target.create "Verify-Format" (fun _ ->
+    dotnet "fantomas" "--check ." |> handleResult "Verify-Format")
+
 let utf8Bom = Encoding.UTF8.GetPreamble()
 
 let hasBom (bytes: ReadOnlySpan<byte>) =
@@ -410,6 +413,7 @@ Target.create "Prepare" doNothing
 Target.create "Build" doNothing
 Target.create "All" doNothing
 Target.create "Prepare-Sources" doNothing
+Target.create "Verify-Sources" doNothing
 Target.create "Before-Push" doNothing
 
 // --- Prepare: restore packages and tools, generate sources ---
@@ -455,17 +459,25 @@ Target.create "Before-Push" doNothing
 "Restore"
     ==> "Format"
 
+"Restore"
+    ==> "Verify-Format"
+
 "Remove-BOM"
     =?> ("Prepare-Sources", verifyEncoding() |> Result.isError)
 
 "Format"
     ==> "Prepare-Sources"
 
-"Before-Push" <== [
-    "Prepare-Sources"
+"Verify-Sources" <== [
     "Verify-Encoding"
     "Verify-Workflow"
+    "Verify-Format"
+]
+
+"Before-Push" <== [
     "Test"
+    "Prepare-Sources"
+    "Verify-Sources"
 ]
 
 if DocoptResult.hasFlag "--list" cli then

Scripts/gen-workflow.fsx

@@ -86,8 +86,7 @@ let workflow = [
             yield! env
             checkoutStep
             setUpDotNetSdkStep
-            buildCliStep "Verify encoding" "Verify-Encoding"
-            buildCliStep "Verify workflow" "Verify-Workflow"
+            buildCliStep "Verify sources" "Verify-Sources"
         ]
 
         job "build" [

paket.dependencies

@@ -5,6 +5,8 @@ group WebServer
     framework: net10.0
 
     nuget Oxpecker 2.1.1
+    nuget FsToolkit.ErrorHandling 5.2.0
+    nuget FSharp.SystemTextJson 1.4.36
 
 group Native
     source https://api.nuget.org/v3/index.json
@@ -25,3 +27,4 @@ group Test
     nuget xunit
     nuget xunit.runner.visualstudio
     nuget coverlet.collector
+    nuget Testcontainers

paket.lock

@@ -16,15 +16,48 @@ GROUP Test
 RESTRICTION: == net10.0
 NUGET
   remote: https://api.nuget.org/v3/index.json
+    BouncyCastle.Cryptography (2.7)
     coverlet.collector (10.1)
+    Docker.DotNet.Enhanced (4.3.3)
+      Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3)
+      Docker.DotNet.Enhanced.LegacyHttp (>= 4.3.3)
+      Docker.DotNet.Enhanced.NativeHttp (>= 4.3.3)
+      Docker.DotNet.Enhanced.NPipe (>= 4.3.3)
+      Docker.DotNet.Enhanced.Unix (>= 4.3.3)
+      Microsoft.Extensions.Logging.Abstractions (>= 8.0.3)
+    Docker.DotNet.Enhanced.Handler.Abstractions (4.3.3)
+      Microsoft.Extensions.Logging.Abstractions (>= 8.0.3)
+    Docker.DotNet.Enhanced.LegacyHttp (4.3.3)
+      Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3)
+    Docker.DotNet.Enhanced.NativeHttp (4.3.3)
+      Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3)
+    Docker.DotNet.Enhanced.NPipe (4.3.3)
+      Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3)
+    Docker.DotNet.Enhanced.Unix (4.3.3)
+      Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3)
+    Docker.DotNet.Enhanced.X509 (4.3.3)
+      Docker.DotNet.Enhanced.Handler.Abstractions (>= 4.3.3)
     FSharp.Core (10.1.402)
     Microsoft.CodeCoverage (18.11)
+    Microsoft.Extensions.DependencyInjection.Abstractions (10.0.13)
+    Microsoft.Extensions.Logging.Abstractions (10.0.13)
+      Microsoft.Extensions.DependencyInjection.Abstractions (>= 10.0.13)
     Microsoft.NET.Test.Sdk (18.11)
       Microsoft.CodeCoverage (>= 18.11)
       Microsoft.TestPlatform.TestHost (>= 18.11)
     Microsoft.TestPlatform.ObjectModel (18.11)
     Microsoft.TestPlatform.TestHost (18.11)
       Microsoft.TestPlatform.ObjectModel (>= 18.11)
+    SharpZipLib (1.4.2)
+    SSH.NET (2026.0)
+      BouncyCastle.Cryptography (>= 2.7)
+      Microsoft.Extensions.Logging.Abstractions (>= 8.0.3)
+    Testcontainers (4.16)
+      Docker.DotNet.Enhanced (>= 4.3.3)
+      Docker.DotNet.Enhanced.X509 (>= 4.3.3)
+      Microsoft.Extensions.Logging.Abstractions (>= 8.0.3)
+      SharpZipLib (>= 1.4.2)
+      SSH.NET (>= 2026.0)
     xunit (2.9.3)
       xunit.analyzers (>= 1.18)
       xunit.assert (>= 2.9.3)
@@ -46,6 +79,11 @@ RESTRICTION: == net10.0
 NUGET
   remote: https://api.nuget.org/v3/index.json
     FSharp.Core (10.1.402)
+    FSharp.SystemTextJson (1.4.36)
+      FSharp.Core (>= 4.7)
+      System.Text.Json (>= 6.0.10)
+    FsToolkit.ErrorHandling (5.2)
+      FSharp.Core (>= 9.0.300)
     Microsoft.Extensions.ObjectPool (10.0.13)
     Microsoft.IO.RecyclableMemoryStream (3.0.1)
     Oxpecker (2.1.1)
@@ -55,5 +93,6 @@ NUGET
     Oxpecker.ViewEngine (2.1)
       FSharp.Core (>= 10.0.100)
       Microsoft.Extensions.ObjectPool (>= 10.0)
+    System.Text.Json (10.0.13)
     TypeShape (10.0)
       FSharp.Core (>= 4.5.4)

src/Ankit.Server/AnkiSyncWorker.fs

@@ -0,0 +1,115 @@
+namespace Ankit.Server
+
+open System
+open System.Threading
+open System.Threading.Tasks
+open Ankit.Native
+open Ankit.Server.Env
+open Ankit.Server.Services
+open Microsoft.Extensions.Hosting
+open System.Threading.Channels
+open Microsoft.Extensions.Logging
+open Microsoft.Extensions.Options
+
+[<RequireQualifiedAccess>]
+type AnkiError =
+    | CreateDeckError
+    | AddNoteError
+    | FullSyncError of SyncFailed
+
+type AnkiCommand =
+    | CreateDeck of Anki.Decks.Deck
+    | AddNote of Anki.Notes.AddNoteRequest
+    | FullSync of SyncDirection
+
+type AnkiSyncMsg = AnkiSyncMsg of cmd: AnkiCommand * reply: TaskCompletionSource<Result<unit, AnkiError>>
+
+type AnkiSyncWorker
+    (
+        backend: RslibBackend,
+        syncService: ISyncService,
+        ch: Channel<AnkiSyncMsg>,
+        logger: ILogger<AnkiSyncWorker>,
+        settings: IOptionsMonitor<AnkitServerSettings>
+    ) =
+    inherit BackgroundService()
+
+    let getDebounceTime () =
+        TimeSpan.FromMilliseconds(float settings.CurrentValue.SyncDebounceTime)
+
+    let applyCommand (cmd: AnkiCommand) : Result<unit, AnkiError> =
+        match cmd with
+        | FullSync direction -> syncService.FullSync direction |> Result.mapError AnkiError.FullSyncError
+        | CreateDeck _
+        | AddNote _ ->
+            try
+                match cmd with
+                | CreateDeck deck -> backend.AddDeck deck |> ignore
+                | AddNote noteRequest -> backend.AddNote noteRequest |> ignore
+                | FullSync _ -> ()
+
+                Ok()
+            with :? RslibException as e ->
+                logger.LogError(e, "Failed to apply {Command}", cmd)
+
+                match cmd with
+                | CreateDeck _ -> Error AnkiError.CreateDeckError
+                | _ -> Error AnkiError.AddNoteError
+
+    override this.ExecuteAsync(stoppingToken) =
+        task {
+            logger.LogInformation("AnkiSyncWorker is starting.")
+
+            let rd = ch.Reader
+            let requiresSync = ref false
+
+            let applyPending () =
+                let mutable msg = Unchecked.defaultof<AnkiSyncMsg>
+
+                while rd.TryRead(&msg) do
+                    let (AnkiSyncMsg(cmd, reply)) = msg
+                    let result = applyCommand cmd
+
+                    match cmd, result with
+                    | FullSync _, Ok() -> requiresSync.Value <- false
+                    | _, Ok() -> requiresSync.Value <- true
+                    | _, Error _ -> ()
+
+                    reply.TrySetResult result |> ignore
+
+            try
+                let mutable running = true
+
+                while running do
+                    let! canRead = rd.WaitToReadAsync(stoppingToken)
+
+                    if not canRead then
+                        running <- false
+                    else
+                        applyPending ()
+                        use window = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken)
+                        window.CancelAfter(getDebounceTime ())
+
+                        try
+                            let mutable windowOpen = true
+
+                            while windowOpen do
+                                let! more = rd.WaitToReadAsync(window.Token)
+                                if more then applyPending () else windowOpen <- false
+                        with :? OperationCanceledException when not stoppingToken.IsCancellationRequested ->
+                            ()
+
+                        if requiresSync.Value then
+                            requiresSync.Value <- false
+                            syncService.TrySync() |> ignore
+            with :? OperationCanceledException when stoppingToken.IsCancellationRequested ->
+                ()
+
+            let mutable msg = Unchecked.defaultof<AnkiSyncMsg>
+
+            while rd.TryRead(&msg) do
+                let (AnkiSyncMsg(_, reply)) = msg
+                reply.TrySetCanceled(stoppingToken) |> ignore
+
+            logger.LogInformation("AnkiSyncWorker is stopping due to cancellation.")
+        }

src/Ankit.Server/Ankit.Server.fsproj

@@ -7,16 +7,32 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="Utils.fs" />
+    <Compile Include="Env.fs" />
+    <Compile Include="Services.fs" />
+    <Compile Include="AnkiSyncWorker.fs" />
+    <Compile Include="Dto.fs" />
+    <Compile Include="ErrorHandler.fs" />
+    <Compile Include="Handlers.fs" />
     <Compile Include="Program.fs"/>
   </ItemGroup>
 
   <ItemGroup>
     <Content Include="..\..\.dockerignore">
       <Link>.dockerignore</Link>
     </Content>
+    <Content Include="appsettings.json" CopyToOutputDirectory="PreserveNewest" />
     <Content Include="Dockerfile" />
     <Content Include="paket.references" />
   </ItemGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\Ankit.Native\Ankit.Native.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Ankit.Native\Ankit.Native.csproj" />
+  </ItemGroup>
+
   <Import Project="..\..\.paket\Paket.Restore.targets" />
 </Project>