dumili-api: Queue-based tasks

apps/dumili/api/.env

@@ -11,3 +11,5 @@ MYSQL_ROOT_PASSWORD=changeme
 SENTRY_DSN=
 
 TOKEN_SECRET=3543c30fe79047b4f73cfb61aa1eb52cb3173de4b3941e0fc4ec1b127bbeed6019695a1a453a81c33c2eea964ccc577e69c7df994124bd2751e262a311ea23a1
+
+REDIS_URL=redis://localhost:6379

apps/dumili/api/Dockerfile.worker

@@ -0,0 +1,11 @@
+FROM node:26-slim
+LABEL org.opencontainers.image.authors="Bruno Perel"
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y openssl && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+COPY apps/dumili/api/.env /app/
+COPY apps/dumili/api/dist/worker.mjs /app/worker.mjs
+COPY apps/dumili/api/dist/instrument.mjs /app/instrument.mjs
+
+CMD ["node", "--import", "./instrument.mjs", "worker.mjs"]

apps/dumili/api/docker-compose-dev.yml

@@ -22,6 +22,15 @@ services:
     build:
       context: ../../..
       dockerfile: apps/dumili/api/paddleocr/Dockerfile
+  redis:
+    container_name: dumili-redis
+    restart: always
+    image: redis:7-alpine
+    command: redis-server --appendonly yes
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis-data:/data
   db:
     container_name: dumili-db
     restart: always
@@ -35,3 +44,6 @@ services:
     environment:
       MYSQL_DATABASE: dumili
       MARIADB_ROOT_PASSWORD: $MYSQL_ROOT_PASSWORD
+
+volumes:
+  redis-data:

apps/dumili/api/docker-compose.yml

@@ -23,10 +23,29 @@ services:
     container_name: dumili-api
     image: ghcr.io/bperel/dumili-api
     restart: always
+    environment:
+      REDIS_URL: redis://redis:6379
     networks:
       - public-network
       - dumili-network
       - dm-api-network
+
+  dumili-worker:
+    container_name: dumili-worker
+    image: ghcr.io/bperel/dumili-worker
+    restart: always
+    environment:
+      REDIS_URL: redis://redis:6379
+    networks:
+      - dumili-network
+      - dm-api-network
+
+  redis:
+    container_name: dumili-redis
+    image: redis:7-alpine
+    restart: always
+    networks:
+      - dumili-network
   db:
     container_name: dumili-db
     restart: always

apps/dumili/api/index.ts

@@ -8,12 +8,14 @@ dotenv.config({
   override: true,
 });
 
+import { createAdapter } from "@socket.io/redis-adapter";
 import { v2 as cloudinary } from "cloudinary";
 import { createServer, type IncomingMessage, type ServerResponse } from "http";
 import { Server } from "socket.io";
 
 import type { SessionUser } from "~dm-types/SessionUser";
 
+import { createRedisClient } from "./queue/connection";
 import { authenticateUser } from "./services/_auth";
 import type { FullIndexation } from "./services/indexation";
 import { handleHttpFileUpload, server as indexation } from "./services/indexation";
@@ -40,6 +42,9 @@ const io = new Server(httpServer, {
   maxHttpBufferSize: 100 * 1024 * 1024,
 });
 
+const pubClient = createRedisClient();
+io.adapter(createAdapter(pubClient, pubClient.duplicate()));
+
 indexations(io);
 indexation(io);
 

apps/dumili/api/package.json

@@ -7,15 +7,18 @@
   },
   "scripts": {
     "clean": "rm -rf dist",
-    "build": "prisma generate && for source in index.ts instrument.ts; do bun build --entry-naming [dir]/[name].mjs --sourcemap=linked --target node --external @napi-rs/canvas $source --outdir dist; done && pnpm run copy-wasm && pnpm sentry:sourcemaps",
+    "build": "prisma generate && for source in index.ts instrument.ts worker.ts; do bun build --entry-naming [dir]/[name].mjs --sourcemap=linked --target node --external @napi-rs/canvas $source --outdir dist; done && pnpm run copy-wasm && pnpm sentry:sourcemaps",
     "copy-wasm": "cp node_modules/node-unrar-js/dist/js/unrar.wasm dist/ && cp node_modules/pdfjs-dist/build/pdf.worker.mjs dist/",
     "prisma-pull-generate": "prisma db pull && prisma generate",
     "prisma-generate": "prisma generate",
     "lint": "eslint --fix .",
     "dev": "docker compose -f docker-compose-dev.yml up --force-recreate -d && bash ../../../packages/prisma-schemas/wait-until-db-ready.bash && prisma generate && concurrently --kill-others-on-fail -n docker-compose,typecheck \"docker compose -f docker-compose-dev.yml logs -f\" \"tsc --noEmit\"",
     "dev:bun": "bun --inspect run index.ts",
+    "dev:worker": "bun run worker.ts",
+    "dev:all": "concurrently --kill-others-on-fail -n api,worker \"bun --inspect run index.ts\" \"bun run worker.ts\"",
     "prod:deploy": "DIR=apps/dumili/api pnpm -F '~ci' prod:docker-compose-up",
     "prod-build-docker-api": "REPO_NAME=ghcr.io/bperel/dumili-api pnpm -F '~ci' prod:build-docker -f apps/dumili/api/Dockerfile",
+    "prod-build-docker-worker": "REPO_NAME=ghcr.io/bperel/dumili-worker pnpm -F '~ci' prod:build-docker -f apps/dumili/api/Dockerfile.worker",
     "prod-build-docker-kumiko": "REPO_NAME=ghcr.io/bperel/kumiko pnpm -F '~ci' prod:build-docker -f apps/dumili/api/kumiko/Dockerfile",
     "prod-build-docker-paddleocr": "REPO_NAME=ghcr.io/bperel/paddleocr pnpm -F '~ci' prod:build-docker -f apps/dumili/api/paddleocr/Dockerfile",
     "prod:build-docker": "pnpm run '/^prod-build-docker-/'",
@@ -29,18 +32,22 @@
     "@prisma/client": "^7.8.0",
     "@sentry/node": "^10.62.0",
     "@sentry/profiling-node": "^10.62.0",
+    "@socket.io/redis-adapter": "^8.3.0",
+    "@socket.io/redis-emitter": "^5.1.0",
     "axios": "^1.18.1",
     "axios-cache-interceptor": "^1.12.0",
+    "bullmq": "^5.79.2",
     "busboy": "^1.6.0",
     "cloudinary": "^1.41.3",
     "dotenv": "^17.4.2",
     "fflate": "^0.8.3",
+    "ioredis": "^5.11.1",
     "jsonwebtoken": "^9.0.3",
     "node-unrar-js": "^2.0.2",
     "pdfjs-dist": "^6.1.200",
     "sharp": "^0.33.5",
     "socket-call-client": "^0.7.7",
-    "socket-call-server": "^0.7.7",
+    "socket-call-server": "^0.8.0",
     "socket.io": "^4.8.3",
     "unpdf": "^1.6.2",
     "~api": "workspace:*",