feat(eDd): add s3 object store + test page

Cargo.toml

@@ -13,6 +13,12 @@ publish = false
 
 [workspace.dependencies]
 axum = "0.8"
+# S3-compatible object storage (libs/object-storage). Default HTTPS client: rustls on aws-lc-rs.
+aws-sdk-s3 = { version = "1.157.0", default-features = false, features = [
+  "default-https-client",
+  "http-1x",
+  "rt-tokio",
+] }
 axum-extra = { version = "0.12", features = ["cookie-private", "typed-header"] }
 http = "1"
 tokio = { version = "1.53", features = ["full"] }
@@ -38,6 +44,7 @@ oauth2 = { version = "5.0.0", default-features = false }
 serde = { version = "1.0.229", features = ["derive"] }
 time = "0.3.55"
 url = "2.5.8"
+bytes = "1.12.1"
 sha2 = "0.11.1"
 # aws-lc-rs: the rustls provider reqwest already uses.
 sqlx = { version = "0.9.0", default-features = false, features = [
@@ -53,6 +60,7 @@ sqlx = { version = "0.9.0", default-features = false, features = [
 uuid = "1.28.0"
 cdb = { path = "databases/curriculum" }
 edb = { path = "databases/examiner" }
+object-storage = { path = "libs/object-storage" }
 runtime = { path = "libs/runtime" }
 
 [workspace.lints.clippy]

README.md

@@ -31,9 +31,9 @@ apps/
 databases/
   | curriculum/
   | examiner/
-  | moderation/
 docs/ # mdbook docs
 libs/ # Common across one or more applications
+  | object-storage/ # S3-compatible client, one bucket per store, configured from env
   | runtime/ # env config, tracing + Sentry, Axum serving + graceful shutdown, exit codes
 ```
 

apps/examiner-dashboard/.env.example

@@ -19,3 +19,13 @@ COOKIE_KEY=
 
 # Hosted Postgres: append `?sslmode=verify-full`.
 EDB_DATABASE_URL=postgres://edd:<EDB_EDD_PASSWORD>@127.0.0.1:15432/examiner
+
+# Curriculum bucket (question objects): any S3-compatible provider. DO Spaces: endpoint
+# https://<region>.digitaloceanspaces.com, region us-east-1, an access key scoped to the bucket.
+CURRICULUM_S3_ENDPOINT=https://nyc3.digitaloceanspaces.com
+CURRICULUM_S3_REGION=us-east-1
+CURRICULUM_S3_BUCKET=
+CURRICULUM_S3_ACCESS_KEY_ID=
+CURRICULUM_S3_SECRET_ACCESS_KEY=
+# true for path-style providers (Garage, SeaweedFS). Default false.
+CURRICULUM_S3_FORCE_PATH_STYLE=

apps/examiner-dashboard/.oxfmtrc.json

@@ -1,5 +1,5 @@
 {
   "$schema": "./node_modules/oxfmt/configuration_schema.json",
-  "ignorePatterns": ["server/**", "dist/**", ".astro/**"],
+  "ignorePatterns": ["server/**", "dist/**", ".astro/**", "CHANGELOG.md"],
   "astro": true
 }

apps/examiner-dashboard/Cargo.toml

@@ -10,10 +10,11 @@ path = "server/main.rs"
 name = "server"
 
 [dependencies]
-axum.workspace = true
+axum = { workspace = true, features = ["multipart"] }
 axum-extra.workspace = true
 edb.workspace = true
 http.workspace = true
+object-storage.workspace = true
 tokio.workspace = true
 tower-http = { workspace = true, features = ["fs"] }
 tracing.workspace = true
@@ -23,7 +24,7 @@ oauth2.workspace = true
 reqwest = { workspace = true, features = ["json", "rustls"] }
 serde.workspace = true
 sqlx.workspace = true
-time.workspace = true
+time = { workspace = true, features = ["formatting"] }
 url.workspace = true
 
 [dev-dependencies]

apps/examiner-dashboard/client/src/components/page-shell.tsx

@@ -1,11 +1,13 @@
 import { Box, Button, Center, Flex, Heading, HStack, Text } from "@chakra-ui/react";
 import type { ReactNode } from "react";
-import { LuBlocks, LuGraduationCap, LuHouse } from "react-icons/lu";
+import { LuBlocks, LuDatabase, LuFile, LuGraduationCap, LuHouse } from "react-icons/lu";
 import { ColorModeButton } from "@/components/ui/color-mode";
 
 const navItems = [
   { href: "/", label: "Home", icon: LuHouse },
   { href: "/question-builder", label: "Question Builder", icon: LuBlocks },
+  { href: "/file-dragger", label: "File Dragger", icon: LuFile },
+  { href: "/object-store", label: "Object Store", icon: LuDatabase },
 ];
 
 export function PageShell({

apps/examiner-dashboard/client/src/pages/file-dragger.astro

@@ -0,0 +1,8 @@
+---
+import Base from "@/layouts/base.astro";
+import { FileDragger } from "@/views/file-dragger";
+---
+
+<Base title="File Dragger">
+  <FileDragger client:load />
+</Base>

apps/examiner-dashboard/client/src/pages/object-store.astro

@@ -0,0 +1,8 @@
+---
+import Base from "@/layouts/base.astro";
+import { ObjectStore } from "@/views/object-store";
+---
+
+<Base title="Object Store">
+  <ObjectStore client:load />
+</Base>

apps/examiner-dashboard/client/src/utils/fetch.ts

@@ -25,8 +25,7 @@ async function responseError(res: Response): Promise<HttpError> {
   return new HttpError(res.status, `${res.status} - ${body || res.statusText}`);
 }
 
-// For every request needing login. 401 (no or expired session) sends the user to /login; other
-// failures, 403 included, reject with the server's message.
+// For every request needing login. 401 (no or expired session) sends the user to /login; other failures, 403 included, reject with the server's message.
 export async function authorizedFetch(url: string, options?: RequestInit): Promise<Response> {
   const res = await fetch(url, options);
 
@@ -66,3 +65,83 @@ export async function postDevLogin(user: { name: string; email: string }): Promi
     throw await responseError(res);
   }
 }
+
+// An object's eDb record.
+export interface ObjectRecord {
+  // What a question's `question_objects` row stores.
+  sha256: string;
+  // Sniffed by the server from the bytes.
+  contentType: string;
+  // As first uploaded, unless renamed.
+  fileName: string;
+  // RFC 3339.
+  createdAt: string;
+}
+
+// A recorded object: its record, with its bytes' key and a link to them.
+export interface RecordedObject extends ObjectRecord {
+  // In the curriculum bucket: `sha256/<sha256>.<ext>`.
+  objectKey: string;
+  byteSize: number;
+  // Presigned, expires in minutes. Display only: never store it.
+  url: string;
+}
+
+export async function getObjects(): Promise<RecordedObject[]> {
+  const res = await authorizedFetch("/api/objects");
+  return (await res.json()) as RecordedObject[];
+}
+
+// Already-stored files come back as stored, under their first upload's name.
+export async function uploadObjects(files: File[]): Promise<RecordedObject[]> {
+  const body = new FormData();
+  for (const file of files) {
+    body.append("file", file);
+  }
+
+  const res = await authorizedFetch("/api/objects", { method: "POST", body });
+  return (await res.json()) as RecordedObject[];
+}
+
+// `recorded` - in the bucket and eDb.
+// `unrecorded` - in the bucket only.
+// `missing` - in eDb only, its bytes gone.
+export type StoreEntryStatus = "recorded" | "unrecorded" | "missing";
+
+// One key in the curriculum bucket, eDb or both.
+export interface StoreEntry {
+  key: string;
+  status: StoreEntryStatus;
+  // Null when unrecorded.
+  record: ObjectRecord | null;
+  // The bucket's when listed, else eDb's.
+  byteSize: number;
+  // RFC 3339. Null when missing.
+  lastModified: string | null;
+  // Presigned, expires in minutes. Null when missing. Display only: never store it.
+  url: string | null;
+}
+
+// The key goes in the query, never the path: a valid S3 key such as `a/../b` would be normalized by the browser into a request for another key.
+function storeObjectUrl(key: string): string {
+  return `/api/object-store/object?${new URLSearchParams({ key })}`;
+}
+
+export async function getStoreEntries(): Promise<StoreEntry[]> {
+  const res = await authorizedFetch("/api/object-store");
+  return (await res.json()) as StoreEntry[];
+}
+
+// Recorded objects only.
+export async function renameObject(key: string, fileName: string): Promise<void> {
+  await authorizedFetch(storeObjectUrl(key), {
+    method: "PATCH",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ fileName }),
+  });
+}
+
+// Permanent. Unchecked: cDb commits that reference the object lose its bytes.
+export async function deleteObject(key: string): Promise<void> {
+  await authorizedFetch(storeObjectUrl(key), { method: "DELETE" });
+}

apps/examiner-dashboard/client/src/utils/objects.ts

@@ -0,0 +1,43 @@
+// Upload policy for question objects. Mirrors server/routes/objects.rs (`KINDS` and the limits), so
+// the dropzone rejects what the server would. The server sniffs the bytes and rejects anything else.
+
+export const MAX_FILE_BYTES = 10 * 1024 * 1024;
+export const MAX_FILES_PER_REQUEST = 5;
+
+interface ObjectKind {
+  name: string;
+  // Browsers report some types under other names (`audio/x-m4a`, `audio/x-wav`), or none at all, so
+  // extensions match too.
+  contentTypes: string[];
+  extensions: string[];
+}
+
+export const OBJECT_KINDS: ObjectKind[] = [
+  { name: "PNG", contentTypes: ["image/png"], extensions: [".png"] },
+  { name: "JPEG", contentTypes: ["image/jpeg"], extensions: [".jpg", ".jpeg"] },
+  { name: "GIF", contentTypes: ["image/gif"], extensions: [".gif"] },
+  { name: "WebP", contentTypes: ["image/webp"], extensions: [".webp"] },
+  { name: "MP3", contentTypes: ["audio/mpeg", "audio/mp3"], extensions: [".mp3"] },
+  {
+    name: "WAV",
+    contentTypes: ["audio/wav", "audio/x-wav", "audio/wave", "audio/vnd.wave"],
+    extensions: [".wav"],
+  },
+  { name: "Ogg", contentTypes: ["audio/ogg"], extensions: [".ogg", ".oga"] },
+  { name: "FLAC", contentTypes: ["audio/flac", "audio/x-flac"], extensions: [".flac"] },
+  { name: "M4A", contentTypes: ["audio/mp4", "audio/x-m4a"], extensions: [".m4a"] },
+];
+
+// For the file picker and dropzone: a file matching any entry is accepted.
+export const ACCEPT = OBJECT_KINDS.flatMap((kind) => [...kind.contentTypes, ...kind.extensions]);
+
+// "PNG, JPEG, ... or M4A".
+export const KIND_NAMES = new Intl.ListFormat("en", { type: "disjunction" }).format(
+  OBJECT_KINDS.map((kind) => kind.name),
+);
+
+export function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KiB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
+}