Merge pull request #742 from devforth/feature/AdminForth/1955/implement-ability-for-agent-pl

Feature/admin forth/1955/implement ability for agent pl

Author: srelon

dev-demo/globalPlugins.ts

@@ -107,6 +107,7 @@ export const globalPlugins = [
       responseField: 'response',
       debugField: 'dubbug',
       checkpointIdField: 'checkpoint_id',
+      attachmentsField: 'attachments',
     },
     checkpointResource: {
       resourceId: 'agent_checkpoints',

dev-demo/migrations/prisma/sqlite/migrations/20261028120000_add_turns_attachments_field/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "turns" ADD COLUMN "attachments" TEXT;

dev-demo/migrations/prisma/sqlite/schema.prisma

@@ -164,6 +164,7 @@ model turns {
   response      String?
   dubbug        String?
   checkpoint_id String?
+  attachments   String?
 }
 
 model agent_checkpoints {

dev-demo/resources/agent_resources/checkpoints.ts

@@ -8,6 +8,14 @@ export default {
   table: 'agent_checkpoints',
   resourceId: 'agent_checkpoints',
   label: 'Agent Checkpoints',
+  // written by the agent only; edits here would bypass its checks and its file bookkeeping
+  options: {
+    allowedActions: {
+      create: false,
+      edit: false,
+      delete: false,
+    },
+  },
   columns: [
     { name: 'id', primaryKey: true, type: AdminForthDataTypes.STRING },
     { name: 'thread_id', type: AdminForthDataTypes.STRING },

dev-demo/resources/agent_resources/turns.ts

@@ -1,12 +1,23 @@
 import AdminForth, { AdminForthDataTypes } from 'adminforth';
 import type { AdminForthResourceInput, AdminUser } from 'adminforth';
 import { randomUUID } from 'crypto';
+import UploadPlugin from '../../../plugins/adminforth-upload/index.js';
+import AdminForthStorageAdapterLocalFilesystem from '../../../adapters/adminforth-storage-adapter-local/index.js';
+import AdminForthAdapterS3Storage from '../../../adapters/adminforth-storage-adapter-amazon-s3/index.js';
 
 export default {
   dataSource: 'sqlite',
   table: 'turns',
   resourceId: 'turns',
   label: 'Turns',
+  // written by the agent only; edits here would bypass its checks and its file bookkeeping
+  options: {
+    allowedActions: {
+      create: false,
+      edit: false,
+      delete: false,
+    },
+  },
   columns: [
     {
       name: 'id',
@@ -70,5 +81,39 @@ export default {
         filter: false,
       },
     },
+    {
+      name: 'attachments',
+      type: AdminForthDataTypes.JSON,
+      isArray: {
+        enabled: true,
+        itemType: AdminForthDataTypes.STRING,
+      },
+      showIn: {
+        list: false,
+        show: true,
+        edit: false,
+        create: false,
+        filter: false,
+      },
+    },
+  ],
+  plugins: [
+    new UploadPlugin({
+      pathColumnName: 'attachments',
+      storageAdapter: process.env.USE_S3 !== 'true' ? new AdminForthStorageAdapterLocalFilesystem({
+        fileSystemFolder: './db/uploads_agent_attachments',
+        mode: 'private',
+        signingSecret: '1241245',
+      }) : new AdminForthAdapterS3Storage({
+        bucket: process.env.AWS_BUCKET_NAME as string,
+        region: process.env.AWS_REGION as string,
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID as string,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY as string,
+        strictContentType: true,
+      }),
+      allowedFileExtensions: ['png', 'jpg', 'jpeg', 'webp'],
+      maxFileSize: 1024 * 1024 * 5,
+      filePath: ({ originalExtension }) => `agent_attachments/${randomUUID()}.${originalExtension}`,
+    }),
   ],
 } as AdminForthResourceInput;

documentation/docs/tutorial/06-Adapters/04-storage-adapters.md

@@ -30,7 +30,7 @@ new AdminForthAdapterS3Storage({
 }),
 ```
 
-Before turning it on, update `@adminforth/bulk-ai-flow`, `@adminforth/markdown`, `@adminforth/rich-editor` and `@adminforth/oauth`: their older versions send the type reported by the browser, and the upload fails when it differs from the one picked by the extension. Your own code that uploads files to the URL from the [upload plugin API](../09-Plugins/05-1-upload-api.md) has to send the `contentType` returned with the URL as well. While the option is off, the adapter logs a warning on start. It will be on by default in the next major version.
+The option is available since `@adminforth/storage-adapter-amazon-s3` 1.3.0. Before turning it on, update `@adminforth/bulk-ai-flow`, `@adminforth/markdown`, `@adminforth/rich-editor` and `@adminforth/oauth`: their older versions send the type reported by the browser, and the upload fails when it differs from the one picked by the extension. Your own code that uploads files to the URL from the [upload plugin API](../09-Plugins/05-1-upload-api.md) has to send the `contentType` returned with the URL as well. While the option is off, the adapter logs a warning on start. It will be on by default in the next major version.
 
 ## S3 compatible adapter
 ```bash
@@ -39,7 +39,7 @@ pnpm add @adminforth/storage-adapter-s3-compatible
 
 Provides S3 compatible interface for object storage services such as MinIO, Wasabi, Cloudflare R2 or other third-party S3 providers.
 
-It supports the same `strictContentType` option as the Amazon S3 adapter, see [Strict content type](#strict-content-type).
+It supports the same `strictContentType` option as the Amazon S3 adapter since version 1.2.0, see [Strict content type](#strict-content-type).
 
 ### How the adapter works
 

documentation/docs/tutorial/09-Plugins/01-agent.md

@@ -302,6 +302,81 @@ Each item in `modes` defines a user-selectable preset in the chat UI. The select
 
 The plugin adds a chat surface to the admin UI, keeps session history per admin user, and shows a mode picker when `modes` are configured.
 
+## Image attachments
+
+Users can paste, attach or drop images (PNG, JPEG, WebP) in the chat, up to 5 per message, and the agent sees them. The images are stored by the [Upload plugin](./05-0-upload.md) on a column of the turns resource.
+
+1. Add a nullable column for the paths of the images to the turns table and describe it as a string array:
+
+```ts title="./resources/turns.ts"
+{
+  name: 'attachments',
+  type: AdminForthDataTypes.JSON,
+  isArray: {
+    enabled: true,
+    itemType: AdminForthDataTypes.STRING,
+  },
+  showIn: { list: false, show: true, edit: false, create: false, filter: false },
+},
+```
+
+2. Install the Upload plugin on this column, with images only:
+
+```ts title="./resources/turns.ts"
+plugins: [
+  new UploadPlugin({
+    pathColumnName: 'attachments',
+    storageAdapter: new AdminForthAdapterS3Storage({
+      // ...bucket, region, credentials
+      strictContentType: true,
+    }),
+    allowedFileExtensions: ['png', 'jpg', 'jpeg', 'webp'],
+    filePath: ({ originalExtension }) => `agent_attachments/${randomUUID()}.${originalExtension}`,
+  }),
+],
+```
+
+With an S3 storage `strictContentType: true` is required: the browser uploads the image straight to the bucket, and without it any content can be stored under an image name, see [Strict content type](../06-Adapters/04-storage-adapters.md#strict-content-type). The option is available since `@adminforth/storage-adapter-amazon-s3` 1.3.0 and `@adminforth/storage-adapter-s3-compatible` 1.2.0.
+
+3. Make the turns and checkpoints resources read-only in the admin panel. The agent writes them itself, while create or edit rights on the turns resource would also let admins download any file of its storage through the Upload plugin:
+
+```ts title="./resources/turns.ts"
+options: {
+  allowedActions: {
+    create: false,
+    edit: false,
+    delete: false,
+  },
+},
+```
+
+4. Pass the column to the plugin:
+
+```ts
+turnResource: {
+  // ...
+  attachmentsField: 'attachments',
+},
+```
+
+Before the upload the browser scales an image down to fit 1920×1080 and converts it to WebP (to JPEG in a browser which can not encode WebP, e.g. Safari). The plugin accepts images up to 3.75 MB: the model gets them as base64, and the smallest per-image limit of the providers is 5 MB. Every image costs tokens on every model call, so the model sees an image for a while after it was sent, and only the latest few at once. Older images are replaced with a note, and the agent loads them again with the `view_image` tool when they matter for the answer.
+
+The limits are options of the plugin, each a whole number from 1:
+
+```ts
+new AdminForthAgent({
+  // ...
+  imageAttachments: {
+    // images one message can have, 5 by default
+    maxPerMessage: 5,
+    // minutes the model sees an image after it was sent or loaded again with view_image, 30 by default
+    visibleMinutes: 30,
+    // images the model sees at once, the latest ones, 5 by default
+    maxVisible: 5,
+  },
+}),
+```
+
 ## Chat surfaces (Telegram, etc.)
 
 By default, the Agent plugin exposes a chat surface inside the AdminForth admin UI.