diff --git a/cmd/gomodel/docs/docs.go b/cmd/gomodel/docs/docs.go index 24ffb6836..ae15c7e4d 100644 --- a/cmd/gomodel/docs/docs.go +++ b/cmd/gomodel/docs/docs.go @@ -252,6 +252,12 @@ const docTemplate = `{ "name": "stream", "in": "query" }, + { + "type": "string", + "description": "Comma-separated endpoint operations to hide, e.g. mcp,provider_passthrough,audio_speech; other entries, including unclassified ones, stay", + "name": "exclude_operation", + "in": "query" + }, { "type": "string", "description": "Search across request_id/requested_model/provider/method/path/session_id/error_type/error_message", @@ -381,6 +387,12 @@ const docTemplate = `{ "name": "stream", "in": "query" }, + { + "type": "string", + "description": "Comma-separated endpoint operations to hide, e.g. mcp,provider_passthrough,audio_speech; other entries, including unclassified ones, stay", + "name": "exclude_operation", + "in": "query" + }, { "type": "string", "description": "Search across request_id/requested_model/provider/method/path/session_id/error_type/error_message", diff --git a/config/env.go b/config/env.go index 17c6dfcd5..00919d895 100644 --- a/config/env.go +++ b/config/env.go @@ -66,7 +66,7 @@ func applyPluginsLoadEnv(cfg *Config) { return } load := make([]PluginFileConfig, 0, 4) - for _, item := range strings.Split(v, ",") { + for item := range strings.SplitSeq(v, ",") { if entry := parsePluginLoadEntry(item); entry.File != "" { load = append(load, entry) } diff --git a/docs/openapi.json b/docs/openapi.json index 4d8e196fa..f425f6bee 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -330,6 +330,14 @@ "type": "boolean" } }, + { + "description": "Comma-separated endpoint operations to hide, e.g. mcp,provider_passthrough,audio_speech; other entries, including unclassified ones, stay", + "name": "exclude_operation", + "in": "query", + "schema": { + "type": "string" + } + }, { "description": "Search across request_id/requested_model/provider/method/path/session_id/error_type/error_message", "name": "search", @@ -507,6 +515,14 @@ "type": "boolean" } }, + { + "description": "Comma-separated endpoint operations to hide, e.g. mcp,provider_passthrough,audio_speech; other entries, including unclassified ones, stay", + "name": "exclude_operation", + "in": "query", + "schema": { + "type": "string" + } + }, { "description": "Search across request_id/requested_model/provider/method/path/session_id/error_type/error_message", "name": "search", diff --git a/internal/admin/handler_audit.go b/internal/admin/handler_audit.go index 124732f62..9f8712e3a 100644 --- a/internal/admin/handler_audit.go +++ b/internal/admin/handler_audit.go @@ -51,6 +51,7 @@ const conversationBuildTimeout = 10 * time.Second // @Param error_type query string false "Filter by error type" // @Param status_code query int false "Filter by status code" // @Param stream query bool false "Filter by stream mode (true/false)" +// @Param exclude_operation query string false "Comma-separated endpoint operations to hide, e.g. mcp,provider_passthrough,audio_speech; other entries, including unclassified ones, stay" // @Param search query string false "Search across request_id/requested_model/provider/method/path/session_id/error_type/error_message" // @Param limit query int false "Page size (default 25, max 100)" // @Param offset query int false "Offset for pagination" @@ -169,6 +170,14 @@ func parseAuditLogQueryParams(c *echo.Context) (auditlog.LogQueryParams, error) params.Stream = &parsed } + if raw := c.QueryParam("exclude_operation"); raw != "" { + ops, unknown, ok := core.ParseOperations(raw) + if !ok { + return params, core.NewInvalidRequestError("invalid exclude_operation: "+unknown, nil) + } + params.ExcludeOperations = ops + } + if l := c.QueryParam("limit"); l != "" { parsed, err := strconv.Atoi(l) if err != nil || parsed <= 0 { @@ -211,6 +220,7 @@ func parseAuditLogQueryParams(c *echo.Context) (auditlog.LogQueryParams, error) // @Param error_type query string false "Filter by error type" // @Param status_code query int false "Filter by status code" // @Param stream query bool false "Filter by stream mode (true/false)" +// @Param exclude_operation query string false "Comma-separated endpoint operations to hide, e.g. mcp,provider_passthrough,audio_speech; other entries, including unclassified ones, stay" // @Param search query string false "Search across request_id/requested_model/provider/method/path/session_id/error_type/error_message" // @Param limit query int false "Page size in threads (default 25, max 100)" // @Param offset query int false "Offset for pagination" diff --git a/internal/admin/handler_audit_sessions_test.go b/internal/admin/handler_audit_sessions_test.go index 6c4f2fa2b..fb8af79db 100644 --- a/internal/admin/handler_audit_sessions_test.go +++ b/internal/admin/handler_audit_sessions_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/enterpilot/gomodel/internal/auditlog" + "github.com/enterpilot/gomodel/internal/core" "github.com/enterpilot/gomodel/internal/echotest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -137,3 +138,17 @@ func TestAuditLog_SessionIDSkipsDefaultDateWindow(t *testing.T) { require.False(t, reader.lastQuery.StartDate.IsZero()) require.False(t, reader.lastQuery.EndDate.IsZero()) } + +func TestAuditLog_ExcludeOperationFilter(t *testing.T) { + reader := &mockAuditReader{logResult: &auditlog.LogListResult{}} + h := NewHandler(nil, nil, WithAuditReader(reader)) + + c, _ := echotest.Get(t, "/admin/audit/log?exclude_operation=mcp,audio_speech") + require.NoError(t, h.AuditLog(c)) + assert.Equal(t, []core.Operation{core.OperationMCP, core.OperationAudioSpeech}, reader.lastQuery.ExcludeOperations) + + c, rec := echotest.Get(t, "/admin/audit/log?exclude_operation=mcp,nope") + require.NoError(t, h.AuditLog(c)) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "invalid exclude_operation: nope") +} diff --git a/internal/auditlog/reader.go b/internal/auditlog/reader.go index 9554a6b24..6dfcaa056 100644 --- a/internal/auditlog/reader.go +++ b/internal/auditlog/reader.go @@ -3,6 +3,8 @@ package auditlog import ( "context" "time" + + "github.com/enterpilot/gomodel/internal/core" ) // QueryParams specifies the date range for audit log retrieval. @@ -24,8 +26,12 @@ type LogQueryParams struct { Search string StatusCode *int Stream *bool - Limit int - Offset int + // ExcludeOperations drops entries whose path belongs to one of these + // operations. Entries outside every operation (e.g. authentication + // events) always stay. + ExcludeOperations []core.Operation + Limit int + Offset int // OmitAttempts excludes provider attempts from returned entries. The default is false. OmitAttempts bool // ExactUserPath matches only UserPath instead of its subtree. The default is false. diff --git a/internal/auditlog/reader_mongodb.go b/internal/auditlog/reader_mongodb.go index 0069df70b..76555e42f 100644 --- a/internal/auditlog/reader_mongodb.go +++ b/internal/auditlog/reader_mongodb.go @@ -283,6 +283,9 @@ func mongoLogMatchFilters(params LogQueryParams) (bson.D, error) { if params.Stream != nil { matchFilters = append(matchFilters, bson.E{Key: "stream", Value: *params.Stream}) } + if len(params.ExcludeOperations) > 0 { + matchFilters = append(matchFilters, mongoExcludeOperationsFilter(params.ExcludeOperations)) + } if params.Search != "" && isCanonicalUUID(params.Search) { // A full canonical UUID is a pasted identifier: match the indexed // identity fields by equality (both spellings — stored ids are @@ -417,3 +420,25 @@ func (r *MongoDBReader) findConversationEntry(ctx context.Context, filter bson.D return row.toLogEntry(), nil } + +// mongoExcludeOperationsFilter drops paths belonging to any of the +// operations; entries without a path stay. Exact paths also match with one +// trailing slash, as DescribeEndpoint does. +func mongoExcludeOperationsFilter(ops []core.Operation) bson.E { + var exact bson.A + var nor bson.A + for _, op := range ops { + paths, _ := core.PathsForOperation(op) + for _, path := range paths.Exact { + exact = append(exact, path, path+"/") + } + for _, prefix := range paths.Prefixes { + exact = append(exact, prefix) + nor = append(nor, bson.D{{Key: "path", Value: bson.D{ + {Key: "$regex", Value: "^" + regexp.QuoteMeta(prefix+"/")}, + }}}) + } + } + nor = append(nor, bson.D{{Key: "path", Value: bson.D{{Key: "$in", Value: exact}}}}) + return bson.E{Key: "$nor", Value: nor} +} diff --git a/internal/auditlog/reader_sql.go b/internal/auditlog/reader_sql.go index 05ea05160..e47548052 100644 --- a/internal/auditlog/reader_sql.go +++ b/internal/auditlog/reader_sql.go @@ -12,6 +12,7 @@ import ( "github.com/goccy/go-json" + "github.com/enterpilot/gomodel/internal/core" "github.com/enterpilot/gomodel/internal/storage/sqlutil" "github.com/enterpilot/gomodel/internal/storage/sqlx" ) @@ -243,6 +244,10 @@ func (r *SQLReader) logFilters(ctx context.Context, params LogQueryParams) ([]st if params.Stream != nil { add("stream = ?", *params.Stream) } + if len(params.ExcludeOperations) > 0 { + condition, values := excludeOperationsSQLFilter(params.ExcludeOperations) + add(condition, values...) + } if params.Search != "" { condition, values := r.searchFilter(params.Search, r.searchIsIndexed(ctx)) add(condition, values...) @@ -560,3 +565,23 @@ func isMissingAuditAttemptsTable(err error) bool { return strings.Contains(message, "audit_log_attempts") && (strings.Contains(message, "no such table") || strings.Contains(message, "does not exist")) } + +// excludeOperationsSQLFilter drops paths belonging to any of the operations. +// Exact paths also match with one trailing slash, as DescribeEndpoint does. +// The prefixes hold no LIKE wildcards, so they need no escaping. +func excludeOperationsSQLFilter(ops []core.Operation) (string, []any) { + var clauses []string + var args []any + for _, op := range ops { + paths, _ := core.PathsForOperation(op) + for _, exact := range paths.Exact { + clauses = append(clauses, "path = ?", "path = ?") + args = append(args, exact, exact+"/") + } + for _, prefix := range paths.Prefixes { + clauses = append(clauses, "path = ?", "path LIKE ?") + args = append(args, prefix, prefix+"/%") + } + } + return "(path IS NULL OR NOT (" + strings.Join(clauses, " OR ") + "))", args +} diff --git a/internal/auditlog/reader_suite_test.go b/internal/auditlog/reader_suite_test.go index decbeb639..c40be48f3 100644 --- a/internal/auditlog/reader_suite_test.go +++ b/internal/auditlog/reader_suite_test.go @@ -2,11 +2,13 @@ package auditlog import ( "context" + "fmt" "testing" "time" "go.mongodb.org/mongo-driver/v2/mongo" + "github.com/enterpilot/gomodel/internal/core" "github.com/enterpilot/gomodel/internal/storage/mongotest" "github.com/enterpilot/gomodel/internal/storage/sqlx" "github.com/enterpilot/gomodel/internal/storage/sqlx/sqlxtest" @@ -103,3 +105,57 @@ func TestReader_GetLastUsedByAuthKeys(t *testing.T) { assert.False(t, ok) }) } + +func TestReader_GetLogsExcludesOperations(t *testing.T) { + runReaderSuite(t, func(t *testing.T, store LogStore, reader Reader) { + ctx := context.Background() + base := time.Date(2026, 1, 16, 12, 0, 0, 0, time.UTC) + paths := []string{ + "/v1/chat/completions", "/mcp", "/mcp/github", "/mcpx", + "/v1/audio/speech", "/v1/audio/speech/", "/v1/audio/transcriptions", + "/p/openai/v1/models", "/sso/callback", "", + } + entries := make([]*LogEntry, 0, len(paths)) + for i, path := range paths { + entries = append(entries, &LogEntry{ + ID: fmt.Sprintf("op-%d", i), + Timestamp: base.Add(time.Duration(i) * time.Minute), + Path: path, + }) + } + require.NoError(t, store.WriteBatch(ctx, entries)) + + tests := []struct { + name string + ops []core.Operation + want []string + }{ + {name: "mcp prefix", ops: []core.Operation{core.OperationMCP}, want: []string{ + "/v1/chat/completions", "/mcpx", "/v1/audio/speech", "/v1/audio/speech/", + "/v1/audio/transcriptions", "/p/openai/v1/models", "/sso/callback", "", + }}, + {name: "exact with trailing slash and prefix", ops: []core.Operation{ + core.OperationAudioSpeech, core.OperationProviderPassthrough, + }, want: []string{ + "/v1/chat/completions", "/mcp", "/mcp/github", "/mcpx", + "/v1/audio/transcriptions", "/sso/callback", "", + }}, + {name: "every classified type keeps unclassified rows", ops: []core.Operation{ + core.OperationChatCompletions, core.OperationMCP, core.OperationAudioSpeech, + core.OperationAudioTranscriptions, core.OperationProviderPassthrough, + }, want: []string{"/mcpx", "/sso/callback", ""}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := reader.GetLogs(ctx, LogQueryParams{ExcludeOperations: tt.ops, Limit: 50}) + require.NoError(t, err) + got := make([]string, 0, len(result.Entries)) + for _, entry := range result.Entries { + got = append(got, entry.Path) + } + assert.ElementsMatch(t, tt.want, got) + assert.Equal(t, len(tt.want), result.Total) + }) + } + }) +} diff --git a/internal/core/endpoint_operations.go b/internal/core/endpoint_operations.go new file mode 100644 index 000000000..6ab6df986 --- /dev/null +++ b/internal/core/endpoint_operations.go @@ -0,0 +1,58 @@ +package core + +import "strings" + +// OperationPaths lists the request paths that DescribeEndpoint classifies as +// one operation, in a shape storage filters can match: Exact paths compare by +// equality, and each Prefix matches itself or anything under "Prefix/". +type OperationPaths struct { + Exact []string + Prefixes []string +} + +// operationPaths mirrors describeEndpointPath. TestOperationPathsMatchDescribeEndpoint +// keeps the two in sync. +var operationPaths = map[Operation]OperationPaths{ + OperationChatCompletions: {Exact: []string{"/v1/chat/completions", "/v1/messages", "/v1/messages/count_tokens"}}, + OperationResponses: {Prefixes: []string{"/v1/responses"}}, + OperationConversations: {Prefixes: []string{"/v1/conversations"}}, + OperationEmbeddings: {Exact: []string{"/v1/embeddings"}}, + OperationBatches: {Prefixes: []string{"/v1/batches", "/v1/messages/batches"}}, + OperationFiles: {Prefixes: []string{"/v1/files"}}, + OperationAudioSpeech: {Exact: []string{"/v1/audio/speech"}}, + OperationAudioTranscriptions: {Exact: []string{"/v1/audio/transcriptions"}}, + OperationAudioTranslations: {Exact: []string{"/v1/audio/translations"}}, + OperationImageGenerations: {Exact: []string{"/v1/images/generations"}}, + OperationImageEdits: {Exact: []string{"/v1/images/edits"}}, + OperationRealtime: {Exact: []string{ + "/v1/realtime", "/v1/realtime/calls", "/v1/realtime/client_secrets", + "/v1/realtime/translations", "/v1/realtime/translations/calls", "/v1/realtime/translations/client_secrets", + }}, + OperationMCP: {Prefixes: []string{"/mcp"}}, + OperationProviderPassthrough: {Prefixes: []string{"/p"}}, +} + +// PathsForOperation returns the paths of a known operation. +func PathsForOperation(op Operation) (OperationPaths, bool) { + paths, ok := operationPaths[op] + return paths, ok +} + +// ParseOperations parses a comma-separated operation list, ignoring blanks +// and duplicates. It reports the first unknown name. +func ParseOperations(raw string) ([]Operation, string, bool) { + var ops []Operation + seen := map[Operation]bool{} + for part := range strings.SplitSeq(raw, ",") { + op := Operation(strings.ToLower(strings.TrimSpace(part))) + if op == "" || seen[op] { + continue + } + if _, ok := operationPaths[op]; !ok { + return nil, string(op), false + } + seen[op] = true + ops = append(ops, op) + } + return ops, "", true +} diff --git a/internal/core/endpoint_operations_test.go b/internal/core/endpoint_operations_test.go new file mode 100644 index 000000000..ba2bd8edd --- /dev/null +++ b/internal/core/endpoint_operations_test.go @@ -0,0 +1,72 @@ +package core + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOperationPathsMatchDescribeEndpoint(t *testing.T) { + for op, paths := range operationPaths { + for _, path := range paths.Exact { + assert.Equal(t, op, DescribeEndpointPath(path).Operation, path) + } + for _, prefix := range paths.Prefixes { + assert.Equal(t, op, DescribeEndpointPath(prefix+"/x").Operation, prefix+"/x") + } + } +} + +func TestOperationPathsCoverClassifiedPaths(t *testing.T) { + paths := []string{ + "/v1/chat/completions", "/v1/messages", "/v1/messages/count_tokens", + "/v1/responses", "/v1/responses/resp_1/input_items", "/v1/conversations/conv_1", + "/v1/embeddings", "/v1/batches/b_1/cancel", "/v1/messages/batches/b_1", + "/v1/files/f_1/content", "/v1/audio/speech", "/v1/audio/transcriptions", + "/v1/audio/translations", "/v1/images/generations", "/v1/images/edits", + "/v1/realtime", "/v1/realtime/translations/calls", "/mcp", "/mcp/github", + "/p/openai/v1/models", + } + for _, path := range paths { + want := DescribeEndpointPath(path).Operation + require.NotEmpty(t, want, path) + rule, ok := PathsForOperation(want) + require.True(t, ok, path) + assert.True(t, rule.matches(path), path) + } +} + +func (p OperationPaths) matches(path string) bool { + if slices.Contains(p.Exact, path) { + return true + } + for _, prefix := range p.Prefixes { + if path == prefix || len(path) > len(prefix) && path[:len(prefix)+1] == prefix+"/" { + return true + } + } + return false +} + +func TestParseOperations(t *testing.T) { + tests := []struct { + name string + raw string + want []Operation + unknown string + }{ + {name: "empty", raw: ""}, + {name: "list", raw: " MCP, audio_speech,,mcp ", want: []Operation{OperationMCP, OperationAudioSpeech}}, + {name: "unknown", raw: "mcp,nope", unknown: "nope"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, unknown, ok := ParseOperations(tt.raw) + assert.Equal(t, tt.unknown == "", ok) + assert.Equal(t, tt.unknown, unknown) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/web/dashboard/messages/de.json b/web/dashboard/messages/de.json index 4fd061389..5add5e321 100644 --- a/web/dashboard/messages/de.json +++ b/web/dashboard/messages/de.json @@ -225,6 +225,18 @@ "audit_filter_all_modes": "Alle Modi", "audit_filter_streaming": "Streaming", "audit_filter_non_streaming": "Ohne Streaming", + "audit_filter_type_label": "Filter nach Anfragetyp", + "audit_filter_all_types": "Alle Typen", + "audit_filter_types_hidden": "Typen: {count} ausgeblendet", + "audit_type_chat": "Chat", + "audit_type_responses": "Responses", + "audit_type_embeddings": "Embeddings", + "audit_type_audio": "Audio", + "audit_type_images": "Bilder", + "audit_type_batches": "Batches & Dateien", + "audit_type_realtime": "Realtime", + "audit_type_passthrough": "Passthrough", + "audit_type_mcp": "MCP", "audit_live_connecting": "Verbindung zum Live-Stream…", "audit_live": "Live", "audit_live_pause_date_range": "Live pausiert — der gewählte Zeitraum schließt heute nicht ein. Stelle ihn auf heute ein, um fortzufahren.", diff --git a/web/dashboard/messages/en.json b/web/dashboard/messages/en.json index 2a60dae34..9e12f9e6e 100644 --- a/web/dashboard/messages/en.json +++ b/web/dashboard/messages/en.json @@ -225,6 +225,18 @@ "audit_filter_all_modes": "All Modes", "audit_filter_streaming": "Streaming", "audit_filter_non_streaming": "Non-streaming", + "audit_filter_type_label": "Request type filter", + "audit_filter_all_types": "All Types", + "audit_filter_types_hidden": "Types: {count} hidden", + "audit_type_chat": "Chat", + "audit_type_responses": "Responses", + "audit_type_embeddings": "Embeddings", + "audit_type_audio": "Audio", + "audit_type_images": "Images", + "audit_type_batches": "Batches & files", + "audit_type_realtime": "Realtime", + "audit_type_passthrough": "Passthrough", + "audit_type_mcp": "MCP", "audit_live_connecting": "Live stream connecting…", "audit_live": "Live", "audit_live_pause_date_range": "Live paused — the selected date range does not include today. Set it to today to resume.", diff --git a/web/dashboard/messages/pl.json b/web/dashboard/messages/pl.json index ee9a53914..ba9c432e1 100644 --- a/web/dashboard/messages/pl.json +++ b/web/dashboard/messages/pl.json @@ -227,6 +227,18 @@ "audit_filter_all_modes": "Wszystkie tryby", "audit_filter_streaming": "Streaming", "audit_filter_non_streaming": "Bez streamingu", + "audit_filter_type_label": "Filtr typu żądania", + "audit_filter_all_types": "Wszystkie typy", + "audit_filter_types_hidden": "Typy: ukryte {count}", + "audit_type_chat": "Chat", + "audit_type_responses": "Responses", + "audit_type_embeddings": "Embeddingi", + "audit_type_audio": "Audio", + "audit_type_images": "Obrazy", + "audit_type_batches": "Batche i pliki", + "audit_type_realtime": "Realtime", + "audit_type_passthrough": "Passthrough", + "audit_type_mcp": "MCP", "audit_live_connecting": "Łączenie ze strumieniem Live…", "audit_live": "Live", "audit_live_pause_date_range": "Live wstrzymany — wybrany zakres dat nie obejmuje dzisiaj. Ustaw datę dzisiejszą, aby wznowić.", diff --git a/web/dashboard/messages/zh-CN.json b/web/dashboard/messages/zh-CN.json index db1ba6496..16c2eb5f1 100644 --- a/web/dashboard/messages/zh-CN.json +++ b/web/dashboard/messages/zh-CN.json @@ -216,6 +216,18 @@ "audit_filter_all_modes": "全部模式", "audit_filter_streaming": "流式", "audit_filter_non_streaming": "非流式", + "audit_filter_type_label": "请求类型筛选", + "audit_filter_all_types": "全部类型", + "audit_filter_types_hidden": "类型:已隐藏 {count} 个", + "audit_type_chat": "对话", + "audit_type_responses": "Responses", + "audit_type_embeddings": "嵌入", + "audit_type_audio": "音频", + "audit_type_images": "图像", + "audit_type_batches": "批处理与文件", + "audit_type_realtime": "实时", + "audit_type_passthrough": "透传", + "audit_type_mcp": "MCP", "audit_live_connecting": "正在连接实时流…", "audit_live": "实时", "audit_live_pause_date_range": "实时流已暂停 — 所选日期范围未包含今天。将其设为今天即可恢复。", diff --git a/web/dashboard/src/pages/audit-logs/AuditFilters.svelte b/web/dashboard/src/pages/audit-logs/AuditFilters.svelte index 3f7744b96..f3adf91ca 100644 --- a/web/dashboard/src/pages/audit-logs/AuditFilters.svelte +++ b/web/dashboard/src/pages/audit-logs/AuditFilters.svelte @@ -1,15 +1,35 @@
@@ -69,6 +89,25 @@ +
+ + {hiddenCount > 0 ? m.audit_filter_types_hidden({ count: hiddenCount }) : m.audit_filter_all_types()} + +
+ {m.audit_filter_type_label()} + {#each AUDIT_TYPES as type (type.key)} + {@const visible = !auditList.auditHiddenTypes.includes(type.key)} + + {/each} +
+