diff --git a/cmd/community/server.go b/cmd/community/server.go index 7b9edd94f67..36a4b495bf6 100644 --- a/cmd/community/server.go +++ b/cmd/community/server.go @@ -46,6 +46,7 @@ import ( "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/types/authtypes" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/version" "github.com/SigNoz/signoz/pkg/zeus" @@ -103,8 +104,8 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, authtypes.NewRegistry()), nil }, - func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, _ querier.Querier, _ licensing.Licensing, tagModule tag.Module) dashboard.Module { - return impldashboard.NewModule(impldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, tagModule) + func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, _ querier.Querier, _ licensing.Licensing, tagModule tag.Module, systemDashboardRegistry dashboardtypes.SystemDashboardRegistry) dashboard.Module { + return impldashboard.NewModule(impldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, tagModule, systemDashboardRegistry) }, func(_ licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config] { return noopgateway.NewProviderFactory() diff --git a/cmd/enterprise/server.go b/cmd/enterprise/server.go index 3dd09a970a3..db79628b561 100644 --- a/cmd/enterprise/server.go +++ b/cmd/enterprise/server.go @@ -63,6 +63,7 @@ import ( "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/version" "github.com/SigNoz/signoz/pkg/zeus" @@ -136,8 +137,8 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e } return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, licensing, onBeforeRoleDelete, authtypes.NewRegistry()), nil }, - func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing, tagModule tag.Module) dashboard.Module { - return impldashboard.NewModule(pkgimpldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, querier, licensing, tagModule) + func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing, tagModule tag.Module, systemDashboardRegistry dashboardtypes.SystemDashboardRegistry) dashboard.Module { + return impldashboard.NewModule(pkgimpldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, querier, licensing, tagModule, systemDashboardRegistry) }, func(licensing licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config] { return httpgateway.NewProviderFactory(licensing) diff --git a/docs/api/openapi.yml b/docs/api/openapi.yml index ebd5aaf550f..a7b713179ce 100644 --- a/docs/api/openapi.yml +++ b/docs/api/openapi.yml @@ -2944,6 +2944,46 @@ components: publicDashboard: $ref: '#/components/schemas/DashboardtypesGettablePublicDasbhboard' type: object + DashboardtypesGettableSystemDashboard: + properties: + createdAt: + format: date-time + type: string + createdBy: + type: string + image: + type: string + locked: + type: boolean + name: + type: string + orgId: + type: string + schemaVersion: + type: string + source: + $ref: '#/components/schemas/DashboardtypesSource' + spec: + $ref: '#/components/schemas/DashboardtypesDashboardSpec' + tags: + items: + $ref: '#/components/schemas/TagtypesGettableTag' + nullable: true + type: array + updatedAt: + format: date-time + type: string + updatedBy: + type: string + required: + - orgId + - locked + - source + - schemaVersion + - name + - tags + - spec + type: object DashboardtypesHistogramBuckets: properties: bucketCount: @@ -15501,6 +15541,73 @@ paths: summary: Migrate dashboard to v2 tags: - dashboard + /api/v2/dashboards/system/{name}: + get: + deprecated: false + description: Returns a dashboard SigNoz ships and owns, addressed by its stable + definition name (e.g. `ai-o11y-overview`) rather than its id. System dashboards + are read-only and upgraded through releases. The dashboard's own `name` field + carries a reserved prefix that the path segment must not include. + operationId: GetSystemDashboard + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + properties: + data: + $ref: '#/components/schemas/DashboardtypesGettableSystemDashboard' + status: + type: string + required: + - status + - data + type: object + description: OK + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Bad Request + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Forbidden + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Not Found + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Internal Server Error + security: + - api_key: + - dashboard:read + - tokenizer: + - dashboard:read + summary: Get system dashboard + tags: + - dashboard /api/v2/factor_password/forgot: post: deprecated: false diff --git a/ee/modules/dashboard/impldashboard/module.go b/ee/modules/dashboard/impldashboard/module.go index ad89c492c5a..15025de9867 100644 --- a/ee/modules/dashboard/impldashboard/module.go +++ b/ee/modules/dashboard/impldashboard/module.go @@ -32,9 +32,9 @@ type module struct { tagModule tag.Module } -func NewModule(store dashboardtypes.Store, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing, tagModule tag.Module) dashboard.Module { +func NewModule(store dashboardtypes.Store, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing, tagModule tag.Module, systemDashboardRegistry dashboardtypes.SystemDashboardRegistry) dashboard.Module { scopedProviderSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/ee/modules/dashboard/impldashboard") - pkgDashboardModule := pkgimpldashboard.NewModule(store, settings, analytics, orgGetter, queryParser, tagModule) + pkgDashboardModule := pkgimpldashboard.NewModule(store, settings, analytics, orgGetter, queryParser, tagModule, systemDashboardRegistry) return &module{ pkgDashboardModule: pkgDashboardModule, @@ -361,6 +361,14 @@ func (module *module) LockUnlock(ctx context.Context, orgID valuer.UUID, id valu return module.pkgDashboardModule.LockUnlock(ctx, orgID, id, updatedBy, isAdmin, lock) } +func (module *module) ReconcileSystemDashboards(ctx context.Context, orgID valuer.UUID) error { + return module.pkgDashboardModule.ReconcileSystemDashboards(ctx, orgID) +} + +func (module *module) GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.DashboardV2, error) { + return module.pkgDashboardModule.GetSystemDashboard(ctx, orgID, name) +} + func (module *module) delete(ctx context.Context, orgID, id valuer.UUID) error { return module.store.RunInTx(ctx, func(ctx context.Context) error { if err := module.store.DeletePublic(ctx, id.String()); err != nil && !errors.Ast(err, errors.TypeNotFound) { diff --git a/frontend/src/api/generated/services/dashboard/index.ts b/frontend/src/api/generated/services/dashboard/index.ts index 8a3019cb380..6120e805ea9 100644 --- a/frontend/src/api/generated/services/dashboard/index.ts +++ b/frontend/src/api/generated/services/dashboard/index.ts @@ -46,6 +46,8 @@ import type { GetPublicDashboardPathParameters, GetPublicDashboardWidgetQueryRange200, GetPublicDashboardWidgetQueryRangePathParameters, + GetSystemDashboard200, + GetSystemDashboardPathParameters, ListDashboardViews200, ListDashboardsForUserV2200, ListDashboardsForUserV2Params, @@ -1885,6 +1887,108 @@ export const useMigrateDashboardV2 = < > => { return useMutation(getMigrateDashboardV2MutationOptions(options)); }; +/** + * Returns a dashboard SigNoz ships and owns, addressed by its stable definition name (e.g. `ai-o11y-overview`) rather than its id. System dashboards are read-only and upgraded through releases. The dashboard's own `name` field carries a reserved prefix that the path segment must not include. + * @summary Get system dashboard + */ +export const getSystemDashboard = ( + { name }: GetSystemDashboardPathParameters, + signal?: AbortSignal, +) => { + return GeneratedAPIInstance({ + url: `/api/v2/dashboards/system/${name}`, + method: 'GET', + signal, + }); +}; + +export const getGetSystemDashboardQueryKey = ({ + name, +}: GetSystemDashboardPathParameters) => { + return [`/api/v2/dashboards/system/${name}`] as const; +}; + +export const getGetSystemDashboardQueryOptions = < + TData = Awaited>, + TError = ErrorType, +>( + { name }: GetSystemDashboardPathParameters, + options?: { + query?: UseQueryOptions< + Awaited>, + TError, + TData + >; + }, +) => { + const { query: queryOptions } = options ?? {}; + + const queryKey = + queryOptions?.queryKey ?? getGetSystemDashboardQueryKey({ name }); + + const queryFn: QueryFunction< + Awaited> + > = ({ signal }) => getSystemDashboard({ name }, signal); + + return { + queryKey, + queryFn, + enabled: !!name, + ...queryOptions, + } as UseQueryOptions< + Awaited>, + TError, + TData + > & { queryKey: QueryKey }; +}; + +export type GetSystemDashboardQueryResult = NonNullable< + Awaited> +>; +export type GetSystemDashboardQueryError = ErrorType; + +/** + * @summary Get system dashboard + */ + +export function useGetSystemDashboard< + TData = Awaited>, + TError = ErrorType, +>( + { name }: GetSystemDashboardPathParameters, + options?: { + query?: UseQueryOptions< + Awaited>, + TError, + TData + >; + }, +): UseQueryResult & { queryKey: QueryKey } { + const queryOptions = getGetSystemDashboardQueryOptions({ name }, options); + + const query = useQuery(queryOptions) as UseQueryResult & { + queryKey: QueryKey; + }; + + return { ...query, queryKey: queryOptions.queryKey }; +} + +/** + * @summary Get system dashboard + */ +export const invalidateGetSystemDashboard = async ( + queryClient: QueryClient, + { name }: GetSystemDashboardPathParameters, + options?: InvalidateOptions, +): Promise => { + await queryClient.invalidateQueries( + { queryKey: getGetSystemDashboardQueryKey({ name }) }, + options, + ); + + return queryClient; +}; + /** * This endpoint returns the sanitized v2-shape dashboard data for public access. Each panel query is reduced to a safe field subset, so filters and raw query strings are not exposed. * @summary Get public dashboard data (v2) diff --git a/frontend/src/api/generated/services/sigNoz.schemas.ts b/frontend/src/api/generated/services/sigNoz.schemas.ts index 0453f403cbc..2a86ee28f69 100644 --- a/frontend/src/api/generated/services/sigNoz.schemas.ts +++ b/frontend/src/api/generated/services/sigNoz.schemas.ts @@ -4960,6 +4960,53 @@ export interface DashboardtypesGettablePublicDashboardDataV2DTO { publicDashboard?: DashboardtypesGettablePublicDasbhboardDTO; } +export interface DashboardtypesGettableSystemDashboardDTO { + /** + * @type string + * @format date-time + */ + createdAt?: string; + /** + * @type string + */ + createdBy?: string; + /** + * @type string + */ + image?: string; + /** + * @type boolean + */ + locked: boolean; + /** + * @type string + */ + name: string; + /** + * @type string + */ + orgId: string; + /** + * @type string + */ + schemaVersion: string; + source: DashboardtypesSourceDTO; + spec: DashboardtypesDashboardSpecDTO; + /** + * @type array,null + */ + tags: TagtypesGettableTagDTO[] | null; + /** + * @type string + * @format date-time + */ + updatedAt?: string; + /** + * @type string + */ + updatedBy?: string; +} + export enum DashboardtypesPatchOpDTO { add = 'add', remove = 'remove', @@ -11471,6 +11518,17 @@ export type MigrateDashboardV2200 = { status: string; }; +export type GetSystemDashboardPathParameters = { + name: string; +}; +export type GetSystemDashboard200 = { + data: DashboardtypesGettableSystemDashboardDTO; + /** + * @type string + */ + status: string; +}; + export type GetFeatures200 = { /** * @type array diff --git a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx index 03c9e7c5123..93cabb8f171 100644 --- a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx +++ b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx @@ -38,19 +38,20 @@ import { TEST_ENDPOINT, } from '../../__tests__/fixtures'; +const SAMPLE_SPAN = JSON.parse(SAMPLE_SPAN_JSON) as { + attributes: Record; + resource: Record; +}; + +const MAPPED_ATTRIBUTE_KEY = 'gen_ai.content.prompt'; + +// Deriving from the sample keeps exactly one key added, so the single `populated` badge assertion below stays exact. const RESULT_SPAN = { attributes: { - 'my_company.llm.input': 'What is quantum computing?', - 'llm.input_messages': 'What is quantum computing?', - 'gen_ai.request.model': 'gpt-4', - 'gen_ai.usage.total_tokens': 1250, - 'gen_ai.content.completion': 'Quantum computing leverages...', - 'gen_ai.content.prompt': 'What is quantum computing?', - }, - resource: { - 'service.name': 'llm-gateway', - 'deployment.environment': 'production', + ...SAMPLE_SPAN.attributes, + [MAPPED_ATTRIBUTE_KEY]: SAMPLE_SPAN.attributes['input.value'], }, + resource: SAMPLE_SPAN.resource, }; const EDITED_SPAN_JSON = `{ @@ -97,7 +98,7 @@ describe('TestTab — sample-span flow', () => { ).resolves.toBeInTheDocument(); expect(screen.getByTestId('test-result-0')).toBeInTheDocument(); expect(screen.getByTestId('test-result-0-attributes')).toHaveTextContent( - 'gen_ai.content.prompt', + MAPPED_ATTRIBUTE_KEY, ); expect(screen.getByText('populated')).toBeInTheDocument(); expect(screen.queryByTestId('test-error')).not.toBeInTheDocument(); diff --git a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/spanInputStorage.ts b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/spanInputStorage.ts index fc2440a19a0..3fe7bb480e1 100644 --- a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/spanInputStorage.ts +++ b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/spanInputStorage.ts @@ -7,11 +7,14 @@ import { parseSpanInput } from './testPayload'; export const SAMPLE_SPAN_JSON = `{ "attributes": { - "my_company.llm.input": "What is quantum computing?", - "llm.input_messages": "What is quantum computing?", - "gen_ai.request.model": "gpt-4", - "gen_ai.usage.total_tokens": 1250, - "gen_ai.content.completion": "Quantum computing leverages..." + "llm.model_name": "gpt-4o", + "llm.provider": "openai", + "llm.token_count.prompt": 1024, + "llm.token_count.completion": 226, + "llm.token_count.prompt_details.cache_read": 512, + "input.value": "What is quantum computing?", + "output.value": "Quantum computing leverages superposition and entanglement...", + "session.id": "chat-8f2e41" }, "resource": { "service.name": "llm-gateway", diff --git a/pkg/apiserver/signozapiserver/dashboard.go b/pkg/apiserver/signozapiserver/dashboard.go index d84aee62e48..5b963726bba 100644 --- a/pkg/apiserver/signozapiserver/dashboard.go +++ b/pkg/apiserver/signozapiserver/dashboard.go @@ -332,6 +332,33 @@ func (provider *provider) addDashboardRoutes(router *mux.Router) error { return err } + if err := router.Handle("/api/v2/dashboards/system/{name}", handler.New( + provider.authzMiddleware.CheckResources(provider.dashboardHandler.GetSystemDashboard, authtypes.SigNozAdminRoleName, authtypes.SigNozEditorRoleName, authtypes.SigNozViewerRoleName), + handler.OpenAPIDef{ + ID: "GetSystemDashboard", + Tags: []string{"dashboard"}, + Summary: "Get system dashboard", + Description: "Returns a dashboard SigNoz ships and owns, addressed by its stable definition name (e.g. `ai-o11y-overview`) rather than its id. System dashboards are read-only and upgraded through releases. The dashboard's own `name` field carries a reserved prefix that the path segment must not include.", + Request: nil, + RequestContentType: "", + Response: new(dashboardtypes.GettableSystemDashboard), + ResponseContentType: "application/json", + SuccessStatusCode: http.StatusOK, + ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + Deprecated: false, + SecuritySchemes: newScopedSecuritySchemes([]string{coretypes.ResourceMetaResourceDashboard.Scope(coretypes.VerbRead)}), + }, + handler.WithResourceDefs(handler.BasicResourceDef{ + Resource: coretypes.ResourceMetaResourceDashboard, + Verb: coretypes.VerbRead, + Category: coretypes.ActionCategoryDataAccess, + ID: provider.systemDashboardID(), + Selector: coretypes.IDSelector, + }), + )).Methods(http.MethodGet).GetError(); err != nil { + return err + } + // Pinning mutates the calling user's pin list, not the dashboard, so it rides // on the collection-level list permission rather than a per-dashboard check. // The id is still extracted, for audit. @@ -718,3 +745,23 @@ func (provider *provider) addDashboardRoutes(router *mux.Router) error { return nil } + +// systemDashboardID resolves the {name} path param to the dashboard's id. Authz +// tuples and audit records are written against ids, so the name has to be +// resolved before either runs. +func (provider *provider) systemDashboardID() coretypes.ResourceIDExtractor { + return coretypes.NewResourceIDExtractor(coretypes.PhaseRequest, func(ec coretypes.ExtractorContext) (string, error) { + ctx := ec.Request.Context() + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + return "", err + } + + systemDashboard, err := provider.dashboardModule.GetSystemDashboard(ctx, valuer.MustNewUUID(claims.OrgID), mux.Vars(ec.Request)["name"]) + if err != nil { + return "", err + } + + return systemDashboard.ID.StringValue(), nil + }) +} diff --git a/pkg/modules/cloudintegration/config.go b/pkg/modules/cloudintegration/config.go index 69de8b9e464..6d35946e5f4 100644 --- a/pkg/modules/cloudintegration/config.go +++ b/pkg/modules/cloudintegration/config.go @@ -22,7 +22,7 @@ func newConfig() factory.Config { Agent: AgentConfig{ // we will maintain the latest version of cloud integration agent from here, // till we automate it externally or figure out a way to validate it. - Version: "v0.0.13", + Version: "v0.0.14", }, } } diff --git a/pkg/modules/dashboard/dashboard.go b/pkg/modules/dashboard/dashboard.go index 063c20294ba..f8c33e0f0b0 100644 --- a/pkg/modules/dashboard/dashboard.go +++ b/pkg/modules/dashboard/dashboard.go @@ -99,6 +99,14 @@ type Module interface { DeleteView(ctx context.Context, orgID valuer.UUID, id valuer.UUID) error GetByMetricNamesV2(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]dashboardtypes.DashboardPanelRef, error) + + // ════════════════════════════════════════════════════════════════════════ + // System dashboard methods + // ════════════════════════════════════════════════════════════════════════ + + ReconcileSystemDashboards(ctx context.Context, orgID valuer.UUID) error + + GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.DashboardV2, error) } type Handler interface { @@ -162,4 +170,6 @@ type Handler interface { UpdateView(http.ResponseWriter, *http.Request) DeleteView(http.ResponseWriter, *http.Request) + + GetSystemDashboard(http.ResponseWriter, *http.Request) } diff --git a/pkg/modules/dashboard/impldashboard/fs/definitions/ai-o11y-overview.json b/pkg/modules/dashboard/impldashboard/fs/definitions/ai-o11y-overview.json new file mode 100644 index 00000000000..3a2d8e7aad4 --- /dev/null +++ b/pkg/modules/dashboard/impldashboard/fs/definitions/ai-o11y-overview.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "definition": { + "schemaVersion": "v6", + "name": "signoz---ai-o11y-overview", + "tags": [], + "spec": { + "display": { + "name": "AI Observability Overview", + "description": "Overview of LLM traffic. Panels ship in an upcoming release." + }, + "variables": [], + "panels": {}, + "layouts": [] + } + } +} diff --git a/pkg/modules/dashboard/impldashboard/module.go b/pkg/modules/dashboard/impldashboard/module.go index 2e29754e0ab..3b322026709 100644 --- a/pkg/modules/dashboard/impldashboard/module.go +++ b/pkg/modules/dashboard/impldashboard/module.go @@ -21,23 +21,25 @@ import ( ) type module struct { - store dashboardtypes.Store - settings factory.ScopedProviderSettings - analytics analytics.Analytics - orgGetter organization.Getter - queryParser queryparser.QueryParser - tagModule tag.Module + store dashboardtypes.Store + settings factory.ScopedProviderSettings + analytics analytics.Analytics + orgGetter organization.Getter + queryParser queryparser.QueryParser + tagModule tag.Module + systemDashboardRegistry dashboardtypes.SystemDashboardRegistry } -func NewModule(store dashboardtypes.Store, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, tagModule tag.Module) dashboard.Module { +func NewModule(store dashboardtypes.Store, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, tagModule tag.Module, systemDashboardRegistry dashboardtypes.SystemDashboardRegistry) dashboard.Module { scopedProviderSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard") return &module{ - store: store, - settings: scopedProviderSettings, - analytics: analytics, - orgGetter: orgGetter, - queryParser: queryParser, - tagModule: tagModule, + store: store, + settings: scopedProviderSettings, + analytics: analytics, + orgGetter: orgGetter, + queryParser: queryParser, + tagModule: tagModule, + systemDashboardRegistry: systemDashboardRegistry, } } diff --git a/pkg/modules/dashboard/impldashboard/store.go b/pkg/modules/dashboard/impldashboard/store.go index 5a7391fc6b6..056e0f7894a 100644 --- a/pkg/modules/dashboard/impldashboard/store.go +++ b/pkg/modules/dashboard/impldashboard/store.go @@ -3,6 +3,7 @@ package impldashboard import ( "context" "strings" + "time" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/sqlstore" @@ -64,6 +65,23 @@ func (store *store) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID) return storableDashboard, nil } +func (store *store) GetByName(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.StorableDashboard, error) { + storableDashboard := new(dashboardtypes.StorableDashboard) + err := store. + sqlstore. + BunDB(). + NewSelect(). + Model(storableDashboard). + Where("name = ?", name). + Where("org_id = ?", orgID). + Scan(ctx) + if err != nil { + return nil, store.sqlstore.WrapNotFoundErrf(err, errors.CodeNotFound, "dashboard with name %s doesn't exist", name) + } + + return storableDashboard, nil +} + // ListForUser emits the joined dashboard ⨝ user_dashboard_preference query the // spec calls for. Aliases: // @@ -613,3 +631,60 @@ func (store *store) DeleteDashboardView(ctx context.Context, orgID valuer.UUID, } return nil } + +func (store *store) CreateSystemDashboard(ctx context.Context, storable *dashboardtypes.StorableSystemDashboard) error { + _, err := store. + sqlstore. + BunDBCtx(ctx). + NewInsert(). + Model(storable). + Exec(ctx) + if err != nil { + return store.sqlstore.WrapAlreadyExistsErrf(err, dashboardtypes.ErrCodeSystemDashboardAlreadyProvisioned, "system dashboard %s is already provisioned", storable.Name) + } + + return nil +} + +func (store *store) GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.StorableSystemDashboard, error) { + storable := new(dashboardtypes.StorableSystemDashboard) + err := store. + sqlstore. + BunDBCtx(ctx). + NewSelect(). + Model(storable). + Where("org_id = ?", orgID). + Where("name = ?", name). + Scan(ctx) + if err != nil { + return nil, store.sqlstore.WrapNotFoundErrf(err, dashboardtypes.ErrCodeSystemDashboardNotFound, "system dashboard %s is not provisioned", name) + } + + return storable, nil +} + +func (store *store) UpdateSystemDashboardVersion(ctx context.Context, orgID valuer.UUID, name string, version int) error { + result, err := store. + sqlstore. + BunDBCtx(ctx). + NewUpdate(). + Model(new(dashboardtypes.StorableSystemDashboard)). + Set("version = ?", version). + Set("updated_at = ?", time.Now()). + Where("org_id = ?", orgID). + Where("name = ?", name). + Exec(ctx) + if err != nil { + return err + } + + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return errors.Newf(errors.TypeNotFound, dashboardtypes.ErrCodeSystemDashboardNotFound, "system dashboard %s is not provisioned", name) + } + + return nil +} diff --git a/pkg/modules/dashboard/impldashboard/system_dashboard_definitions.go b/pkg/modules/dashboard/impldashboard/system_dashboard_definitions.go new file mode 100644 index 00000000000..216732577ca --- /dev/null +++ b/pkg/modules/dashboard/impldashboard/system_dashboard_definitions.go @@ -0,0 +1,46 @@ +package impldashboard + +import ( + "embed" + "io/fs" + "path" + + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" +) + +const definitionsRoot = "fs/definitions" + +//go:embed fs/definitions/*.json +var definitionFiles embed.FS + +// NewSystemDashboardRegistry parses every embedded definition. Definitions are +// build-time assets validated by a test, so a failure here means the binary +// shipped broken JSON. +func NewSystemDashboardRegistry() (dashboardtypes.SystemDashboardRegistry, error) { + entries, err := fs.ReadDir(definitionFiles, definitionsRoot) + if err != nil { + return dashboardtypes.SystemDashboardRegistry{}, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read system dashboard definitions") + } + + definitions := make([]dashboardtypes.SystemDashboardDefinition, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + + file := path.Join(definitionsRoot, entry.Name()) + raw, err := definitionFiles.ReadFile(file) + if err != nil { + return dashboardtypes.SystemDashboardRegistry{}, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read %s", file) + } + + definition, err := dashboardtypes.NewSystemDashboardDefinition(raw) + if err != nil { + return dashboardtypes.SystemDashboardRegistry{}, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "couldn't parse %s", file) + } + definitions = append(definitions, definition) + } + + return dashboardtypes.NewSystemDashboardRegistry(definitions) +} diff --git a/pkg/modules/dashboard/impldashboard/system_dashboard_service.go b/pkg/modules/dashboard/impldashboard/system_dashboard_service.go new file mode 100644 index 00000000000..9ea0df23262 --- /dev/null +++ b/pkg/modules/dashboard/impldashboard/system_dashboard_service.go @@ -0,0 +1,81 @@ +package impldashboard + +import ( + "context" + "log/slog" + "time" + + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/modules/dashboard" + "github.com/SigNoz/signoz/pkg/modules/organization" +) + +const reconcileRetryInterval = 30 * time.Second + +type service struct { + settings factory.ScopedProviderSettings + module dashboard.Module + orgGetter organization.Getter + stopC chan struct{} + healthyC chan struct{} +} + +// NewService reconciles every org's system dashboards once at startup. Orgs +// created later are reconciled by the organization setter instead. +func NewService(providerSettings factory.ProviderSettings, module dashboard.Module, orgGetter organization.Getter) factory.Service { + return &service{ + settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"), + module: module, + orgGetter: orgGetter, + stopC: make(chan struct{}), + healthyC: make(chan struct{}), + } +} + +func (service *service) Start(ctx context.Context) error { + ticker := time.NewTicker(reconcileRetryInterval) + defer ticker.Stop() + + for { + err := service.reconcile(ctx) + if err == nil { + close(service.healthyC) + <-service.stopC + return nil + } + + service.settings.Logger().WarnContext(ctx, "system dashboard reconciliation failed, retrying", errors.Attr(err)) + + select { + case <-service.stopC: + return nil + case <-ticker.C: + } + } +} + +func (service *service) Healthy() <-chan struct{} { + return service.healthyC +} + +func (service *service) Stop(_ context.Context) error { + close(service.stopC) + return nil +} + +func (service *service) reconcile(ctx context.Context) error { + orgs, err := service.orgGetter.ListByOwnedKeyRange(ctx) + if err != nil { + return err + } + + for _, org := range orgs { + if err := service.module.ReconcileSystemDashboards(ctx, org.ID); err != nil { + return errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "couldn't reconcile system dashboards for org %s", org.ID.StringValue()) + } + } + + service.settings.Logger().InfoContext(ctx, "system dashboard reconciliation completed", slog.Int("orgs", len(orgs))) + return nil +} diff --git a/pkg/modules/dashboard/impldashboard/v2_handler.go b/pkg/modules/dashboard/impldashboard/v2_handler.go index 5d99ac3d848..83034daa970 100644 --- a/pkg/modules/dashboard/impldashboard/v2_handler.go +++ b/pkg/modules/dashboard/impldashboard/v2_handler.go @@ -502,3 +502,28 @@ func (handler *handler) GetPublicWidgetQueryRangeV2(rw http.ResponseWriter, r *h render.Success(rw, http.StatusOK, queryRangeResults) } + +func (handler *handler) GetSystemDashboard(rw http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + render.Error(rw, err) + return + } + + name := mux.Vars(r)["name"] + if name == "" { + render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "name is missing in the path")) + return + } + + systemDashboard, err := handler.module.GetSystemDashboard(ctx, valuer.MustNewUUID(claims.OrgID), name) + if err != nil { + render.Error(rw, err) + return + } + + render.Success(rw, http.StatusOK, systemDashboard.ToGettableSystemDashboard()) +} diff --git a/pkg/modules/dashboard/impldashboard/v2_module.go b/pkg/modules/dashboard/impldashboard/v2_module.go index 95d3e97ce6f..74b43be5147 100644 --- a/pkg/modules/dashboard/impldashboard/v2_module.go +++ b/pkg/modules/dashboard/impldashboard/v2_module.go @@ -2,6 +2,8 @@ package impldashboard import ( "context" + "log/slog" + "strings" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/transition" @@ -19,9 +21,12 @@ func (m *module) CreateV2(ctx context.Context, orgID valuer.UUID, createdBy stri return nil, err } - dashboard := postable.NewDashboardV2(orgID, createdBy, source) + dashboard, err := postable.NewDashboardV2(orgID, createdBy, source) + if err != nil { + return nil, err + } - err := m.store.RunInTx(ctx, func(ctx context.Context) error { + err = m.store.RunInTx(ctx, func(ctx context.Context) error { resolvedTags, err := m.tagModule.SyncTags(ctx, orgID, coretypes.KindDashboard, dashboard.ID, postable.Tags) if err != nil { return err @@ -120,6 +125,20 @@ func (module *module) GetV2(ctx context.Context, orgID valuer.UUID, id valuer.UU return storable.ToDashboardV2(tags) } +func (module *module) getByNameV2(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.DashboardV2, error) { + storable, err := module.store.GetByName(ctx, orgID, name) + if err != nil { + return nil, err + } + + tags, err := module.tagModule.ListForResource(ctx, orgID, coretypes.KindDashboard, storable.ID) + if err != nil { + return nil, err + } + + return storable.ToDashboardV2(tags) +} + // MigrateV2 retries the v1→v2 migration on a dashboard still stored as v1 (one the // bulk 103 migration skipped or failed). Idempotent: an already-v2 one is unchanged. func (module *module) MigrateV2(ctx context.Context, orgID valuer.UUID, id valuer.UUID) (*dashboardtypes.DashboardV2, error) { @@ -179,13 +198,33 @@ func (module *module) UpdateV2(ctx context.Context, orgID valuer.UUID, id valuer return nil, err } - err = module.store.RunInTx(ctx, func(ctx context.Context) error { - resolvedTags, err := module.tagModule.SyncTags(ctx, orgID, coretypes.KindDashboard, id, updatable.Tags) + return module.updateV2(ctx, orgID, existing, updatedBy, updatable, existing.Update) +} + +// updateUnsafeV2 updates a dashboard bypassing the guards. Intended for internal system callers. +func (module *module) updateUnsafeV2(ctx context.Context, orgID valuer.UUID, id valuer.UUID, updatedBy string, updatable dashboardtypes.UpdatableDashboardV2) (*dashboardtypes.DashboardV2, error) { + if err := updatable.Validate(); err != nil { + return nil, err + } + + existing, err := module.GetV2(ctx, orgID, id) + if err != nil { + return nil, err + } + + return module.updateV2(ctx, orgID, existing, updatedBy, updatable, existing.UpdateUnsafe) +} + +// apply is existing.Update or existing.UpdateUnsafe, so the gated path keeps its +// in-transaction checks and only updateUnsafeV2 skips them. +func (module *module) updateV2(ctx context.Context, orgID valuer.UUID, existing *dashboardtypes.DashboardV2, updatedBy string, updatable dashboardtypes.UpdatableDashboardV2, apply func(dashboardtypes.UpdatableDashboardV2, string, []*tagtypes.Tag) error) (*dashboardtypes.DashboardV2, error) { + err := module.store.RunInTx(ctx, func(ctx context.Context) error { + resolvedTags, err := module.tagModule.SyncTags(ctx, orgID, coretypes.KindDashboard, existing.ID, updatable.Tags) if err != nil { return err } - err = existing.Update(updatable, updatedBy, resolvedTags) + err = apply(updatable, updatedBy, resolvedTags) if err != nil { return err } @@ -296,3 +335,98 @@ func (module *module) UnpinV2(ctx context.Context, orgID valuer.UUID, userID val func (module *module) DeletePreferencesForUser(ctx context.Context, orgID valuer.UUID, userID valuer.UUID) error { return module.store.DeletePreferencesForUser(ctx, orgID, userID) } + +func (m *module) ReconcileSystemDashboards(ctx context.Context, orgID valuer.UUID) error { + for _, definition := range m.systemDashboardRegistry.List() { + if err := m.reconcileSystemDashboard(ctx, orgID, definition); err != nil { + return err + } + } + + return nil +} + +func (m *module) reconcileSystemDashboard(ctx context.Context, orgID valuer.UUID, definition dashboardtypes.SystemDashboardDefinition) error { + existing, err := m.getByNameV2(ctx, orgID, definition.Name()) + if err != nil { + if !errors.Ast(err, errors.TypeNotFound) { + return err + } + return m.provisionSystemDashboard(ctx, orgID, definition) + } + + state, err := m.store.GetSystemDashboard(ctx, orgID, definition.Name()) + if err != nil { + return err + } + // Only ever move forward: a downgrade must not rewrite the newer content. + if state.Version >= definition.Version { + return nil + } + + return m.upgradeSystemDashboard(ctx, orgID, existing.ID, definition) +} + +// provisionSystemDashboard creates the dashboard and its state row in one transaction, +// so a system dashboard can never exist without the version it was provisioned at. +// A concurrent provisioner (another replica, or the org-creation hook racing the +// startup sweep) loses on the state row's unique (org_id, name) index and rolls back. +func (m *module) provisionSystemDashboard(ctx context.Context, orgID valuer.UUID, definition dashboardtypes.SystemDashboardDefinition) error { + err := m.store.RunInTx(ctx, func(ctx context.Context) error { + created, err := m.CreateV2( + ctx, + orgID, + dashboardtypes.ProvisionerIdentity, + valuer.UUID{}, + dashboardtypes.SourceSystem, + definition.Dashboard, + ) + if err != nil { + return err + } + + return m.store.CreateSystemDashboard(ctx, dashboardtypes.NewStorableSystemDashboard(orgID, created.ID, definition.Name(), definition.Version)) + }) + if err != nil { + if errors.Ast(err, errors.TypeAlreadyExists) { + m.settings.Logger().DebugContext(ctx, "system dashboard already provisioned concurrently", slog.String("name", definition.Name()), slog.String("org_id", orgID.StringValue())) + return nil + } + return err + } + + m.settings.Logger().InfoContext(ctx, "provisioned system dashboard", slog.String("name", definition.Name()), slog.Int("version", definition.Version), slog.String("org_id", orgID.StringValue())) + return nil +} + +func (m *module) upgradeSystemDashboard(ctx context.Context, orgID valuer.UUID, id valuer.UUID, definition dashboardtypes.SystemDashboardDefinition) error { + err := m.store.RunInTx(ctx, func(ctx context.Context) error { + if _, err := m.updateUnsafeV2(ctx, orgID, id, dashboardtypes.ProvisionerIdentity, definition.ToUpdatable()); err != nil { + return err + } + + return m.store.UpdateSystemDashboardVersion(ctx, orgID, definition.Name(), definition.Version) + }) + if err != nil { + return err + } + + m.settings.Logger().InfoContext(ctx, "upgraded system dashboard", slog.String("name", definition.Name()), slog.Int("version", definition.Version), slog.String("org_id", orgID.StringValue())) + return nil +} + +func (m *module) GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.DashboardV2, error) { + if strings.HasPrefix(name, dashboardtypes.SystemDashboardNamePrefix) { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "name must not carry the %q prefix", dashboardtypes.SystemDashboardNamePrefix) + } + + existing, err := m.getByNameV2(ctx, orgID, dashboardtypes.SystemDashboardNamePrefix+name) + if err != nil { + return nil, err + } + if err := existing.ErrIfNotSystem(); err != nil { + return nil, err + } + + return existing, nil +} diff --git a/pkg/modules/dashboard/impldashboard/v2_module_test.go b/pkg/modules/dashboard/impldashboard/v2_module_test.go new file mode 100644 index 00000000000..1c020dd2baa --- /dev/null +++ b/pkg/modules/dashboard/impldashboard/v2_module_test.go @@ -0,0 +1,191 @@ +package impldashboard + +import ( + "context" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/SigNoz/signoz/pkg/analytics/analyticstest" + "github.com/SigNoz/signoz/pkg/factory/factorytest" + "github.com/SigNoz/signoz/pkg/modules/tag/impltag" + "github.com/SigNoz/signoz/pkg/queryparser" + "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" + "github.com/SigNoz/signoz/pkg/types/tagtypes" + "github.com/SigNoz/signoz/pkg/valuer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testDashboardName = "test-overview" + +func newTestSQLStore(t *testing.T) sqlstore.SQLStore { + t.Helper() + + store, err := sqlitesqlstore.New(context.Background(), factorytest.NewSettings(), sqlstore.Config{ + Provider: "sqlite", + Connection: sqlstore.ConnectionConfig{MaxOpenConns: 10}, + Sqlite: sqlstore.SqliteConfig{ + Path: filepath.Join(t.TempDir(), "test.db"), + Mode: "wal", + BusyTimeout: 5 * time.Second, + TransactionMode: "deferred", + }, + }) + require.NoError(t, err) + + for _, model := range []any{ + (*dashboardtypes.StorableDashboard)(nil), + (*tagtypes.Tag)(nil), + (*tagtypes.TagRelation)(nil), + (*dashboardtypes.StorableSystemDashboard)(nil), + } { + _, err := store.BunDB().NewCreateTable().Model(model).IfNotExists().Exec(context.Background()) + require.NoError(t, err) + } + + _, err = store.BunDB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_system_dashboard_org_name ON system_dashboard (org_id, name)`) + require.NoError(t, err) + + return store +} + +func newTestModule(t *testing.T, sqlStore sqlstore.SQLStore, definitions ...dashboardtypes.SystemDashboardDefinition) *module { + t.Helper() + + registry, err := dashboardtypes.NewSystemDashboardRegistry(definitions) + require.NoError(t, err) + + providerSettings := factorytest.NewSettings() + return NewModule( + NewStore(sqlStore), + providerSettings, + analyticstest.New(), + nil, + queryparser.New(providerSettings), + impltag.NewModule(impltag.NewStore(sqlStore)), + registry, + ).(*module) +} + +func newTestDefinition(t *testing.T, version int, displayName string) dashboardtypes.SystemDashboardDefinition { + t.Helper() + + raw := `{ + "version": ` + strconv.Itoa(version) + `, + "definition": { + "schemaVersion": "` + dashboardtypes.SchemaVersion + `", + "name": "` + dashboardtypes.SystemDashboardNamePrefix + testDashboardName + `", + "tags": [], + "spec": {"display": {"name": "` + displayName + `"}, "variables": [], "panels": {}, "layouts": []} + } + }` + + definition, err := dashboardtypes.NewSystemDashboardDefinition([]byte(raw)) + require.NoError(t, err) + + return definition +} + +func TestReconcileProvisionsThenUpgrades(t *testing.T) { + ctx := context.Background() + sqlStore := newTestSQLStore(t) + orgID := valuer.GenerateUUID() + + dashboardModule := newTestModule(t, sqlStore, newTestDefinition(t, 1, "v1")) + require.NoError(t, dashboardModule.ReconcileSystemDashboards(ctx, orgID)) + + provisioned, err := dashboardModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + assert.Equal(t, dashboardtypes.SourceSystem, provisioned.Source) + assert.Equal(t, dashboardtypes.ProvisionerIdentity, provisioned.CreatedBy) + assert.Equal(t, "v1", provisioned.Spec.Display.Name) + assert.Equal(t, 1, stateVersion(t, dashboardModule, ctx, orgID)) + + // Reconciling the same version again is a no-op. + require.NoError(t, dashboardModule.ReconcileSystemDashboards(ctx, orgID)) + unchanged, err := dashboardModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + assert.Equal(t, provisioned.UpdatedAt, unchanged.UpdatedAt) + + // An unmodified copy is upgraded in place, keeping its id. + upgradingModule := newTestModule(t, sqlStore, newTestDefinition(t, 2, "v2")) + require.NoError(t, upgradingModule.ReconcileSystemDashboards(ctx, orgID)) + + upgraded, err := upgradingModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + assert.Equal(t, provisioned.ID, upgraded.ID) + assert.Equal(t, "v2", upgraded.Spec.Display.Name) + assert.Equal(t, 2, stateVersion(t, upgradingModule, ctx, orgID)) +} + +func stateVersion(t *testing.T, module *module, ctx context.Context, orgID valuer.UUID) int { + t.Helper() + + state, err := module.store.GetSystemDashboard(ctx, orgID, dashboardtypes.SystemDashboardNamePrefix+testDashboardName) + require.NoError(t, err) + + return state.Version +} + +func TestSystemDashboardsAreImmutableToUsers(t *testing.T) { + ctx := context.Background() + sqlStore := newTestSQLStore(t) + orgID := valuer.GenerateUUID() + + dashboardModule := newTestModule(t, sqlStore, newTestDefinition(t, 1, "v1")) + require.NoError(t, dashboardModule.ReconcileSystemDashboards(ctx, orgID)) + + provisioned, err := dashboardModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + + _, err = dashboardModule.UpdateV2(ctx, orgID, provisioned.ID, "user@signoz.io", newTestDefinition(t, 1, "edited").ToUpdatable()) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be modified") +} + +func TestReconcileDoesNotDowngrade(t *testing.T) { + ctx := context.Background() + sqlStore := newTestSQLStore(t) + orgID := valuer.GenerateUUID() + + newerModule := newTestModule(t, sqlStore, newTestDefinition(t, 3, "v3")) + require.NoError(t, newerModule.ReconcileSystemDashboards(ctx, orgID)) + + olderModule := newTestModule(t, sqlStore, newTestDefinition(t, 2, "v2")) + require.NoError(t, olderModule.ReconcileSystemDashboards(ctx, orgID)) + + got, err := newerModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + assert.Equal(t, "v3", got.Spec.Display.Name) + assert.Equal(t, 3, stateVersion(t, newerModule, ctx, orgID)) +} + +func TestGetRejectsANonSystemDashboard(t *testing.T) { + ctx := context.Background() + sqlStore := newTestSQLStore(t) + orgID := valuer.GenerateUUID() + + dashboardModule := newTestModule(t, sqlStore) + + var postable dashboardtypes.PostableDashboardV2 + require.NoError(t, postable.UnmarshalJSON([]byte(`{ + "schemaVersion": "`+dashboardtypes.SchemaVersion+`", + "name": "a-user-dashboard", + "tags": [], + "spec": {"display": {"name": "user"}, "variables": [], "panels": {}, "layouts": []} + }`))) + _, err := dashboardModule.CreateV2(ctx, orgID, "user@signoz.io", valuer.GenerateUUID(), dashboardtypes.SourceUser, postable) + require.NoError(t, err) + + // The server-side prefix makes user names structurally unreachable here. + _, err = dashboardModule.GetSystemDashboard(ctx, orgID, "a-user-dashboard") + require.Error(t, err) + + _, err = dashboardModule.GetSystemDashboard(ctx, orgID, dashboardtypes.SystemDashboardNamePrefix+testDashboardName) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not carry") +} diff --git a/pkg/modules/organization/implorganization/setter.go b/pkg/modules/organization/implorganization/setter.go index 33e24c1d4c1..56e7805021f 100644 --- a/pkg/modules/organization/implorganization/setter.go +++ b/pkg/modules/organization/implorganization/setter.go @@ -4,6 +4,7 @@ import ( "context" "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/modules/dashboard" "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/modules/quickfilter" "github.com/SigNoz/signoz/pkg/types" @@ -14,10 +15,11 @@ type setter struct { store types.OrganizationStore alertmanager alertmanager.Alertmanager quickfilter quickfilter.Module + dashboard dashboard.Module } -func NewSetter(store types.OrganizationStore, alertmanager alertmanager.Alertmanager, quickfilter quickfilter.Module) organization.Setter { - return &setter{store: store, alertmanager: alertmanager, quickfilter: quickfilter} +func NewSetter(store types.OrganizationStore, alertmanager alertmanager.Alertmanager, quickfilter quickfilter.Module, dashboard dashboard.Module) organization.Setter { + return &setter{store: store, alertmanager: alertmanager, quickfilter: quickfilter, dashboard: dashboard} } func (module *setter) Create(ctx context.Context, organization *types.Organization, createManagedRoles func(context.Context, valuer.UUID) error) error { @@ -37,6 +39,10 @@ func (module *setter) Create(ctx context.Context, organization *types.Organizati return err } + if err := module.dashboard.ReconcileSystemDashboards(ctx, organization.ID); err != nil { + return err + } + return nil } diff --git a/pkg/querier/consume.go b/pkg/querier/consume.go index 34ee097ca1a..99935b9cf84 100644 --- a/pkg/querier/consume.go +++ b/pkg/querier/consume.go @@ -1,6 +1,7 @@ package querier import ( + "encoding/json" "fmt" "math" "reflect" @@ -11,12 +12,12 @@ import ( "strings" "time" + "github.com/ClickHouse/clickhouse-go/v2/lib/chcol" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" - "github.com/SigNoz/signoz/pkg/errors" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/spantypes" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" - "github.com/bytedance/sonic" ) var ( @@ -30,8 +31,6 @@ var ( // written clickhouse query. The column alias indcate which value is // to be considered as final result (or target). legacyReservedColumnTargetAliases = []string{"__result", "__value", "result", "res", "value"} - - CodeFailUnmarshalJSONColumn = errors.MustNewCode("fail_unmarshal_json_column") ) // stripKeyAlias removes the __SELECT_KEY__ / __GROUP_BY_KEY__ prefix from a result @@ -40,6 +39,32 @@ func stripKeyAlias(name string) string { return keyAliasRe.ReplaceAllString(name, "") } +// unwrapVariant returns the concrete value inside the chcol.Variant envelope the driver scans a +// Dynamic column — a JSON path such as body_v2.level — into. +func unwrapVariant(val any) any { + if v, ok := val.(chcol.Variant); ok { + return v.Any() + } + return val +} + +// labelValue renders a group-by value the payload cannot carry as a scalar — a JSON column, or a +// Dynamic one — as a stable string, so that rows differing only in that value land in different +// series. JSON goes through encoding/json for its sorted map keys: ClickHouse groups documents by +// structure, so two rows it considers equal have to produce the same label. +func labelValue(val any) string { + val = unwrapVariant(val) + if val == nil { + return "" + } + if v, ok := val.(telemetrystoretypes.JSONValue); ok { + if raw, err := json.Marshal(v); err == nil { + return string(raw) + } + } + return fmt.Sprint(val) +} + // consume reads every row and shapes it into the payload expected for the // given request type. // @@ -205,6 +230,14 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt Value: *val, }) + case *telemetrystoretypes.JSONValue, *chcol.Variant: + val := labelValue(derefValue(ptr)) + lblVals = append(lblVals, val) + lblObjs = append(lblObjs, &qbtypes.Label{ + Key: telemetrytypes.TelemetryFieldKey{Name: name}, + Value: val, + }) + default: continue } @@ -345,7 +378,7 @@ func readAsScalar(rows driver.Rows, queryName string) (*qbtypes.ScalarData, erro // 2. deref each slot into the output row row := make([]any, len(scan)) for i, cell := range scan { - row[i] = derefValue(cell) + row[i] = unwrapVariant(derefValue(cell)) } data = append(data, row) } @@ -382,31 +415,13 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) { colTypes := rows.ColumnTypes() colCnt := len(colNames) - // Helper that decides scan target per column based on DB type - makeScanTarget := func(i int) any { - dbt := strings.ToUpper(colTypes[i].DatabaseTypeName()) - if strings.HasPrefix(dbt, "JSON") { - // Since the driver fails to decode JSON/Dynamic into native Go values, we read it as raw bytes - // TODO: check in future if fixed in the driver - var v []byte - return &v - } - return reflect.New(colTypes[i].ScanType()).Interface() - } - - // Build a template slice of correctly-typed pointers once - scanTpl := make([]any, colCnt) - for i := range colTypes { - scanTpl[i] = makeScanTarget(i) - } - var outRows []*qbtypes.RawRow for rows.Next() { // fresh copy of the scan slice (otherwise the driver reuses pointers) scan := make([]any, colCnt) - for i := range scanTpl { - scan[i] = makeScanTarget(i) + for i := range colTypes { + scan[i] = reflect.New(colTypes[i].ScanType()).Interface() } if err := rows.Scan(scan...); err != nil { @@ -421,21 +436,7 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) { name := stripKeyAlias(colNames[i]) // de-reference the typed pointer to any - val := reflect.ValueOf(cellPtr).Elem().Interface() - // Post-process JSON columns: unmarshal bytes into map[string]any - if strings.HasPrefix(strings.ToUpper(colTypes[i].DatabaseTypeName()), "JSON") { - switch x := val.(type) { - case []byte: - var m map[string]any - err := sonic.Unmarshal(x, &m) - if err != nil { - return nil, errors.WrapInternalf(err, CodeFailUnmarshalJSONColumn, "failed to unmarshal JSON column %s", name) - } - val = m - default: - // already a structured type (map[string]any, []any, etc.) - } - } + val := unwrapVariant(reflect.ValueOf(cellPtr).Elem().Interface()) // special-case: timestamp column if name == "timestamp" || name == "timestamp_datetime" { diff --git a/pkg/querier/consume_test.go b/pkg/querier/consume_test.go index 9fb5ab6132f..17438a24933 100644 --- a/pkg/querier/consume_test.go +++ b/pkg/querier/consume_test.go @@ -3,8 +3,16 @@ package querier import ( "reflect" "testing" + "time" + "github.com/ClickHouse/clickhouse-go/v2/lib/chcol" + cmock "github.com/SigNoz/clickhouse-go-mock" + "github.com/SigNoz/signoz/pkg/telemetrystore" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/spantypes" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMergeSpanAttributeColumns_ParsesEventsAndLinks(t *testing.T) { @@ -75,6 +83,103 @@ func TestMergeSpanAttributeColumns_ParsesEventsAndLinks(t *testing.T) { } } +// A ClickHouse query can put a JSON column in the result of any request type — e.g. +// `select * from signoz_logs.logs_v2` on a body_v2 stack, where `*` covers body_v2. +func TestConsume_JSONColumn(t *testing.T) { + ts := time.Date(2026, 8, 14, 10, 0, 0, 0, time.UTC) + body := `{"level":"error","attrs":{"code":500}}` + wantBody := telemetrystoretypes.JSONValue{ + "level": "error", + "attrs": map[string]any{"code": float64(500)}, + } + + // the scalar reader reuses its scan slots across rows, so each row must still carry its own body + t.Run("scalar", func(t *testing.T) { + rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{ + {Name: "body_v2", Type: "JSON"}, + {Name: "__result_0", Type: "UInt64"}, + }, [][]any{{body, uint64(3)}, {`{"level":"warn"}`, uint64(1)}})) + + payload, err := consume(rows, qbtypes.RequestTypeScalar, nil, qbtypes.Step{}, "A") + require.NoError(t, err) + + data := payload.(*qbtypes.ScalarData) + require.Len(t, data.Data, 2) + assert.Equal(t, wantBody, data.Data[0][0]) + assert.Equal(t, uint64(3), data.Data[0][1]) + assert.Equal(t, telemetrystoretypes.JSONValue{"level": "warn"}, data.Data[1][0]) + assert.Equal(t, uint64(1), data.Data[1][1]) + }) + + t.Run("time series", func(t *testing.T) { + rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{ + {Name: "ts", Type: "DateTime"}, + {Name: "body_v2", Type: "JSON"}, + {Name: "__result_0", Type: "UInt64"}, + }, [][]any{{ts, body, uint64(3)}})) + + payload, err := consume(rows, qbtypes.RequestTypeTimeSeries, nil, qbtypes.Step{}, "A") + require.NoError(t, err) + + data := payload.(*qbtypes.TimeSeriesData) + require.Len(t, data.Aggregations, 1) + require.Len(t, data.Aggregations[0].Series, 1) + require.Len(t, data.Aggregations[0].Series[0].Values, 1) + assert.Equal(t, float64(3), data.Aggregations[0].Series[0].Values[0].Value) + }) + + // grouping by a JSON column is legal in ClickHouse, so each document has to label its own + // series rather than being dropped, which would merge every group into one + t.Run("time series grouped by the JSON column", func(t *testing.T) { + rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{ + {Name: "ts", Type: "DateTime"}, + {Name: "body_v2", Type: "JSON"}, + {Name: "__result_0", Type: "UInt64"}, + }, [][]any{ + {ts, `{"level":"error"}`, uint64(7)}, + {ts, `{"level":"warn"}`, uint64(2)}, + })) + + payload, err := consume(rows, qbtypes.RequestTypeTimeSeries, nil, qbtypes.Step{}, "A") + require.NoError(t, err) + + data := payload.(*qbtypes.TimeSeriesData) + require.Len(t, data.Aggregations, 1) + require.Len(t, data.Aggregations[0].Series, 2) + + got := map[string]float64{} + for _, series := range data.Aggregations[0].Series { + require.Len(t, series.Labels, 1) + require.Len(t, series.Values, 1) + got[series.Labels[0].Value.(string)] = series.Values[0].Value + } + assert.Equal(t, map[string]float64{`{"level":"error"}`: 7, `{"level":"warn"}`: 2}, got) + }) + + t.Run("raw", func(t *testing.T) { + rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{ + {Name: "timestamp", Type: "DateTime"}, + {Name: "body_v2", Type: "JSON"}, + }, [][]any{{ts, body}})) + + payload, err := consume(rows, qbtypes.RequestTypeRaw, nil, qbtypes.Step{}, "A") + require.NoError(t, err) + + data := payload.(*qbtypes.RawData) + require.Len(t, data.Rows, 1) + assert.Equal(t, ts, data.Rows[0].Timestamp.UTC()) + assert.Equal(t, wantBody, data.Rows[0].Data["body_v2"]) + }) +} + +// A JSON path (e.g. `body_v2.level`) comes back as a Dynamic column, which the driver scans +// into a chcol.Variant envelope rather than the value itself. +func TestUnwrapVariant(t *testing.T) { + assert.Equal(t, "error", unwrapVariant(chcol.NewDynamicWithType("error", "String"))) + assert.Nil(t, unwrapVariant(chcol.Dynamic{})) + assert.Equal(t, uint64(3), unwrapVariant(uint64(3))) +} + func TestMergeSpanAttributeColumns_EmptyEventsAndLinks(t *testing.T) { data := map[string]any{ "events": []string{}, diff --git a/pkg/querier/postprocess.go b/pkg/querier/postprocess.go index 954ef89d0f9..dd5c1ee7118 100644 --- a/pkg/querier/postprocess.go +++ b/pkg/querier/postprocess.go @@ -13,9 +13,10 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/flagger" - "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/querybuilder" + "github.com/SigNoz/signoz/pkg/types/featuretypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/valuer" ) @@ -72,9 +73,13 @@ func (q *querier) postProcessResults(ctx context.Context, orgID valuer.UUID, res case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]: if result, ok := typedResults[spec.Name]; ok { result = postProcessBuilderQuery(q, result, spec, req) - result = q.postProcessLogBody(ctx, orgID, result, req) + result = q.postProcessLogBody(ctx, orgID, result) typedResults[spec.Name] = result } + case qbtypes.ClickHouseQuery: + if result, ok := typedResults[spec.Name]; ok { + typedResults[spec.Name] = q.postProcessLogBody(ctx, orgID, result) + } case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]: if result, ok := typedResults[spec.Name]; ok { result = postProcessMetricQuery(q, result, spec, req) @@ -1051,32 +1056,44 @@ func (q *querier) calculateFormulaStep(expression string, req *qbtypes.QueryRang return result } -// postProcessLogBody removes the "message" key from the body map when it is empty. -// Only runs for raw list queries with the use_json_body feature enabled. -func (q *querier) postProcessLogBody(ctx context.Context, orgID valuer.UUID, result *qbtypes.Result, req *qbtypes.QueryRangeRequest) *qbtypes.Result { - if req.RequestType != qbtypes.RequestTypeRaw { - return result - } +// postProcessLogBody removes the empty "message" the typed body path materializes into every +// document, wherever a decoded body lands in the payload — raw rows and scalar cells, under the +// column's own name or the builder's `body` alias. Only runs with the use_json_body feature +// enabled. A time-series label keeps the document verbatim: it is the group key. +func (q *querier) postProcessLogBody(ctx context.Context, orgID valuer.UUID, result *qbtypes.Result) *qbtypes.Result { if !q.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) { return result } - rawData, ok := result.Value.(*qbtypes.RawData) - if !ok { - return result - } - for _, row := range rawData.Rows { - bodyMap, ok := row.Data["body"].(map[string]any) - if !ok { - continue + switch data := result.Value.(type) { + case *qbtypes.RawData: + for _, row := range data.Rows { + for _, name := range []string{"body", "body_v2"} { + stripEmptyBodyMessage(row.Data[name]) + } } - if msg, exists := bodyMap["message"]; exists { - switch v := msg.(type) { - case string: - if v == "" { - delete(bodyMap, "message") - } + case *qbtypes.ScalarData: + for idx, column := range data.Columns { + if column.Name != "body" && column.Name != "body_v2" { + continue + } + for _, row := range data.Data { + stripEmptyBodyMessage(row[idx]) } } } return result } + +// stripEmptyBodyMessage drops `message: ""` from a decoded body document: the message path is +// typed String in the JSON column, so ClickHouse materializes it even for documents that never +// carried one. Anything that is not a decoded document — the legacy string body, a NULL cell — +// is legal under these names and left alone. +func stripEmptyBodyMessage(val any) { + bodyMap, ok := val.(telemetrystoretypes.JSONValue) + if !ok { + return + } + if msg, ok := bodyMap["message"].(string); ok && msg == "" { + delete(bodyMap, "message") + } +} diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index 17e40364d40..85913b87b72 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -189,6 +189,7 @@ func TestRunExecutesQueriesConcurrently(t *testing.T) { q := &querier{ logger: instrumentationtest.New().Logger(), + fl: flaggertest.New(t), maxConcurrentQueries: numQueries, } @@ -236,6 +237,7 @@ func TestRunRespectsMaxConcurrentQueries(t *testing.T) { q := &querier{ logger: instrumentationtest.New().Logger(), + fl: flaggertest.New(t), maxConcurrentQueries: limit, } @@ -273,6 +275,7 @@ func TestRunRespectsMaxConcurrentQueries(t *testing.T) { func TestRunQueryErrorCancelsSiblings(t *testing.T) { q := &querier{ logger: instrumentationtest.New().Logger(), + fl: flaggertest.New(t), maxConcurrentQueries: 4, } diff --git a/pkg/signoz/handler_test.go b/pkg/signoz/handler_test.go index 5d88a12b9e0..9b53f3dedd5 100644 --- a/pkg/signoz/handler_test.go +++ b/pkg/signoz/handler_test.go @@ -49,7 +49,9 @@ func TestNewHandlers(t *testing.T) { queryParser := queryparser.New(providerSettings) require.NoError(t, err) tagModule := impltag.NewModule(impltag.NewStore(sqlstore)) - dashboardModule := impldashboard.NewModule(impldashboard.NewStore(sqlstore), providerSettings, nil, orgGetter, queryParser, tagModule) + systemDashboardRegistry, err := impldashboard.NewSystemDashboardRegistry() + require.NoError(t, err) + dashboardModule := impldashboard.NewModule(impldashboard.NewStore(sqlstore), providerSettings, nil, orgGetter, queryParser, tagModule, systemDashboardRegistry) flagger, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry()) require.NoError(t, err) diff --git a/pkg/signoz/module.go b/pkg/signoz/module.go index 7baf3c3a1aa..eaa37a68dd9 100644 --- a/pkg/signoz/module.go +++ b/pkg/signoz/module.go @@ -67,35 +67,35 @@ import ( ) type Modules struct { - OrgGetter organization.Getter - OrgSetter organization.Setter - Preference preference.Module - UserSetter user.Setter - UserGetter user.Getter - RetentionGetter retention.Getter - SavedView savedview.Module - Apdex apdex.Module - Dashboard dashboard.Module - QuickFilter quickfilter.Module - TraceFunnel tracefunnel.Module - RawDataExport rawdataexport.Module - AuthDomain authdomain.Module - Session session.Module - Services services.Module - SpanPercentile spanpercentile.Module - MetricsExplorer metricsexplorer.Module - MetricReductionRule metricreductionrule.Module - InfraMonitoring inframonitoring.Module + OrgGetter organization.Getter + OrgSetter organization.Setter + Preference preference.Module + UserSetter user.Setter + UserGetter user.Getter + RetentionGetter retention.Getter + SavedView savedview.Module + Apdex apdex.Module + Dashboard dashboard.Module + QuickFilter quickfilter.Module + TraceFunnel tracefunnel.Module + RawDataExport rawdataexport.Module + AuthDomain authdomain.Module + Session session.Module + Services services.Module + SpanPercentile spanpercentile.Module + MetricsExplorer metricsexplorer.Module + MetricReductionRule metricreductionrule.Module + InfraMonitoring inframonitoring.Module Promote promote.Module ServiceAccount serviceaccount.Module ServiceAccountGetter serviceaccount.Getter CloudIntegration cloudintegration.Module - LogsPipeline logspipeline.Module - RuleStateHistory rulestatehistory.Module - TraceDetail tracedetail.Module - SpanMapper spanmapper.Module - LLMPricingRule llmpricingrule.Module - Tag tag.Module + LogsPipeline logspipeline.Module + RuleStateHistory rulestatehistory.Module + TraceDetail tracedetail.Module + SpanMapper spanmapper.Module + LLMPricingRule llmpricingrule.Module + Tag tag.Module } func NewModules( @@ -126,7 +126,7 @@ func NewModules( metricReductionRule metricreductionrule.Module, ) Modules { quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore)) - orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter) + orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter, dashboard) // Cleanup callbacks from other modules, invoked when a user is deleted. onDeleteUser := []user.OnDeleteUser{ dashboard.DeletePreferencesForUser, @@ -136,34 +136,34 @@ func NewModules( authDomainModule := implauthdomain.NewModule(implauthdomain.NewStore(sqlstore), authNs, authz) return Modules{ - OrgGetter: orgGetter, - OrgSetter: orgSetter, - Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewAvailablePreference()), - SavedView: implsavedview.NewModule(implsavedview.NewStore(sqlstore)), - Apdex: implapdex.NewModule(sqlstore), - Dashboard: dashboard, - UserSetter: userSetter, - UserGetter: userGetter, - RetentionGetter: retentionGetter, - QuickFilter: quickfilter, - TraceFunnel: impltracefunnel.NewModule(impltracefunnel.NewStore(sqlstore)), - RawDataExport: implrawdataexport.NewModule(querier), - AuthDomain: authDomainModule, - Session: implsession.NewModule(providerSettings, authNs, userSetter, userGetter, authDomainModule, tokenizer, orgGetter, authz, config.Global), - SpanPercentile: implspanpercentile.NewModule(querier, providerSettings), - Services: implservices.NewModule(querier, telemetryStore), - MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, fl, providerSettings, config.MetricsExplorer), - MetricReductionRule: metricReductionRule, - InfraMonitoring: implinframonitoring.NewModule(telemetryStore, telemetryMetadataStore, querier, fl, providerSettings, config.InfraMonitoring), - Promote: implpromote.NewModule(telemetryMetadataStore, telemetryStore), + OrgGetter: orgGetter, + OrgSetter: orgSetter, + Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewAvailablePreference()), + SavedView: implsavedview.NewModule(implsavedview.NewStore(sqlstore)), + Apdex: implapdex.NewModule(sqlstore), + Dashboard: dashboard, + UserSetter: userSetter, + UserGetter: userGetter, + RetentionGetter: retentionGetter, + QuickFilter: quickfilter, + TraceFunnel: impltracefunnel.NewModule(impltracefunnel.NewStore(sqlstore)), + RawDataExport: implrawdataexport.NewModule(querier), + AuthDomain: authDomainModule, + Session: implsession.NewModule(providerSettings, authNs, userSetter, userGetter, authDomainModule, tokenizer, orgGetter, authz, config.Global), + SpanPercentile: implspanpercentile.NewModule(querier, providerSettings), + Services: implservices.NewModule(querier, telemetryStore), + MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, fl, providerSettings, config.MetricsExplorer), + MetricReductionRule: metricReductionRule, + InfraMonitoring: implinframonitoring.NewModule(telemetryStore, telemetryMetadataStore, querier, fl, providerSettings, config.InfraMonitoring), + Promote: implpromote.NewModule(telemetryMetadataStore, telemetryStore), ServiceAccount: serviceAccount, ServiceAccountGetter: serviceAccountGetter, - LogsPipeline: impllogspipeline.NewModule(sqlstore), - RuleStateHistory: implrulestatehistory.NewModule(implrulestatehistory.NewStore(telemetryStore, telemetryMetadataStore, providerSettings.Logger), ruleStore), - CloudIntegration: cloudIntegrationModule, - TraceDetail: impltracedetail.NewModule(impltracedetail.NewTraceStore(telemetryStore), providerSettings, config.TraceDetail), - SpanMapper: implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), fl), - LLMPricingRule: impllmpricingrule.NewModule(impllmpricingrule.NewStore(sqlstore), fl, querier), - Tag: tagModule, + LogsPipeline: impllogspipeline.NewModule(sqlstore), + RuleStateHistory: implrulestatehistory.NewModule(implrulestatehistory.NewStore(telemetryStore, telemetryMetadataStore, providerSettings.Logger), ruleStore), + CloudIntegration: cloudIntegrationModule, + TraceDetail: impltracedetail.NewModule(impltracedetail.NewTraceStore(telemetryStore), providerSettings, config.TraceDetail), + SpanMapper: implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), fl), + LLMPricingRule: impllmpricingrule.NewModule(impllmpricingrule.NewStore(sqlstore), fl, querier), + Tag: tagModule, } } diff --git a/pkg/signoz/module_test.go b/pkg/signoz/module_test.go index 3a147716d87..82e843664c7 100644 --- a/pkg/signoz/module_test.go +++ b/pkg/signoz/module_test.go @@ -51,7 +51,9 @@ func TestNewModules(t *testing.T) { queryParser := queryparser.New(providerSettings) require.NoError(t, err) tagModule := impltag.NewModule(impltag.NewStore(sqlstore)) - dashboardModule := impldashboard.NewModule(impldashboard.NewStore(sqlstore), providerSettings, nil, orgGetter, queryParser, tagModule) + systemDashboardRegistry, err := impldashboard.NewSystemDashboardRegistry() + require.NoError(t, err) + dashboardModule := impldashboard.NewModule(impldashboard.NewStore(sqlstore), providerSettings, nil, orgGetter, queryParser, tagModule, systemDashboardRegistry) flagger, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry()) require.NoError(t, err) diff --git a/pkg/signoz/provider.go b/pkg/signoz/provider.go index e5dcbe0f093..878d1debb40 100644 --- a/pkg/signoz/provider.go +++ b/pkg/signoz/provider.go @@ -245,6 +245,7 @@ func NewSQLMigrationProviderFactories( sqlmigration.NewMigrateLambdaDashboardsFactory(), sqlmigration.NewAddAuthDomainTuplesFactory(sqlstore), sqlmigration.NewAddDeploymentHostTuplesFactory(sqlstore), + sqlmigration.NewAddSystemDashboardFactory(sqlstore, sqlschema), ) } diff --git a/pkg/signoz/signoz.go b/pkg/signoz/signoz.go index 95f4a74487b..b11e3f74d39 100644 --- a/pkg/signoz/signoz.go +++ b/pkg/signoz/signoz.go @@ -60,6 +60,7 @@ import ( "github.com/SigNoz/signoz/pkg/telemetrystore" pkgtokenizer "github.com/SigNoz/signoz/pkg/tokenizer" "github.com/SigNoz/signoz/pkg/types/authtypes" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/version" @@ -175,7 +176,7 @@ func New( telemetrystoreProviderFactories factory.NamedMap[factory.ProviderFactory[telemetrystore.TelemetryStore, telemetrystore.Config]], authNsCallback func(ctx context.Context, providerSettings factory.ProviderSettings, store authtypes.AuthNStore, licensing licensing.Licensing) (map[authtypes.AuthNProvider]authn.AuthN, error), authzCallback func(context.Context, sqlstore.SQLStore, authz.Config, licensing.Licensing, []authz.OnBeforeRoleDelete) (factory.ProviderFactory[authz.AuthZ, authz.Config], error), - dashboardModuleCallback func(sqlstore.SQLStore, factory.ProviderSettings, analytics.Analytics, organization.Getter, queryparser.QueryParser, querier.Querier, licensing.Licensing, tag.Module) dashboard.Module, + dashboardModuleCallback func(sqlstore.SQLStore, factory.ProviderSettings, analytics.Analytics, organization.Getter, queryparser.QueryParser, querier.Querier, licensing.Licensing, tag.Module, dashboardtypes.SystemDashboardRegistry) dashboard.Module, gatewayProviderFactory func(licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config], auditorProviderFactories func(licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]], meterReporterProviderFactories func(context.Context, factory.ProviderSettings, flagger.Flagger, licensing.Licensing, telemetrystore.TelemetryStore, retention.Getter, organization.Getter, zeus.Zeus) (factory.NamedMap[factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config]], string), @@ -440,8 +441,13 @@ func New( // Initialize query parser (needed for dashboard module) queryParser := queryparser.New(providerSettings) - // Initialize dashboard module - dashboard := dashboardModuleCallback(sqlstore, providerSettings, analytics, orgGetter, queryParser, querier, licensing, tagModule) + // Initialize dashboard module. The system dashboard registry is parsed here so + // a malformed embedded definition fails startup instead of a request. + systemDashboardRegistry, err := impldashboard.NewSystemDashboardRegistry() + if err != nil { + return nil, err + } + dashboard := dashboardModuleCallback(sqlstore, providerSettings, analytics, orgGetter, queryParser, querier, licensing, tagModule, systemDashboardRegistry) // Initialize user getter userGetter := impluser.NewGetter(userStore, userRoleStore, flagger) @@ -610,6 +616,7 @@ func New( factory.NewNamedService(factory.MustNewName("auditor"), auditor), factory.NewNamedService(factory.MustNewName("meterreporter"), meterReporter, factory.MustNewName("licensing")), factory.NewNamedService(factory.MustNewName("ruler"), rulerInstance), + factory.NewNamedService(factory.MustNewName("systemdashboard"), impldashboard.NewService(providerSettings, dashboard, orgGetter)), ) if err != nil { return nil, err diff --git a/pkg/sqlmigration/119_add_system_dashboard.go b/pkg/sqlmigration/119_add_system_dashboard.go new file mode 100644 index 00000000000..b59c0a5d2b8 --- /dev/null +++ b/pkg/sqlmigration/119_add_system_dashboard.go @@ -0,0 +1,93 @@ +package sqlmigration + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/sqlschema" + "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/uptrace/bun" + "github.com/uptrace/bun/migrate" +) + +type addSystemDashboard struct { + sqlstore sqlstore.SQLStore + sqlschema sqlschema.SQLSchema +} + +func NewAddSystemDashboardFactory(sqlstore sqlstore.SQLStore, sqlschema sqlschema.SQLSchema) factory.ProviderFactory[SQLMigration, Config] { + return factory.NewProviderFactory( + factory.MustNewName("add_system_dashboard"), + func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) { + return &addSystemDashboard{sqlstore: sqlstore, sqlschema: sqlschema}, nil + }, + ) +} + +func (migration *addSystemDashboard) Register(migrations *migrate.Migrations) error { + return migrations.Register(migration.Up, migration.Down) +} + +func (migration *addSystemDashboard) Up(ctx context.Context, db *bun.DB) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + sqls := migration.sqlschema.Operator().CreateTable(&sqlschema.Table{ + Name: "system_dashboard", + Columns: []*sqlschema.Column{ + {Name: "id", DataType: sqlschema.DataTypeText, Nullable: false}, + {Name: "org_id", DataType: sqlschema.DataTypeText, Nullable: false}, + {Name: "dashboard_id", DataType: sqlschema.DataTypeText, Nullable: false}, + {Name: "name", DataType: sqlschema.DataTypeText, Nullable: false}, + {Name: "version", DataType: sqlschema.DataTypeBigInt, Nullable: false}, + {Name: "created_at", DataType: sqlschema.DataTypeTimestamp, Nullable: false}, + {Name: "updated_at", DataType: sqlschema.DataTypeTimestamp, Nullable: false}, + }, + PrimaryKeyConstraint: &sqlschema.PrimaryKeyConstraint{ + ColumnNames: []sqlschema.ColumnName{"id"}, + }, + ForeignKeyConstraints: []*sqlschema.ForeignKeyConstraint{ + { + ReferencingColumnName: sqlschema.ColumnName("org_id"), + ReferencedTableName: sqlschema.TableName("organizations"), + ReferencedColumnName: sqlschema.ColumnName("id"), + }, + { + ReferencingColumnName: sqlschema.ColumnName("dashboard_id"), + ReferencedTableName: sqlschema.TableName("dashboard"), + ReferencedColumnName: sqlschema.ColumnName("id"), + }, + }, + }) + + // (org_id, name) is what makes provisioning safe across replicas: the state + // row is written in the same transaction as the dashboard, so a losing racer + // rolls back its dashboard too. + sqls = append(sqls, migration.sqlschema.Operator().CreateIndex( + &sqlschema.UniqueIndex{ + TableName: "system_dashboard", + ColumnNames: []sqlschema.ColumnName{"org_id", "name"}, + }, + )...) + sqls = append(sqls, migration.sqlschema.Operator().CreateIndex( + &sqlschema.UniqueIndex{ + TableName: "system_dashboard", + ColumnNames: []sqlschema.ColumnName{"dashboard_id"}, + }, + )...) + + for _, sql := range sqls { + if _, err := tx.ExecContext(ctx, string(sql)); err != nil { + return err + } + } + + return tx.Commit() +} + +func (migration *addSystemDashboard) Down(context.Context, *bun.DB) error { + return nil +} diff --git a/pkg/telemetrystore/clickhousetelemetrystore/provider.go b/pkg/telemetrystore/clickhousetelemetrystore/provider.go index 9eb2e1f0d95..edf34179f06 100644 --- a/pkg/telemetrystore/clickhousetelemetrystore/provider.go +++ b/pkg/telemetrystore/clickhousetelemetrystore/provider.go @@ -184,7 +184,7 @@ func (p *provider) Query(ctx context.Context, query string, args ...interface{}) } return &rowsWithHooks{ - Rows: rows, + Rows: telemetrystore.WrapRows(rows), ctx: ctx, event: event, onClose: func() { telemetrystore.WrapAfterQuery(p.hooks, ctx, event) }, diff --git a/pkg/telemetrystore/rows.go b/pkg/telemetrystore/rows.go new file mode 100644 index 00000000000..194557a908f --- /dev/null +++ b/pkg/telemetrystore/rows.go @@ -0,0 +1,39 @@ +package telemetrystore + +import ( + "reflect" + "strings" + + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" +) + +// WrapRows reports JSONValue as the scan type of every JSON column. Nested JSON — Array(JSON), +// Map(String, JSON) — is not covered. +func WrapRows(rows driver.Rows) driver.Rows { + return &rowsWithJSONScanType{Rows: rows} +} + +type rowsWithJSONScanType struct { + driver.Rows +} + +func (r *rowsWithJSONScanType) ColumnTypes() []driver.ColumnType { + colTypes := r.Rows.ColumnTypes() + wrapped := make([]driver.ColumnType, len(colTypes)) + for i, colType := range colTypes { + wrapped[i] = colType + if strings.HasPrefix(strings.ToUpper(colType.DatabaseTypeName()), "JSON") { + wrapped[i] = jsonColumnType{ColumnType: colType} + } + } + return wrapped +} + +type jsonColumnType struct { + driver.ColumnType +} + +func (jsonColumnType) ScanType() reflect.Type { + return reflect.TypeFor[telemetrystoretypes.JSONValue]() +} diff --git a/pkg/telemetrystore/telemetrystoretest/conn.go b/pkg/telemetrystore/telemetrystoretest/conn.go new file mode 100644 index 00000000000..601aef5282c --- /dev/null +++ b/pkg/telemetrystore/telemetrystoretest/conn.go @@ -0,0 +1,23 @@ +package telemetrystoretest + +import ( + "context" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/SigNoz/signoz/pkg/telemetrystore" +) + +// conn wraps rows the way the clickhouse provider does, so mocked JSON columns report the scan +// type they do in production. +type conn struct { + clickhouse.Conn +} + +func (c conn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) { + rows, err := c.Conn.Query(ctx, query, args...) + if err != nil { + return nil, err + } + return telemetrystore.WrapRows(rows), nil +} diff --git a/pkg/telemetrystore/telemetrystoretest/provider.go b/pkg/telemetrystore/telemetrystoretest/provider.go index db73cee2851..6aeb52bb91c 100644 --- a/pkg/telemetrystore/telemetrystoretest/provider.go +++ b/pkg/telemetrystore/telemetrystoretest/provider.go @@ -32,7 +32,7 @@ func New(_ telemetrystore.Config, matcher sqlmock.QueryMatcher) *Provider { // ClickhouseDB returns the mock Clickhouse connection. func (p *Provider) ClickhouseDB() clickhouse.Conn { - return p.clickhouseDB.(clickhouse.Conn) + return conn{Conn: p.clickhouseDB.(clickhouse.Conn)} } // Cluster returns the cluster name. diff --git a/pkg/types/dashboardtypes/perses_dashboard.go b/pkg/types/dashboardtypes/perses_dashboard.go index f72116b6b40..d640bdc2916 100644 --- a/pkg/types/dashboardtypes/perses_dashboard.go +++ b/pkg/types/dashboardtypes/perses_dashboard.go @@ -25,6 +25,10 @@ const ( dashboardNameSuffixLen = 8 ) +// SystemDashboardNamePrefix is reserved for dashboards SigNoz ships and owns. Generated +// names never contain consecutive hyphens, so only a typed name can carry it — create rejects that. +const SystemDashboardNamePrefix = "signoz---" + const ( dashboardIconPathPrefix = "/assets/Icons/" dashboardLogoPathPrefix = "/assets/Logos/" @@ -75,8 +79,8 @@ type DashboardV2 struct { } func (d *DashboardV2) ErrIfNotMutable() error { - if d.Source == SourceIntegration { - return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "integration dashboards cannot be modified") + if d.Source != SourceUser { + return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "%s dashboards cannot be modified", d.Source) } return nil } @@ -95,6 +99,11 @@ func (d *DashboardV2) Update(updatable UpdatableDashboardV2, updatedBy string, r if err := d.ErrIfNotUpdatable(); err != nil { return err } + return d.UpdateUnsafe(updatable, updatedBy, resolvedTags) +} + +// UpdateUnsafe applies the update without the source/lock gate. Intended for internal system callers. +func (d *DashboardV2) UpdateUnsafe(updatable UpdatableDashboardV2, updatedBy string, resolvedTags []*tagtypes.Tag) error { if updatable.Name != d.Name { return errors.NewInvalidInputf(ErrCodeDashboardImmutable, "name is immutable; cannot change from %q to %q", d.Name, updatable.Name) } @@ -129,6 +138,13 @@ func (d *DashboardV2) LockUnlock(lock bool, isAdmin bool, updatedBy string) erro return nil } +func (d *DashboardV2) ErrIfNotSystem() error { + if d.Source != SourceSystem { + return errors.Newf(errors.TypeNotFound, ErrCodeDashboardNotFound, "dashboard %q is not a system dashboard", d.Name) + } + return nil +} + func (d *DashboardV2) ErrIfNotClonable() error { if !d.Source.isClonable() { return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "%s dashboards cannot be cloned", d.Source) @@ -205,13 +221,21 @@ type PostableDashboardV2 struct { Spec DashboardSpec `json:"spec" required:"true"` } -func (postable PostableDashboardV2) NewDashboardV2(orgID valuer.UUID, createdBy string, source Source) *DashboardV2 { +func (postable PostableDashboardV2) NewDashboardV2(orgID valuer.UUID, createdBy string, source Source) (*DashboardV2, error) { now := time.Now() name := postable.Name if postable.GenerateName { name = generateDashboardName(postable.Spec.Display.Name) } + // Checked on the final name, here rather than in validateName, because only + // the constructor knows the source. + if source != SourceSystem && strings.HasPrefix(name, SystemDashboardNamePrefix) { + return nil, errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "name %q is invalid: the %q prefix is reserved for system dashboards", name, SystemDashboardNamePrefix) + } + if source == SourceSystem && !strings.HasPrefix(name, SystemDashboardNamePrefix) { + return nil, errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "name %q is invalid: system dashboard names must start with the %q prefix", name, SystemDashboardNamePrefix) + } return &DashboardV2{ Identifiable: types.Identifiable{ID: valuer.GenerateUUID()}, @@ -224,7 +248,7 @@ func (postable PostableDashboardV2) NewDashboardV2(orgID valuer.UUID, createdBy Name: name, Tags: tagtypes.NewTagsFromPostableTags(orgID, coretypes.KindDashboard, postable.Tags), Spec: postable.Spec, - } + }, nil } func (p *PostableDashboardV2) UnmarshalJSON(data []byte) error { @@ -365,6 +389,36 @@ func (d DashboardV2) ToGettableDashboardV2() GettableDashboardV2 { } } +// GettableSystemDashboard is the system-dashboard endpoint's response. System +// dashboards are addressed by their stable definition name, so it carries no id. +type GettableSystemDashboard struct { + types.TimeAuditable + types.UserAuditable + + OrgID valuer.UUID `json:"orgId" required:"true"` + Locked bool `json:"locked" required:"true"` + Source Source `json:"source" required:"true"` + + DashboardV2MetadataBase + Name string `json:"name" required:"true"` + Tags []*tagtypes.GettableTag `json:"tags" required:"true"` + Spec DashboardSpec `json:"spec" required:"true"` +} + +func (d DashboardV2) ToGettableSystemDashboard() GettableSystemDashboard { + return GettableSystemDashboard{ + TimeAuditable: d.TimeAuditable, + UserAuditable: d.UserAuditable, + OrgID: d.OrgID, + Locked: d.Locked, + Source: d.Source, + DashboardV2MetadataBase: d.DashboardV2MetadataBase, + Name: d.Name, + Tags: tagtypes.NewGettableTagsFromTags(d.Tags), + Spec: d.Spec, + } +} + // ════════════════════════════════════════════════════════════════════════ // Storable // ════════════════════════════════════════════════════════════════════════ diff --git a/pkg/types/dashboardtypes/perses_dashboard_convertors_test.go b/pkg/types/dashboardtypes/perses_dashboard_convertors_test.go index bc17c854a4a..0faf2d029ff 100644 --- a/pkg/types/dashboardtypes/perses_dashboard_convertors_test.go +++ b/pkg/types/dashboardtypes/perses_dashboard_convertors_test.go @@ -89,21 +89,25 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { cases := []struct { scenario string source Source + name string expectedLocked bool }{ { scenario: "user source is not locked", source: SourceUser, + name: "my-dashboard", expectedLocked: false, }, { scenario: "system source is not locked", source: SourceSystem, + name: SystemDashboardNamePrefix + "my-dashboard", expectedLocked: false, }, { scenario: "integration source is locked", source: SourceIntegration, + name: "my-dashboard", expectedLocked: true, }, } @@ -115,7 +119,7 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { SchemaVersion: SchemaVersion, Image: "img", }, - Name: "my-dashboard", + Name: tc.name, Tags: []tagtypes.PostableTag{ {Key: "team", Value: "platform"}, {Key: "env", Value: "prod"}, @@ -124,7 +128,8 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { } before := time.Now() - dashboard := postable.NewDashboardV2(orgID, "alice", tc.source) + dashboard, err := postable.NewDashboardV2(orgID, "alice", tc.source) + require.NoError(t, err) after := time.Now() require.NotNil(t, dashboard) @@ -160,8 +165,10 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { Spec: DashboardSpec{}, } - first := postable.NewDashboardV2(orgID, "alice", SourceUser) - second := postable.NewDashboardV2(orgID, "alice", SourceUser) + first, err := postable.NewDashboardV2(orgID, "alice", SourceUser) + require.NoError(t, err) + second, err := postable.NewDashboardV2(orgID, "alice", SourceUser) + require.NoError(t, err) assert.NotEqual(t, first.ID, second.ID, "expected distinct UUIDs across invocations") }) @@ -174,7 +181,8 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { }, } - dashboard := postable.NewDashboardV2(orgID, "alice", SourceUser) + dashboard, err := postable.NewDashboardV2(orgID, "alice", SourceUser) + require.NoError(t, err) assert.True(t, strings.HasPrefix(dashboard.Name, "my-dashboard-"), "expected slug prefix, got %q", dashboard.Name) assert.Len(t, dashboard.Name, len("my-dashboard-")+dashboardNameSuffixLen) }) diff --git a/pkg/types/dashboardtypes/perses_dashboard_patch_test.go b/pkg/types/dashboardtypes/perses_dashboard_patch_test.go index 3cd7ba421f0..84d91f7b9d4 100644 --- a/pkg/types/dashboardtypes/perses_dashboard_patch_test.go +++ b/pkg/types/dashboardtypes/perses_dashboard_patch_test.go @@ -109,7 +109,8 @@ func TestPatchableDashboardV2_Apply(t *testing.T) { var p PostableDashboardV2 require.NoError(t, json.Unmarshal([]byte(basePostableJSON), &p), "base postable JSON must validate") testOrgID := valuer.GenerateUUID() - base := p.NewDashboardV2(testOrgID, "somecreatedthisiguess@signoz.io", SourceUser) + base, err := p.NewDashboardV2(testOrgID, "somecreatedthisiguess@signoz.io", SourceUser) + require.NoError(t, err) base.Tags = []*tagtypes.Tag{ {Key: "team", Value: "alpha"}, {Key: "env", Value: "prod"}, diff --git a/pkg/types/dashboardtypes/perses_dashboard_test.go b/pkg/types/dashboardtypes/perses_dashboard_test.go index fd48b4e143e..cdcbe62ee44 100644 --- a/pkg/types/dashboardtypes/perses_dashboard_test.go +++ b/pkg/types/dashboardtypes/perses_dashboard_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/valuer" "github.com/perses/spec/go/dashboard" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1928,3 +1929,37 @@ func TestEnsureSingleExpressionAggregation(t *testing.T) { }) } } + +// Guards the constant: a prefixed name must stay a valid DNS-1123 label. +func TestSystemDashboardNamePrefix(t *testing.T) { + require.NoError(t, validateDashboardName(SystemDashboardNamePrefix+"ai-o11y-overview")) +} + +func TestNewDashboardV2RejectsReservedName(t *testing.T) { + testCases := []struct { + description string + name string + source Source + errContains string + }{ + {description: "reserved name for a system dashboard", name: SystemDashboardNamePrefix + "overview", source: SourceSystem}, + {description: "reserved name for a user dashboard", name: SystemDashboardNamePrefix + "overview", source: SourceUser, errContains: "reserved for system dashboards"}, + {description: "reserved name for an integration dashboard", name: SystemDashboardNamePrefix + "overview", source: SourceIntegration, errContains: "reserved for system dashboards"}, + {description: "unprefixed name for a system dashboard", name: "overview", source: SourceSystem, errContains: "must start with"}, + {description: "ordinary name for a user dashboard", name: "overview", source: SourceUser}, + {description: "fewer hyphens than the prefix for a user dashboard", name: "signoz--overview", source: SourceUser}, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + postable := PostableDashboardV2{Name: testCase.name} + _, err := postable.NewDashboardV2(valuer.GenerateUUID(), "user@signoz.io", testCase.source) + if testCase.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), testCase.errContains) + return + } + require.NoError(t, err) + }) + } +} diff --git a/pkg/types/dashboardtypes/store.go b/pkg/types/dashboardtypes/store.go index 3de1503f0e0..9ef59670471 100644 --- a/pkg/types/dashboardtypes/store.go +++ b/pkg/types/dashboardtypes/store.go @@ -13,6 +13,9 @@ type Store interface { Get(context.Context, valuer.UUID, valuer.UUID) (*StorableDashboard, error) + // GetByName resolves a dashboard by its per-org unique name. + GetByName(ctx context.Context, orgID valuer.UUID, name string) (*StorableDashboard, error) + GetPublic(context.Context, string) (*StorablePublicDashboard, error) GetDashboardByOrgsAndPublicID(context.Context, []string, string) (*StorableDashboard, error) @@ -72,4 +75,13 @@ type Store interface { UpdateDashboardView(ctx context.Context, view *DashboardView) error DeleteDashboardView(ctx context.Context, orgID valuer.UUID, id valuer.UUID) error + + // ════════════════════════════════════════════════════════════════════════ + // System dashboard methods + // ════════════════════════════════════════════════════════════════════════ + CreateSystemDashboard(ctx context.Context, storable *StorableSystemDashboard) error + + GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*StorableSystemDashboard, error) + + UpdateSystemDashboardVersion(ctx context.Context, orgID valuer.UUID, name string, version int) error } diff --git a/pkg/types/dashboardtypes/system_dashboard.go b/pkg/types/dashboardtypes/system_dashboard.go new file mode 100644 index 00000000000..0a43d30c981 --- /dev/null +++ b/pkg/types/dashboardtypes/system_dashboard.go @@ -0,0 +1,45 @@ +package dashboardtypes + +import ( + "time" + + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/valuer" + "github.com/uptrace/bun" +) + +var ( + ErrCodeSystemDashboardNotFound = errors.MustNewCode("system_dashboard_not_found") + ErrCodeSystemDashboardDefinitionInvalid = errors.MustNewCode("system_dashboard_definition_invalid") + ErrCodeSystemDashboardAlreadyProvisioned = errors.MustNewCode("system_dashboard_already_provisioned") +) + +// ProvisionerIdentity is stamped into created_by/updated_by by the reconciler. +const ProvisionerIdentity = "signoz" + +// StorableSystemDashboard records the shipped version each org's copy of a system +// dashboard was last provisioned at. That version is the only thing the dashboard +// row cannot answer, since the binary only embeds the latest definition. +type StorableSystemDashboard struct { + bun.BaseModel `bun:"table:system_dashboard"` + + types.Identifiable + types.TimeAuditable + OrgID valuer.UUID `bun:"org_id,type:text,notnull"` + DashboardID valuer.UUID `bun:"dashboard_id,type:text,notnull"` + Name string `bun:"name,type:text,notnull"` + Version int `bun:"version,notnull"` +} + +func NewStorableSystemDashboard(orgID valuer.UUID, dashboardID valuer.UUID, name string, version int) *StorableSystemDashboard { + now := time.Now() + return &StorableSystemDashboard{ + Identifiable: types.Identifiable{ID: valuer.GenerateUUID()}, + TimeAuditable: types.TimeAuditable{CreatedAt: now, UpdatedAt: now}, + OrgID: orgID, + DashboardID: dashboardID, + Name: name, + Version: version, + } +} diff --git a/pkg/types/dashboardtypes/system_dashboard_definition.go b/pkg/types/dashboardtypes/system_dashboard_definition.go new file mode 100644 index 00000000000..371df1bea01 --- /dev/null +++ b/pkg/types/dashboardtypes/system_dashboard_definition.go @@ -0,0 +1,95 @@ +package dashboardtypes + +import ( + "bytes" + "encoding/json" + "slices" + "strings" + + "github.com/SigNoz/signoz/pkg/errors" +) + +// SystemDashboardDefinition is one shipped system dashboard. Version is bumped on +// every content change and drives upgrade detection; the name is the stable key +// and never changes. +type SystemDashboardDefinition struct { + Version int `json:"version"` + Dashboard PostableDashboardV2 `json:"definition"` +} + +func (definition SystemDashboardDefinition) Name() string { + return definition.Dashboard.Name +} + +func NewSystemDashboardDefinition(raw []byte) (SystemDashboardDefinition, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + + var definition SystemDashboardDefinition + if err := decoder.Decode(&definition); err != nil { + return SystemDashboardDefinition{}, errors.WrapInvalidInputf(err, ErrCodeSystemDashboardDefinitionInvalid, "%s", err.Error()) + } + if err := definition.validate(); err != nil { + return SystemDashboardDefinition{}, err + } + + return definition, nil +} + +func (definition SystemDashboardDefinition) validate() error { + if definition.Version < 1 { + return errors.NewInvalidInputf(ErrCodeSystemDashboardDefinitionInvalid, "version must be at least 1, got %d", definition.Version) + } + if !strings.HasPrefix(definition.Name(), SystemDashboardNamePrefix) { + return errors.NewInvalidInputf(ErrCodeSystemDashboardDefinitionInvalid, "name %q must start with %q", definition.Name(), SystemDashboardNamePrefix) + } + if definition.Dashboard.GenerateName { + return errors.NewInvalidInputf(ErrCodeSystemDashboardDefinitionInvalid, "%s: generateName is not allowed, the name is the stable key", definition.Name()) + } + + return nil +} + +// ToUpdatable is how an upgrade re-applies a definition onto an existing row: +// everything but the dashboard's identity comes from the shipped definition. +func (definition SystemDashboardDefinition) ToUpdatable() UpdatableDashboardV2 { + return UpdatableDashboardV2{ + DashboardV2MetadataBase: definition.Dashboard.DashboardV2MetadataBase, + Name: definition.Dashboard.Name, + Tags: definition.Dashboard.Tags, + Spec: definition.Dashboard.Spec, + } +} + +// SystemDashboardRegistry holds every definition embedded in the binary, keyed by name. +type SystemDashboardRegistry struct { + definitions map[string]SystemDashboardDefinition +} + +func NewSystemDashboardRegistry(definitions []SystemDashboardDefinition) (SystemDashboardRegistry, error) { + byName := make(map[string]SystemDashboardDefinition, len(definitions)) + for _, definition := range definitions { + if _, duplicate := byName[definition.Name()]; duplicate { + return SystemDashboardRegistry{}, errors.NewInvalidInputf(ErrCodeSystemDashboardDefinitionInvalid, "duplicate system dashboard name %q", definition.Name()) + } + byName[definition.Name()] = definition + } + + return SystemDashboardRegistry{definitions: byName}, nil +} + +func (registry SystemDashboardRegistry) Get(name string) (SystemDashboardDefinition, bool) { + definition, ok := registry.definitions[name] + return definition, ok +} + +// List returns the definitions sorted by name so provisioning order is stable. +func (registry SystemDashboardRegistry) List() []SystemDashboardDefinition { + definitions := make([]SystemDashboardDefinition, 0, len(registry.definitions)) + for _, definition := range registry.definitions { + definitions = append(definitions, definition) + } + slices.SortFunc(definitions, func(a, b SystemDashboardDefinition) int { return strings.Compare(a.Name(), b.Name()) }) + + return definitions +} diff --git a/pkg/types/telemetrystoretypes/json.go b/pkg/types/telemetrystoretypes/json.go new file mode 100644 index 00000000000..ea44f5cd7a7 --- /dev/null +++ b/pkg/types/telemetrystoretypes/json.go @@ -0,0 +1,37 @@ +package telemetrystoretypes + +import ( + "github.com/SigNoz/signoz/pkg/errors" + "github.com/bytedance/sonic" +) + +var ErrCodeUnmarshalJSONColumn = errors.MustNewCode("fail_unmarshal_json_column") + +// JSONValue is the scan target for a ClickHouse JSON column: the connection sets +// output_format_native_write_json_as_string, so the column arrives as a raw document rather than +// the chcol.JSON the driver reports as its scan type. +type JSONValue map[string]any + +// Scan decodes into a fresh map every time: a scan target is reused across rows, and unmarshalling +// into the map already there would both keep its keys and hand every row the same map. +func (v *JSONValue) Scan(src any) error { + var raw []byte + switch value := src.(type) { + case nil: + *v = nil + return nil + case string: + raw = []byte(value) + case []byte: + raw = value + default: + return errors.NewInternalf(ErrCodeUnmarshalJSONColumn, "cannot decode %T as a JSON column", src) + } + + decoded := JSONValue{} + if err := sonic.Unmarshal(raw, &decoded); err != nil { + return errors.WrapInternalf(err, ErrCodeUnmarshalJSONColumn, "failed to unmarshal JSON column") + } + *v = decoded + return nil +} diff --git a/tests/integration/tests/dashboard/07_system_dashboard.py b/tests/integration/tests/dashboard/07_system_dashboard.py new file mode 100644 index 00000000000..a2f1b4587ca --- /dev/null +++ b/tests/integration/tests/dashboard/07_system_dashboard.py @@ -0,0 +1,192 @@ +from collections.abc import Callable +from http import HTTPStatus + +import requests +from sqlalchemy import sql + +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.dashboards import DASHBOARDS_BASE_URL, MAX_LIST_LIMIT +from fixtures.types import Operation, SigNoz + +SYSTEM_BASE_URL = "/api/v2/dashboards/system" + +# Provisioned for every org by the reconciler; the path segment is the bare +# definition name, the stored name carries the reserved prefix. +SYSTEM_DASHBOARD_NAME = "ai-o11y-overview" +SYSTEM_DASHBOARD_PREFIX = "signoz---" + + +def test_get_system_dashboard( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{SYSTEM_BASE_URL}/{SYSTEM_DASHBOARD_NAME}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + + assert response.status_code == HTTPStatus.OK, response.text + dashboard = response.json()["data"] + assert dashboard["name"] == SYSTEM_DASHBOARD_PREFIX + SYSTEM_DASHBOARD_NAME + assert dashboard["source"] == "system" + assert dashboard["createdBy"] == "signoz" + assert dashboard["schemaVersion"] == "v6" + assert "id" not in dashboard + + +def test_get_system_dashboard_rejects_prefixed_name( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{SYSTEM_BASE_URL}/{SYSTEM_DASHBOARD_PREFIX}{SYSTEM_DASHBOARD_NAME}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert "must not carry" in response.json()["error"]["message"] + + +def test_get_missing_system_dashboard_returns_not_found( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{SYSTEM_BASE_URL}/no-such-dashboard"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + + assert response.status_code == HTTPStatus.NOT_FOUND, response.text + + +def test_system_dashboard_hidden_from_list_but_gettable_by_id( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + # The API never exposes a system dashboard's id; read it from the state row. + with signoz.sqlstore.conn.connect() as conn: + dashboard_id = conn.execute( + sql.text("SELECT dashboard_id FROM system_dashboard WHERE name = :name"), + {"name": SYSTEM_DASHBOARD_PREFIX + SYSTEM_DASHBOARD_NAME}, + ).scalar_one() + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}?limit={MAX_LIST_LIMIT}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.OK, response.text + listed = response.json()["data"]["dashboards"] or [] + assert all(dashboard["source"] != "system" for dashboard in listed) + assert all(dashboard["id"] != dashboard_id for dashboard in listed) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["data"]["source"] == "system" + + +def test_system_dashboard_is_immutable( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{SYSTEM_BASE_URL}/{SYSTEM_DASHBOARD_NAME}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.OK, response.text + dashboard = response.json()["data"] + + with signoz.sqlstore.conn.connect() as conn: + dashboard_id = conn.execute( + sql.text("SELECT dashboard_id FROM system_dashboard WHERE name = :name"), + {"name": SYSTEM_DASHBOARD_PREFIX + SYSTEM_DASHBOARD_NAME}, + ).scalar_one() + + response = requests.put( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}"), + json={ + "schemaVersion": dashboard["schemaVersion"], + "name": dashboard["name"], + "tags": [], + "spec": dashboard["spec"], + }, + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.json()["error"]["code"] == "dashboard_immutable" + + response = requests.delete( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.json()["error"]["code"] == "dashboard_immutable" + + response = requests.put( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}/lock"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.json()["error"]["code"] == "dashboard_immutable" + + response = requests.post( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}/clone"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.json()["error"]["code"] == "dashboard_immutable" + + +def test_create_rejects_reserved_prefix_name( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.post( + signoz.self.host_configs["8080"].get(DASHBOARDS_BASE_URL), + json={ + "schemaVersion": "v6", + "name": f"{SYSTEM_DASHBOARD_PREFIX}custom", + "tags": [], + "spec": { + "display": {"name": "Custom"}, + "variables": [], + "panels": {}, + "layouts": [], + }, + }, + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert "reserved for system dashboards" in response.json()["error"]["message"] diff --git a/tests/integration/tests/querier_json_body/06_json_column_scan.py b/tests/integration/tests/querier_json_body/06_json_column_scan.py new file mode 100644 index 00000000000..96ccc108fb2 --- /dev/null +++ b/tests/integration/tests/querier_json_body/06_json_column_scan.py @@ -0,0 +1,159 @@ +import json +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +from fixtures import querier, types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.logs import Logs + +# A raw ClickHouse query is the one place a JSON column can land in a scalar or +# time-series result: the builder only selects body_v2 in raw list queries. + +DOC_ERROR = {"level": "error", "attrs": {"code": 500}} +DOC_WARN = {"level": "warn"} + +SERVICE = "json-column-scan" +WHERE = f"resources_string['service.name'] = '{SERVICE}'" + + +def test_clickhouse_scalar_with_json_column( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """A scalar result carrying the body_v2 JSON column decodes it into an object.""" + now = datetime.now(tz=UTC) + insert_logs( + [ + Logs( + timestamp=now - timedelta(seconds=seconds), + resources={"service.name": SERVICE}, + body_v2=json.dumps(doc, separators=(",", ":")), + body_promoted="", + severity_text="INFO", + ) + for seconds, doc in [(3, DOC_ERROR), (2, DOC_ERROR), (1, DOC_WARN)] + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms = int((now - timedelta(minutes=10)).timestamp() * 1000) + end_ms = int(now.timestamp() * 1000) + + # `select *` picks up body_v2 without naming it + response = querier.make_query_request( + signoz, + token, + start_ms, + end_ms, + [ + { + "type": "clickhouse_sql", + "spec": { + "name": "A", + "query": f"SELECT * FROM signoz_logs.distributed_logs_v2 WHERE {WHERE} ORDER BY timestamp LIMIT 1", + "disabled": False, + }, + } + ], + request_type=querier.RequestType.SCALAR, + ) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + + columns = [column["name"] for column in querier.get_scalar_columns(response.json())] + rows = querier.get_scalar_table_data(response.json()) + assert len(rows) == 1 + assert rows[0][columns.index("body_v2")] == DOC_ERROR + + # a JSON column is a legal GROUP BY key, so it can sit next to an aggregation + response = querier.make_query_request( + signoz, + token, + start_ms, + end_ms, + [ + { + "type": "clickhouse_sql", + "spec": { + "name": "A", + "query": f"SELECT body_v2, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE {WHERE} GROUP BY body_v2", + "disabled": False, + }, + } + ], + request_type=querier.RequestType.SCALAR, + ) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + + rows = querier.get_scalar_table_data(response.json()) + counts = {json.dumps(row[0], sort_keys=True): row[1] for row in rows} + assert counts == { + json.dumps(DOC_ERROR, sort_keys=True): 2, + json.dumps(DOC_WARN, sort_keys=True): 1, + } + + +def test_clickhouse_time_series_grouped_by_json_column( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """Grouping a graph by the JSON column labels each series with its document.""" + now = datetime.now(tz=UTC) + insert_logs( + [ + Logs( + timestamp=now - timedelta(seconds=seconds), + resources={"service.name": SERVICE}, + body_v2=json.dumps(doc, separators=(",", ":")), + body_promoted="", + severity_text="INFO", + ) + for seconds, doc in [(3, DOC_ERROR), (2, DOC_ERROR), (1, DOC_WARN)] + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = querier.make_query_request( + signoz, + token, + int((now - timedelta(minutes=10)).timestamp() * 1000), + int(now.timestamp() * 1000), + [ + { + "type": "clickhouse_sql", + "spec": { + "name": "A", + "query": (f"SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, body_v2, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE {WHERE} GROUP BY ts, body_v2"), + "disabled": False, + }, + } + ], + request_type=querier.RequestType.TIME_SERIES, + ) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + + series = querier.get_all_series(response.json(), "A") + assert len(series) == 2 + + # One series per document, labelled with the document rendered with sorted keys. The label is + # the verbatim group key, so it keeps the `message: ""` the typed body path materializes into + # every document; points may split across minute buckets, so compare the per-series sum. + counts = {} + for single_series in series: + labels = single_series["labels"] + assert len(labels) == 1 + assert labels[0]["key"]["name"] == "body_v2" + counts[labels[0]["value"]] = sum(point["value"] for point in single_series["values"]) + assert counts == { + json.dumps(DOC_ERROR | {"message": ""}, sort_keys=True, separators=(",", ":")): 2, + json.dumps(DOC_WARN | {"message": ""}, sort_keys=True, separators=(",", ":")): 1, + }