From aca35a13097bc1a2de14a22c624b4b04a182c2a0 Mon Sep 17 00:00:00 2001 From: Swapnil Nakade Date: Tue, 1 Sep 2026 09:26:50 +0000 Subject: [PATCH 1/4] chore: bumping cloud integration agent version to v0.0.14 (#12737) #### Description Bumping cloud integration agent version from v0.0.13 to v0.0.14 #### Contributes to https://github.com/SigNoz/platform-pod/issues/3038 --- pkg/modules/cloudintegration/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/modules/cloudintegration/config.go b/pkg/modules/cloudintegration/config.go index 69de8b9e464..6d35946e5f4 100644 --- a/pkg/modules/cloudintegration/config.go +++ b/pkg/modules/cloudintegration/config.go @@ -22,7 +22,7 @@ func newConfig() factory.Config { Agent: AgentConfig{ // we will maintain the latest version of cloud integration agent from here, // till we automate it externally or figure out a way to validate it. - Version: "v0.0.13", + Version: "v0.0.14", }, } } From 23dd98bee7ea2921973bb5b2124c229f8239a4e8 Mon Sep 17 00:00:00 2001 From: Gaurav Tewari Date: Tue, 1 Sep 2026 09:30:30 +0000 Subject: [PATCH 2/4] chore: update sample json in test ai-o11y (#12741) #### Description - Update the sample span JSON in the LLM Observability attribute-mapping Test tab to use OpenInference-style attribute keys (`llm.model_name`, `llm.provider`, `llm.token_count.*`, `input.value`, `output.value`) instead of the earlier mix of `gen_ai.*` and placeholder `my_company.*` keys. - The sample is what users see first when trying out attribute mapping, so it should reflect the attribute shape they'll actually be mapping from. #### Issues closed by this PR Closes - https://github.com/orgs/SigNoz/projects/39/views/20?pane=issue&itemId=238442542&issue=SigNoz%7Cengineering-pod%7C5997 #### Screenshots / Screen Recordings image #### Additional Information - Constant-only change (`SAMPLE_SPAN_JSON` in `spanInputStorage.ts`); no parsing or mapping logic touched. - Worth a sanity check that the new keys line up with the attribute names the mapping UI is expected to suggest. --------- Co-authored-by: Gaurav Tewari --- .../TestTab/__tests__/TestTab.test.tsx | 23 ++++++++++--------- .../TestTab/spanInputStorage.ts | 13 +++++++---- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx index 03c9e7c5123..93cabb8f171 100644 --- a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx +++ b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx @@ -38,19 +38,20 @@ import { TEST_ENDPOINT, } from '../../__tests__/fixtures'; +const SAMPLE_SPAN = JSON.parse(SAMPLE_SPAN_JSON) as { + attributes: Record; + resource: Record; +}; + +const MAPPED_ATTRIBUTE_KEY = 'gen_ai.content.prompt'; + +// Deriving from the sample keeps exactly one key added, so the single `populated` badge assertion below stays exact. const RESULT_SPAN = { attributes: { - 'my_company.llm.input': 'What is quantum computing?', - 'llm.input_messages': 'What is quantum computing?', - 'gen_ai.request.model': 'gpt-4', - 'gen_ai.usage.total_tokens': 1250, - 'gen_ai.content.completion': 'Quantum computing leverages...', - 'gen_ai.content.prompt': 'What is quantum computing?', - }, - resource: { - 'service.name': 'llm-gateway', - 'deployment.environment': 'production', + ...SAMPLE_SPAN.attributes, + [MAPPED_ATTRIBUTE_KEY]: SAMPLE_SPAN.attributes['input.value'], }, + resource: SAMPLE_SPAN.resource, }; const EDITED_SPAN_JSON = `{ @@ -97,7 +98,7 @@ describe('TestTab — sample-span flow', () => { ).resolves.toBeInTheDocument(); expect(screen.getByTestId('test-result-0')).toBeInTheDocument(); expect(screen.getByTestId('test-result-0-attributes')).toHaveTextContent( - 'gen_ai.content.prompt', + MAPPED_ATTRIBUTE_KEY, ); expect(screen.getByText('populated')).toBeInTheDocument(); expect(screen.queryByTestId('test-error')).not.toBeInTheDocument(); diff --git a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/spanInputStorage.ts b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/spanInputStorage.ts index fc2440a19a0..3fe7bb480e1 100644 --- a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/spanInputStorage.ts +++ b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/spanInputStorage.ts @@ -7,11 +7,14 @@ import { parseSpanInput } from './testPayload'; export const SAMPLE_SPAN_JSON = `{ "attributes": { - "my_company.llm.input": "What is quantum computing?", - "llm.input_messages": "What is quantum computing?", - "gen_ai.request.model": "gpt-4", - "gen_ai.usage.total_tokens": 1250, - "gen_ai.content.completion": "Quantum computing leverages..." + "llm.model_name": "gpt-4o", + "llm.provider": "openai", + "llm.token_count.prompt": 1024, + "llm.token_count.completion": 226, + "llm.token_count.prompt_details.cache_read": 512, + "input.value": "What is quantum computing?", + "output.value": "Quantum computing leverages superposition and entanglement...", + "session.id": "chat-8f2e41" }, "resource": { "service.name": "llm-gateway", From 74c946fe79885401ffe0a2b14c01306a68715c3c Mon Sep 17 00:00:00 2001 From: Tushar Vats Date: Tue, 1 Sep 2026 10:07:56 +0000 Subject: [PATCH 3/4] fix(querier): decode JSON columns in table and graph queries (#12555) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Description **What was broken:** on a stack using the new JSON log body, any query asking for a **table** (`scalar`) or a **graph** (`time_series`) failed with HTTP 500 if the result contained a JSON column. The logs list view worked fine, which is why this went unnoticed. The smallest way to hit it is a raw ClickHouse panel running `select * from signoz_logs.logs_v2`. **Why it happened:** we ask ClickHouse to send JSON columns as plain text, but the driver reports that such a column needs a different Go type — so the reader prepared the wrong kind of container and the read failed. The driver only reports the correct type *after* the first row has been read, which is too late for code that sets up its containers up front. The original JSON work patched around this inside the logs-list reader only; the connection setting that causes it is global, so the other two readers stayed broken. **The fix:** correct the reported type once, at the connection that sets that option, so every reader gets a container that works and receives a normal map. Concretely: - `pkg/querier` no longer needs its own workaround — the three readers are back to ordinary code. - The older v3/v4 read paths had the same bug and are fixed without any changes of their own. - A JSON path value such as `body_v2.level` now comes back as `"error"` or `7` instead of a driver wrapper object. - Grouping a graph by the whole JSON body used to collapse every group into a single unlabelled line; each document now labels its own series. #### Issues closed by this PR Fixes https://github.com/SigNoz/engineering-pod/issues/5911 #### Additional Information Verified end to end against a local stack with 1,000,000 log rows and 200,002 distinct `trace_id`s: | query | before | after | | --- | --- | --- | | table query over a JSON column | 500 | 200, body returned as an object | | graph grouped by the JSON body | 500 | 200, 22 series, one per document | | graph grouped by `trace_id` (200k groups) | 200 | 200, unchanged | A follow-up PR stacked on this one reworks how the graph reader classifies columns — fixing boolean and small-integer columns in raw SQL panels and cutting the reader's allocations. Known gaps, unchanged from `main` and out of scope here: - Waterfall and flamegraph read rows into structs, which this fix does not cover. Moving span attributes to JSON will need the same type on those fields, and one helper there fails silently rather than erroring. - Dashboard variable queries no longer crash on a JSON column but still reject it as an unsupported value type. - A `Map(String, JSON)` column **panics inside the driver**, which can take the process down. Confirmed still unfixed on `clickhouse-go` main, and not yet reported upstream. --- pkg/querier/consume.go | 81 ++++----- pkg/querier/consume_test.go | 105 ++++++++++++ pkg/querier/postprocess.go | 61 ++++--- pkg/querier/querier_test.go | 3 + .../clickhousetelemetrystore/provider.go | 2 +- pkg/telemetrystore/rows.go | 39 +++++ pkg/telemetrystore/telemetrystoretest/conn.go | 23 +++ .../telemetrystoretest/provider.go | 2 +- pkg/types/telemetrystoretypes/json.go | 37 ++++ .../querier_json_body/06_json_column_scan.py | 159 ++++++++++++++++++ 10 files changed, 448 insertions(+), 64 deletions(-) create mode 100644 pkg/telemetrystore/rows.go create mode 100644 pkg/telemetrystore/telemetrystoretest/conn.go create mode 100644 pkg/types/telemetrystoretypes/json.go create mode 100644 tests/integration/tests/querier_json_body/06_json_column_scan.py diff --git a/pkg/querier/consume.go b/pkg/querier/consume.go index 34ee097ca1a..99935b9cf84 100644 --- a/pkg/querier/consume.go +++ b/pkg/querier/consume.go @@ -1,6 +1,7 @@ package querier import ( + "encoding/json" "fmt" "math" "reflect" @@ -11,12 +12,12 @@ import ( "strings" "time" + "github.com/ClickHouse/clickhouse-go/v2/lib/chcol" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" - "github.com/SigNoz/signoz/pkg/errors" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/spantypes" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" - "github.com/bytedance/sonic" ) var ( @@ -30,8 +31,6 @@ var ( // written clickhouse query. The column alias indcate which value is // to be considered as final result (or target). legacyReservedColumnTargetAliases = []string{"__result", "__value", "result", "res", "value"} - - CodeFailUnmarshalJSONColumn = errors.MustNewCode("fail_unmarshal_json_column") ) // stripKeyAlias removes the __SELECT_KEY__ / __GROUP_BY_KEY__ prefix from a result @@ -40,6 +39,32 @@ func stripKeyAlias(name string) string { return keyAliasRe.ReplaceAllString(name, "") } +// unwrapVariant returns the concrete value inside the chcol.Variant envelope the driver scans a +// Dynamic column — a JSON path such as body_v2.level — into. +func unwrapVariant(val any) any { + if v, ok := val.(chcol.Variant); ok { + return v.Any() + } + return val +} + +// labelValue renders a group-by value the payload cannot carry as a scalar — a JSON column, or a +// Dynamic one — as a stable string, so that rows differing only in that value land in different +// series. JSON goes through encoding/json for its sorted map keys: ClickHouse groups documents by +// structure, so two rows it considers equal have to produce the same label. +func labelValue(val any) string { + val = unwrapVariant(val) + if val == nil { + return "" + } + if v, ok := val.(telemetrystoretypes.JSONValue); ok { + if raw, err := json.Marshal(v); err == nil { + return string(raw) + } + } + return fmt.Sprint(val) +} + // consume reads every row and shapes it into the payload expected for the // given request type. // @@ -205,6 +230,14 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt Value: *val, }) + case *telemetrystoretypes.JSONValue, *chcol.Variant: + val := labelValue(derefValue(ptr)) + lblVals = append(lblVals, val) + lblObjs = append(lblObjs, &qbtypes.Label{ + Key: telemetrytypes.TelemetryFieldKey{Name: name}, + Value: val, + }) + default: continue } @@ -345,7 +378,7 @@ func readAsScalar(rows driver.Rows, queryName string) (*qbtypes.ScalarData, erro // 2. deref each slot into the output row row := make([]any, len(scan)) for i, cell := range scan { - row[i] = derefValue(cell) + row[i] = unwrapVariant(derefValue(cell)) } data = append(data, row) } @@ -382,31 +415,13 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) { colTypes := rows.ColumnTypes() colCnt := len(colNames) - // Helper that decides scan target per column based on DB type - makeScanTarget := func(i int) any { - dbt := strings.ToUpper(colTypes[i].DatabaseTypeName()) - if strings.HasPrefix(dbt, "JSON") { - // Since the driver fails to decode JSON/Dynamic into native Go values, we read it as raw bytes - // TODO: check in future if fixed in the driver - var v []byte - return &v - } - return reflect.New(colTypes[i].ScanType()).Interface() - } - - // Build a template slice of correctly-typed pointers once - scanTpl := make([]any, colCnt) - for i := range colTypes { - scanTpl[i] = makeScanTarget(i) - } - var outRows []*qbtypes.RawRow for rows.Next() { // fresh copy of the scan slice (otherwise the driver reuses pointers) scan := make([]any, colCnt) - for i := range scanTpl { - scan[i] = makeScanTarget(i) + for i := range colTypes { + scan[i] = reflect.New(colTypes[i].ScanType()).Interface() } if err := rows.Scan(scan...); err != nil { @@ -421,21 +436,7 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) { name := stripKeyAlias(colNames[i]) // de-reference the typed pointer to any - val := reflect.ValueOf(cellPtr).Elem().Interface() - // Post-process JSON columns: unmarshal bytes into map[string]any - if strings.HasPrefix(strings.ToUpper(colTypes[i].DatabaseTypeName()), "JSON") { - switch x := val.(type) { - case []byte: - var m map[string]any - err := sonic.Unmarshal(x, &m) - if err != nil { - return nil, errors.WrapInternalf(err, CodeFailUnmarshalJSONColumn, "failed to unmarshal JSON column %s", name) - } - val = m - default: - // already a structured type (map[string]any, []any, etc.) - } - } + val := unwrapVariant(reflect.ValueOf(cellPtr).Elem().Interface()) // special-case: timestamp column if name == "timestamp" || name == "timestamp_datetime" { diff --git a/pkg/querier/consume_test.go b/pkg/querier/consume_test.go index 9fb5ab6132f..17438a24933 100644 --- a/pkg/querier/consume_test.go +++ b/pkg/querier/consume_test.go @@ -3,8 +3,16 @@ package querier import ( "reflect" "testing" + "time" + "github.com/ClickHouse/clickhouse-go/v2/lib/chcol" + cmock "github.com/SigNoz/clickhouse-go-mock" + "github.com/SigNoz/signoz/pkg/telemetrystore" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/spantypes" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMergeSpanAttributeColumns_ParsesEventsAndLinks(t *testing.T) { @@ -75,6 +83,103 @@ func TestMergeSpanAttributeColumns_ParsesEventsAndLinks(t *testing.T) { } } +// A ClickHouse query can put a JSON column in the result of any request type — e.g. +// `select * from signoz_logs.logs_v2` on a body_v2 stack, where `*` covers body_v2. +func TestConsume_JSONColumn(t *testing.T) { + ts := time.Date(2026, 8, 14, 10, 0, 0, 0, time.UTC) + body := `{"level":"error","attrs":{"code":500}}` + wantBody := telemetrystoretypes.JSONValue{ + "level": "error", + "attrs": map[string]any{"code": float64(500)}, + } + + // the scalar reader reuses its scan slots across rows, so each row must still carry its own body + t.Run("scalar", func(t *testing.T) { + rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{ + {Name: "body_v2", Type: "JSON"}, + {Name: "__result_0", Type: "UInt64"}, + }, [][]any{{body, uint64(3)}, {`{"level":"warn"}`, uint64(1)}})) + + payload, err := consume(rows, qbtypes.RequestTypeScalar, nil, qbtypes.Step{}, "A") + require.NoError(t, err) + + data := payload.(*qbtypes.ScalarData) + require.Len(t, data.Data, 2) + assert.Equal(t, wantBody, data.Data[0][0]) + assert.Equal(t, uint64(3), data.Data[0][1]) + assert.Equal(t, telemetrystoretypes.JSONValue{"level": "warn"}, data.Data[1][0]) + assert.Equal(t, uint64(1), data.Data[1][1]) + }) + + t.Run("time series", func(t *testing.T) { + rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{ + {Name: "ts", Type: "DateTime"}, + {Name: "body_v2", Type: "JSON"}, + {Name: "__result_0", Type: "UInt64"}, + }, [][]any{{ts, body, uint64(3)}})) + + payload, err := consume(rows, qbtypes.RequestTypeTimeSeries, nil, qbtypes.Step{}, "A") + require.NoError(t, err) + + data := payload.(*qbtypes.TimeSeriesData) + require.Len(t, data.Aggregations, 1) + require.Len(t, data.Aggregations[0].Series, 1) + require.Len(t, data.Aggregations[0].Series[0].Values, 1) + assert.Equal(t, float64(3), data.Aggregations[0].Series[0].Values[0].Value) + }) + + // grouping by a JSON column is legal in ClickHouse, so each document has to label its own + // series rather than being dropped, which would merge every group into one + t.Run("time series grouped by the JSON column", func(t *testing.T) { + rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{ + {Name: "ts", Type: "DateTime"}, + {Name: "body_v2", Type: "JSON"}, + {Name: "__result_0", Type: "UInt64"}, + }, [][]any{ + {ts, `{"level":"error"}`, uint64(7)}, + {ts, `{"level":"warn"}`, uint64(2)}, + })) + + payload, err := consume(rows, qbtypes.RequestTypeTimeSeries, nil, qbtypes.Step{}, "A") + require.NoError(t, err) + + data := payload.(*qbtypes.TimeSeriesData) + require.Len(t, data.Aggregations, 1) + require.Len(t, data.Aggregations[0].Series, 2) + + got := map[string]float64{} + for _, series := range data.Aggregations[0].Series { + require.Len(t, series.Labels, 1) + require.Len(t, series.Values, 1) + got[series.Labels[0].Value.(string)] = series.Values[0].Value + } + assert.Equal(t, map[string]float64{`{"level":"error"}`: 7, `{"level":"warn"}`: 2}, got) + }) + + t.Run("raw", func(t *testing.T) { + rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{ + {Name: "timestamp", Type: "DateTime"}, + {Name: "body_v2", Type: "JSON"}, + }, [][]any{{ts, body}})) + + payload, err := consume(rows, qbtypes.RequestTypeRaw, nil, qbtypes.Step{}, "A") + require.NoError(t, err) + + data := payload.(*qbtypes.RawData) + require.Len(t, data.Rows, 1) + assert.Equal(t, ts, data.Rows[0].Timestamp.UTC()) + assert.Equal(t, wantBody, data.Rows[0].Data["body_v2"]) + }) +} + +// A JSON path (e.g. `body_v2.level`) comes back as a Dynamic column, which the driver scans +// into a chcol.Variant envelope rather than the value itself. +func TestUnwrapVariant(t *testing.T) { + assert.Equal(t, "error", unwrapVariant(chcol.NewDynamicWithType("error", "String"))) + assert.Nil(t, unwrapVariant(chcol.Dynamic{})) + assert.Equal(t, uint64(3), unwrapVariant(uint64(3))) +} + func TestMergeSpanAttributeColumns_EmptyEventsAndLinks(t *testing.T) { data := map[string]any{ "events": []string{}, diff --git a/pkg/querier/postprocess.go b/pkg/querier/postprocess.go index 954ef89d0f9..dd5c1ee7118 100644 --- a/pkg/querier/postprocess.go +++ b/pkg/querier/postprocess.go @@ -13,9 +13,10 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/flagger" - "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/querybuilder" + "github.com/SigNoz/signoz/pkg/types/featuretypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/valuer" ) @@ -72,9 +73,13 @@ func (q *querier) postProcessResults(ctx context.Context, orgID valuer.UUID, res case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]: if result, ok := typedResults[spec.Name]; ok { result = postProcessBuilderQuery(q, result, spec, req) - result = q.postProcessLogBody(ctx, orgID, result, req) + result = q.postProcessLogBody(ctx, orgID, result) typedResults[spec.Name] = result } + case qbtypes.ClickHouseQuery: + if result, ok := typedResults[spec.Name]; ok { + typedResults[spec.Name] = q.postProcessLogBody(ctx, orgID, result) + } case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]: if result, ok := typedResults[spec.Name]; ok { result = postProcessMetricQuery(q, result, spec, req) @@ -1051,32 +1056,44 @@ func (q *querier) calculateFormulaStep(expression string, req *qbtypes.QueryRang return result } -// postProcessLogBody removes the "message" key from the body map when it is empty. -// Only runs for raw list queries with the use_json_body feature enabled. -func (q *querier) postProcessLogBody(ctx context.Context, orgID valuer.UUID, result *qbtypes.Result, req *qbtypes.QueryRangeRequest) *qbtypes.Result { - if req.RequestType != qbtypes.RequestTypeRaw { - return result - } +// postProcessLogBody removes the empty "message" the typed body path materializes into every +// document, wherever a decoded body lands in the payload — raw rows and scalar cells, under the +// column's own name or the builder's `body` alias. Only runs with the use_json_body feature +// enabled. A time-series label keeps the document verbatim: it is the group key. +func (q *querier) postProcessLogBody(ctx context.Context, orgID valuer.UUID, result *qbtypes.Result) *qbtypes.Result { if !q.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) { return result } - rawData, ok := result.Value.(*qbtypes.RawData) - if !ok { - return result - } - for _, row := range rawData.Rows { - bodyMap, ok := row.Data["body"].(map[string]any) - if !ok { - continue + switch data := result.Value.(type) { + case *qbtypes.RawData: + for _, row := range data.Rows { + for _, name := range []string{"body", "body_v2"} { + stripEmptyBodyMessage(row.Data[name]) + } } - if msg, exists := bodyMap["message"]; exists { - switch v := msg.(type) { - case string: - if v == "" { - delete(bodyMap, "message") - } + case *qbtypes.ScalarData: + for idx, column := range data.Columns { + if column.Name != "body" && column.Name != "body_v2" { + continue + } + for _, row := range data.Data { + stripEmptyBodyMessage(row[idx]) } } } return result } + +// stripEmptyBodyMessage drops `message: ""` from a decoded body document: the message path is +// typed String in the JSON column, so ClickHouse materializes it even for documents that never +// carried one. Anything that is not a decoded document — the legacy string body, a NULL cell — +// is legal under these names and left alone. +func stripEmptyBodyMessage(val any) { + bodyMap, ok := val.(telemetrystoretypes.JSONValue) + if !ok { + return + } + if msg, ok := bodyMap["message"].(string); ok && msg == "" { + delete(bodyMap, "message") + } +} diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index 17e40364d40..85913b87b72 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -189,6 +189,7 @@ func TestRunExecutesQueriesConcurrently(t *testing.T) { q := &querier{ logger: instrumentationtest.New().Logger(), + fl: flaggertest.New(t), maxConcurrentQueries: numQueries, } @@ -236,6 +237,7 @@ func TestRunRespectsMaxConcurrentQueries(t *testing.T) { q := &querier{ logger: instrumentationtest.New().Logger(), + fl: flaggertest.New(t), maxConcurrentQueries: limit, } @@ -273,6 +275,7 @@ func TestRunRespectsMaxConcurrentQueries(t *testing.T) { func TestRunQueryErrorCancelsSiblings(t *testing.T) { q := &querier{ logger: instrumentationtest.New().Logger(), + fl: flaggertest.New(t), maxConcurrentQueries: 4, } diff --git a/pkg/telemetrystore/clickhousetelemetrystore/provider.go b/pkg/telemetrystore/clickhousetelemetrystore/provider.go index 9eb2e1f0d95..edf34179f06 100644 --- a/pkg/telemetrystore/clickhousetelemetrystore/provider.go +++ b/pkg/telemetrystore/clickhousetelemetrystore/provider.go @@ -184,7 +184,7 @@ func (p *provider) Query(ctx context.Context, query string, args ...interface{}) } return &rowsWithHooks{ - Rows: rows, + Rows: telemetrystore.WrapRows(rows), ctx: ctx, event: event, onClose: func() { telemetrystore.WrapAfterQuery(p.hooks, ctx, event) }, diff --git a/pkg/telemetrystore/rows.go b/pkg/telemetrystore/rows.go new file mode 100644 index 00000000000..194557a908f --- /dev/null +++ b/pkg/telemetrystore/rows.go @@ -0,0 +1,39 @@ +package telemetrystore + +import ( + "reflect" + "strings" + + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" +) + +// WrapRows reports JSONValue as the scan type of every JSON column. Nested JSON — Array(JSON), +// Map(String, JSON) — is not covered. +func WrapRows(rows driver.Rows) driver.Rows { + return &rowsWithJSONScanType{Rows: rows} +} + +type rowsWithJSONScanType struct { + driver.Rows +} + +func (r *rowsWithJSONScanType) ColumnTypes() []driver.ColumnType { + colTypes := r.Rows.ColumnTypes() + wrapped := make([]driver.ColumnType, len(colTypes)) + for i, colType := range colTypes { + wrapped[i] = colType + if strings.HasPrefix(strings.ToUpper(colType.DatabaseTypeName()), "JSON") { + wrapped[i] = jsonColumnType{ColumnType: colType} + } + } + return wrapped +} + +type jsonColumnType struct { + driver.ColumnType +} + +func (jsonColumnType) ScanType() reflect.Type { + return reflect.TypeFor[telemetrystoretypes.JSONValue]() +} diff --git a/pkg/telemetrystore/telemetrystoretest/conn.go b/pkg/telemetrystore/telemetrystoretest/conn.go new file mode 100644 index 00000000000..601aef5282c --- /dev/null +++ b/pkg/telemetrystore/telemetrystoretest/conn.go @@ -0,0 +1,23 @@ +package telemetrystoretest + +import ( + "context" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/SigNoz/signoz/pkg/telemetrystore" +) + +// conn wraps rows the way the clickhouse provider does, so mocked JSON columns report the scan +// type they do in production. +type conn struct { + clickhouse.Conn +} + +func (c conn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) { + rows, err := c.Conn.Query(ctx, query, args...) + if err != nil { + return nil, err + } + return telemetrystore.WrapRows(rows), nil +} diff --git a/pkg/telemetrystore/telemetrystoretest/provider.go b/pkg/telemetrystore/telemetrystoretest/provider.go index db73cee2851..6aeb52bb91c 100644 --- a/pkg/telemetrystore/telemetrystoretest/provider.go +++ b/pkg/telemetrystore/telemetrystoretest/provider.go @@ -32,7 +32,7 @@ func New(_ telemetrystore.Config, matcher sqlmock.QueryMatcher) *Provider { // ClickhouseDB returns the mock Clickhouse connection. func (p *Provider) ClickhouseDB() clickhouse.Conn { - return p.clickhouseDB.(clickhouse.Conn) + return conn{Conn: p.clickhouseDB.(clickhouse.Conn)} } // Cluster returns the cluster name. diff --git a/pkg/types/telemetrystoretypes/json.go b/pkg/types/telemetrystoretypes/json.go new file mode 100644 index 00000000000..ea44f5cd7a7 --- /dev/null +++ b/pkg/types/telemetrystoretypes/json.go @@ -0,0 +1,37 @@ +package telemetrystoretypes + +import ( + "github.com/SigNoz/signoz/pkg/errors" + "github.com/bytedance/sonic" +) + +var ErrCodeUnmarshalJSONColumn = errors.MustNewCode("fail_unmarshal_json_column") + +// JSONValue is the scan target for a ClickHouse JSON column: the connection sets +// output_format_native_write_json_as_string, so the column arrives as a raw document rather than +// the chcol.JSON the driver reports as its scan type. +type JSONValue map[string]any + +// Scan decodes into a fresh map every time: a scan target is reused across rows, and unmarshalling +// into the map already there would both keep its keys and hand every row the same map. +func (v *JSONValue) Scan(src any) error { + var raw []byte + switch value := src.(type) { + case nil: + *v = nil + return nil + case string: + raw = []byte(value) + case []byte: + raw = value + default: + return errors.NewInternalf(ErrCodeUnmarshalJSONColumn, "cannot decode %T as a JSON column", src) + } + + decoded := JSONValue{} + if err := sonic.Unmarshal(raw, &decoded); err != nil { + return errors.WrapInternalf(err, ErrCodeUnmarshalJSONColumn, "failed to unmarshal JSON column") + } + *v = decoded + return nil +} diff --git a/tests/integration/tests/querier_json_body/06_json_column_scan.py b/tests/integration/tests/querier_json_body/06_json_column_scan.py new file mode 100644 index 00000000000..96ccc108fb2 --- /dev/null +++ b/tests/integration/tests/querier_json_body/06_json_column_scan.py @@ -0,0 +1,159 @@ +import json +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +from fixtures import querier, types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.logs import Logs + +# A raw ClickHouse query is the one place a JSON column can land in a scalar or +# time-series result: the builder only selects body_v2 in raw list queries. + +DOC_ERROR = {"level": "error", "attrs": {"code": 500}} +DOC_WARN = {"level": "warn"} + +SERVICE = "json-column-scan" +WHERE = f"resources_string['service.name'] = '{SERVICE}'" + + +def test_clickhouse_scalar_with_json_column( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """A scalar result carrying the body_v2 JSON column decodes it into an object.""" + now = datetime.now(tz=UTC) + insert_logs( + [ + Logs( + timestamp=now - timedelta(seconds=seconds), + resources={"service.name": SERVICE}, + body_v2=json.dumps(doc, separators=(",", ":")), + body_promoted="", + severity_text="INFO", + ) + for seconds, doc in [(3, DOC_ERROR), (2, DOC_ERROR), (1, DOC_WARN)] + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms = int((now - timedelta(minutes=10)).timestamp() * 1000) + end_ms = int(now.timestamp() * 1000) + + # `select *` picks up body_v2 without naming it + response = querier.make_query_request( + signoz, + token, + start_ms, + end_ms, + [ + { + "type": "clickhouse_sql", + "spec": { + "name": "A", + "query": f"SELECT * FROM signoz_logs.distributed_logs_v2 WHERE {WHERE} ORDER BY timestamp LIMIT 1", + "disabled": False, + }, + } + ], + request_type=querier.RequestType.SCALAR, + ) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + + columns = [column["name"] for column in querier.get_scalar_columns(response.json())] + rows = querier.get_scalar_table_data(response.json()) + assert len(rows) == 1 + assert rows[0][columns.index("body_v2")] == DOC_ERROR + + # a JSON column is a legal GROUP BY key, so it can sit next to an aggregation + response = querier.make_query_request( + signoz, + token, + start_ms, + end_ms, + [ + { + "type": "clickhouse_sql", + "spec": { + "name": "A", + "query": f"SELECT body_v2, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE {WHERE} GROUP BY body_v2", + "disabled": False, + }, + } + ], + request_type=querier.RequestType.SCALAR, + ) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + + rows = querier.get_scalar_table_data(response.json()) + counts = {json.dumps(row[0], sort_keys=True): row[1] for row in rows} + assert counts == { + json.dumps(DOC_ERROR, sort_keys=True): 2, + json.dumps(DOC_WARN, sort_keys=True): 1, + } + + +def test_clickhouse_time_series_grouped_by_json_column( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """Grouping a graph by the JSON column labels each series with its document.""" + now = datetime.now(tz=UTC) + insert_logs( + [ + Logs( + timestamp=now - timedelta(seconds=seconds), + resources={"service.name": SERVICE}, + body_v2=json.dumps(doc, separators=(",", ":")), + body_promoted="", + severity_text="INFO", + ) + for seconds, doc in [(3, DOC_ERROR), (2, DOC_ERROR), (1, DOC_WARN)] + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = querier.make_query_request( + signoz, + token, + int((now - timedelta(minutes=10)).timestamp() * 1000), + int(now.timestamp() * 1000), + [ + { + "type": "clickhouse_sql", + "spec": { + "name": "A", + "query": (f"SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, body_v2, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE {WHERE} GROUP BY ts, body_v2"), + "disabled": False, + }, + } + ], + request_type=querier.RequestType.TIME_SERIES, + ) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + + series = querier.get_all_series(response.json(), "A") + assert len(series) == 2 + + # One series per document, labelled with the document rendered with sorted keys. The label is + # the verbatim group key, so it keeps the `message: ""` the typed body path materializes into + # every document; points may split across minute buckets, so compare the per-series sum. + counts = {} + for single_series in series: + labels = single_series["labels"] + assert len(labels) == 1 + assert labels[0]["key"]["name"] == "body_v2" + counts[labels[0]["value"]] = sum(point["value"] for point in single_series["values"]) + assert counts == { + json.dumps(DOC_ERROR | {"message": ""}, sort_keys=True, separators=(",", ":")): 2, + json.dumps(DOC_WARN | {"message": ""}, sort_keys=True, separators=(",", ":")): 1, + } From 7eb610287e81e4bd7f812507d6baba28efa20ec7 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Tue, 1 Sep 2026 10:17:10 +0000 Subject: [PATCH 4/4] feat: system dashboards (#12620) #### Description Adding support for system dashboards. * as of now updates are only through new versions in the file. * user cannot update the dashboard * For now kept the dashboard content empty and will raise it separately. Closes https://github.com/SigNoz/engineering-pod/issues/4501 --- cmd/community/server.go | 5 +- cmd/enterprise/server.go | 5 +- docs/api/openapi.yml | 107 ++++++++++ ee/modules/dashboard/impldashboard/module.go | 12 +- .../api/generated/services/dashboard/index.ts | 104 ++++++++++ .../api/generated/services/sigNoz.schemas.ts | 58 ++++++ pkg/apiserver/signozapiserver/dashboard.go | 47 +++++ pkg/modules/dashboard/dashboard.go | 10 + .../fs/definitions/ai-o11y-overview.json | 17 ++ pkg/modules/dashboard/impldashboard/module.go | 28 +-- pkg/modules/dashboard/impldashboard/store.go | 75 +++++++ .../system_dashboard_definitions.go | 46 +++++ .../impldashboard/system_dashboard_service.go | 81 ++++++++ .../dashboard/impldashboard/v2_handler.go | 25 +++ .../dashboard/impldashboard/v2_module.go | 144 ++++++++++++- .../dashboard/impldashboard/v2_module_test.go | 191 +++++++++++++++++ .../organization/implorganization/setter.go | 10 +- pkg/signoz/handler_test.go | 4 +- pkg/signoz/module.go | 106 +++++----- pkg/signoz/module_test.go | 4 +- pkg/signoz/provider.go | 1 + pkg/signoz/signoz.go | 13 +- pkg/sqlmigration/119_add_system_dashboard.go | 93 +++++++++ pkg/types/dashboardtypes/perses_dashboard.go | 62 +++++- .../perses_dashboard_convertors_test.go | 18 +- .../perses_dashboard_patch_test.go | 3 +- .../dashboardtypes/perses_dashboard_test.go | 35 ++++ pkg/types/dashboardtypes/store.go | 12 ++ pkg/types/dashboardtypes/system_dashboard.go | 45 ++++ .../system_dashboard_definition.go | 95 +++++++++ .../tests/dashboard/07_system_dashboard.py | 192 ++++++++++++++++++ 31 files changed, 1554 insertions(+), 94 deletions(-) create mode 100644 pkg/modules/dashboard/impldashboard/fs/definitions/ai-o11y-overview.json create mode 100644 pkg/modules/dashboard/impldashboard/system_dashboard_definitions.go create mode 100644 pkg/modules/dashboard/impldashboard/system_dashboard_service.go create mode 100644 pkg/modules/dashboard/impldashboard/v2_module_test.go create mode 100644 pkg/sqlmigration/119_add_system_dashboard.go create mode 100644 pkg/types/dashboardtypes/system_dashboard.go create mode 100644 pkg/types/dashboardtypes/system_dashboard_definition.go create mode 100644 tests/integration/tests/dashboard/07_system_dashboard.py diff --git a/cmd/community/server.go b/cmd/community/server.go index 7b9edd94f67..36a4b495bf6 100644 --- a/cmd/community/server.go +++ b/cmd/community/server.go @@ -46,6 +46,7 @@ import ( "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/types/authtypes" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/version" "github.com/SigNoz/signoz/pkg/zeus" @@ -103,8 +104,8 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, authtypes.NewRegistry()), nil }, - func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, _ querier.Querier, _ licensing.Licensing, tagModule tag.Module) dashboard.Module { - return impldashboard.NewModule(impldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, tagModule) + func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, _ querier.Querier, _ licensing.Licensing, tagModule tag.Module, systemDashboardRegistry dashboardtypes.SystemDashboardRegistry) dashboard.Module { + return impldashboard.NewModule(impldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, tagModule, systemDashboardRegistry) }, func(_ licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config] { return noopgateway.NewProviderFactory() diff --git a/cmd/enterprise/server.go b/cmd/enterprise/server.go index 3dd09a970a3..db79628b561 100644 --- a/cmd/enterprise/server.go +++ b/cmd/enterprise/server.go @@ -63,6 +63,7 @@ import ( "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/version" "github.com/SigNoz/signoz/pkg/zeus" @@ -136,8 +137,8 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e } return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, licensing, onBeforeRoleDelete, authtypes.NewRegistry()), nil }, - func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing, tagModule tag.Module) dashboard.Module { - return impldashboard.NewModule(pkgimpldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, querier, licensing, tagModule) + func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing, tagModule tag.Module, systemDashboardRegistry dashboardtypes.SystemDashboardRegistry) dashboard.Module { + return impldashboard.NewModule(pkgimpldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, querier, licensing, tagModule, systemDashboardRegistry) }, func(licensing licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config] { return httpgateway.NewProviderFactory(licensing) diff --git a/docs/api/openapi.yml b/docs/api/openapi.yml index ebd5aaf550f..a7b713179ce 100644 --- a/docs/api/openapi.yml +++ b/docs/api/openapi.yml @@ -2944,6 +2944,46 @@ components: publicDashboard: $ref: '#/components/schemas/DashboardtypesGettablePublicDasbhboard' type: object + DashboardtypesGettableSystemDashboard: + properties: + createdAt: + format: date-time + type: string + createdBy: + type: string + image: + type: string + locked: + type: boolean + name: + type: string + orgId: + type: string + schemaVersion: + type: string + source: + $ref: '#/components/schemas/DashboardtypesSource' + spec: + $ref: '#/components/schemas/DashboardtypesDashboardSpec' + tags: + items: + $ref: '#/components/schemas/TagtypesGettableTag' + nullable: true + type: array + updatedAt: + format: date-time + type: string + updatedBy: + type: string + required: + - orgId + - locked + - source + - schemaVersion + - name + - tags + - spec + type: object DashboardtypesHistogramBuckets: properties: bucketCount: @@ -15501,6 +15541,73 @@ paths: summary: Migrate dashboard to v2 tags: - dashboard + /api/v2/dashboards/system/{name}: + get: + deprecated: false + description: Returns a dashboard SigNoz ships and owns, addressed by its stable + definition name (e.g. `ai-o11y-overview`) rather than its id. System dashboards + are read-only and upgraded through releases. The dashboard's own `name` field + carries a reserved prefix that the path segment must not include. + operationId: GetSystemDashboard + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + properties: + data: + $ref: '#/components/schemas/DashboardtypesGettableSystemDashboard' + status: + type: string + required: + - status + - data + type: object + description: OK + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Bad Request + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Forbidden + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Not Found + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Internal Server Error + security: + - api_key: + - dashboard:read + - tokenizer: + - dashboard:read + summary: Get system dashboard + tags: + - dashboard /api/v2/factor_password/forgot: post: deprecated: false diff --git a/ee/modules/dashboard/impldashboard/module.go b/ee/modules/dashboard/impldashboard/module.go index ad89c492c5a..15025de9867 100644 --- a/ee/modules/dashboard/impldashboard/module.go +++ b/ee/modules/dashboard/impldashboard/module.go @@ -32,9 +32,9 @@ type module struct { tagModule tag.Module } -func NewModule(store dashboardtypes.Store, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing, tagModule tag.Module) dashboard.Module { +func NewModule(store dashboardtypes.Store, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing, tagModule tag.Module, systemDashboardRegistry dashboardtypes.SystemDashboardRegistry) dashboard.Module { scopedProviderSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/ee/modules/dashboard/impldashboard") - pkgDashboardModule := pkgimpldashboard.NewModule(store, settings, analytics, orgGetter, queryParser, tagModule) + pkgDashboardModule := pkgimpldashboard.NewModule(store, settings, analytics, orgGetter, queryParser, tagModule, systemDashboardRegistry) return &module{ pkgDashboardModule: pkgDashboardModule, @@ -361,6 +361,14 @@ func (module *module) LockUnlock(ctx context.Context, orgID valuer.UUID, id valu return module.pkgDashboardModule.LockUnlock(ctx, orgID, id, updatedBy, isAdmin, lock) } +func (module *module) ReconcileSystemDashboards(ctx context.Context, orgID valuer.UUID) error { + return module.pkgDashboardModule.ReconcileSystemDashboards(ctx, orgID) +} + +func (module *module) GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.DashboardV2, error) { + return module.pkgDashboardModule.GetSystemDashboard(ctx, orgID, name) +} + func (module *module) delete(ctx context.Context, orgID, id valuer.UUID) error { return module.store.RunInTx(ctx, func(ctx context.Context) error { if err := module.store.DeletePublic(ctx, id.String()); err != nil && !errors.Ast(err, errors.TypeNotFound) { diff --git a/frontend/src/api/generated/services/dashboard/index.ts b/frontend/src/api/generated/services/dashboard/index.ts index 8a3019cb380..6120e805ea9 100644 --- a/frontend/src/api/generated/services/dashboard/index.ts +++ b/frontend/src/api/generated/services/dashboard/index.ts @@ -46,6 +46,8 @@ import type { GetPublicDashboardPathParameters, GetPublicDashboardWidgetQueryRange200, GetPublicDashboardWidgetQueryRangePathParameters, + GetSystemDashboard200, + GetSystemDashboardPathParameters, ListDashboardViews200, ListDashboardsForUserV2200, ListDashboardsForUserV2Params, @@ -1885,6 +1887,108 @@ export const useMigrateDashboardV2 = < > => { return useMutation(getMigrateDashboardV2MutationOptions(options)); }; +/** + * Returns a dashboard SigNoz ships and owns, addressed by its stable definition name (e.g. `ai-o11y-overview`) rather than its id. System dashboards are read-only and upgraded through releases. The dashboard's own `name` field carries a reserved prefix that the path segment must not include. + * @summary Get system dashboard + */ +export const getSystemDashboard = ( + { name }: GetSystemDashboardPathParameters, + signal?: AbortSignal, +) => { + return GeneratedAPIInstance({ + url: `/api/v2/dashboards/system/${name}`, + method: 'GET', + signal, + }); +}; + +export const getGetSystemDashboardQueryKey = ({ + name, +}: GetSystemDashboardPathParameters) => { + return [`/api/v2/dashboards/system/${name}`] as const; +}; + +export const getGetSystemDashboardQueryOptions = < + TData = Awaited>, + TError = ErrorType, +>( + { name }: GetSystemDashboardPathParameters, + options?: { + query?: UseQueryOptions< + Awaited>, + TError, + TData + >; + }, +) => { + const { query: queryOptions } = options ?? {}; + + const queryKey = + queryOptions?.queryKey ?? getGetSystemDashboardQueryKey({ name }); + + const queryFn: QueryFunction< + Awaited> + > = ({ signal }) => getSystemDashboard({ name }, signal); + + return { + queryKey, + queryFn, + enabled: !!name, + ...queryOptions, + } as UseQueryOptions< + Awaited>, + TError, + TData + > & { queryKey: QueryKey }; +}; + +export type GetSystemDashboardQueryResult = NonNullable< + Awaited> +>; +export type GetSystemDashboardQueryError = ErrorType; + +/** + * @summary Get system dashboard + */ + +export function useGetSystemDashboard< + TData = Awaited>, + TError = ErrorType, +>( + { name }: GetSystemDashboardPathParameters, + options?: { + query?: UseQueryOptions< + Awaited>, + TError, + TData + >; + }, +): UseQueryResult & { queryKey: QueryKey } { + const queryOptions = getGetSystemDashboardQueryOptions({ name }, options); + + const query = useQuery(queryOptions) as UseQueryResult & { + queryKey: QueryKey; + }; + + return { ...query, queryKey: queryOptions.queryKey }; +} + +/** + * @summary Get system dashboard + */ +export const invalidateGetSystemDashboard = async ( + queryClient: QueryClient, + { name }: GetSystemDashboardPathParameters, + options?: InvalidateOptions, +): Promise => { + await queryClient.invalidateQueries( + { queryKey: getGetSystemDashboardQueryKey({ name }) }, + options, + ); + + return queryClient; +}; + /** * This endpoint returns the sanitized v2-shape dashboard data for public access. Each panel query is reduced to a safe field subset, so filters and raw query strings are not exposed. * @summary Get public dashboard data (v2) diff --git a/frontend/src/api/generated/services/sigNoz.schemas.ts b/frontend/src/api/generated/services/sigNoz.schemas.ts index 0453f403cbc..2a86ee28f69 100644 --- a/frontend/src/api/generated/services/sigNoz.schemas.ts +++ b/frontend/src/api/generated/services/sigNoz.schemas.ts @@ -4960,6 +4960,53 @@ export interface DashboardtypesGettablePublicDashboardDataV2DTO { publicDashboard?: DashboardtypesGettablePublicDasbhboardDTO; } +export interface DashboardtypesGettableSystemDashboardDTO { + /** + * @type string + * @format date-time + */ + createdAt?: string; + /** + * @type string + */ + createdBy?: string; + /** + * @type string + */ + image?: string; + /** + * @type boolean + */ + locked: boolean; + /** + * @type string + */ + name: string; + /** + * @type string + */ + orgId: string; + /** + * @type string + */ + schemaVersion: string; + source: DashboardtypesSourceDTO; + spec: DashboardtypesDashboardSpecDTO; + /** + * @type array,null + */ + tags: TagtypesGettableTagDTO[] | null; + /** + * @type string + * @format date-time + */ + updatedAt?: string; + /** + * @type string + */ + updatedBy?: string; +} + export enum DashboardtypesPatchOpDTO { add = 'add', remove = 'remove', @@ -11471,6 +11518,17 @@ export type MigrateDashboardV2200 = { status: string; }; +export type GetSystemDashboardPathParameters = { + name: string; +}; +export type GetSystemDashboard200 = { + data: DashboardtypesGettableSystemDashboardDTO; + /** + * @type string + */ + status: string; +}; + export type GetFeatures200 = { /** * @type array diff --git a/pkg/apiserver/signozapiserver/dashboard.go b/pkg/apiserver/signozapiserver/dashboard.go index d84aee62e48..5b963726bba 100644 --- a/pkg/apiserver/signozapiserver/dashboard.go +++ b/pkg/apiserver/signozapiserver/dashboard.go @@ -332,6 +332,33 @@ func (provider *provider) addDashboardRoutes(router *mux.Router) error { return err } + if err := router.Handle("/api/v2/dashboards/system/{name}", handler.New( + provider.authzMiddleware.CheckResources(provider.dashboardHandler.GetSystemDashboard, authtypes.SigNozAdminRoleName, authtypes.SigNozEditorRoleName, authtypes.SigNozViewerRoleName), + handler.OpenAPIDef{ + ID: "GetSystemDashboard", + Tags: []string{"dashboard"}, + Summary: "Get system dashboard", + Description: "Returns a dashboard SigNoz ships and owns, addressed by its stable definition name (e.g. `ai-o11y-overview`) rather than its id. System dashboards are read-only and upgraded through releases. The dashboard's own `name` field carries a reserved prefix that the path segment must not include.", + Request: nil, + RequestContentType: "", + Response: new(dashboardtypes.GettableSystemDashboard), + ResponseContentType: "application/json", + SuccessStatusCode: http.StatusOK, + ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + Deprecated: false, + SecuritySchemes: newScopedSecuritySchemes([]string{coretypes.ResourceMetaResourceDashboard.Scope(coretypes.VerbRead)}), + }, + handler.WithResourceDefs(handler.BasicResourceDef{ + Resource: coretypes.ResourceMetaResourceDashboard, + Verb: coretypes.VerbRead, + Category: coretypes.ActionCategoryDataAccess, + ID: provider.systemDashboardID(), + Selector: coretypes.IDSelector, + }), + )).Methods(http.MethodGet).GetError(); err != nil { + return err + } + // Pinning mutates the calling user's pin list, not the dashboard, so it rides // on the collection-level list permission rather than a per-dashboard check. // The id is still extracted, for audit. @@ -718,3 +745,23 @@ func (provider *provider) addDashboardRoutes(router *mux.Router) error { return nil } + +// systemDashboardID resolves the {name} path param to the dashboard's id. Authz +// tuples and audit records are written against ids, so the name has to be +// resolved before either runs. +func (provider *provider) systemDashboardID() coretypes.ResourceIDExtractor { + return coretypes.NewResourceIDExtractor(coretypes.PhaseRequest, func(ec coretypes.ExtractorContext) (string, error) { + ctx := ec.Request.Context() + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + return "", err + } + + systemDashboard, err := provider.dashboardModule.GetSystemDashboard(ctx, valuer.MustNewUUID(claims.OrgID), mux.Vars(ec.Request)["name"]) + if err != nil { + return "", err + } + + return systemDashboard.ID.StringValue(), nil + }) +} diff --git a/pkg/modules/dashboard/dashboard.go b/pkg/modules/dashboard/dashboard.go index 063c20294ba..f8c33e0f0b0 100644 --- a/pkg/modules/dashboard/dashboard.go +++ b/pkg/modules/dashboard/dashboard.go @@ -99,6 +99,14 @@ type Module interface { DeleteView(ctx context.Context, orgID valuer.UUID, id valuer.UUID) error GetByMetricNamesV2(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]dashboardtypes.DashboardPanelRef, error) + + // ════════════════════════════════════════════════════════════════════════ + // System dashboard methods + // ════════════════════════════════════════════════════════════════════════ + + ReconcileSystemDashboards(ctx context.Context, orgID valuer.UUID) error + + GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.DashboardV2, error) } type Handler interface { @@ -162,4 +170,6 @@ type Handler interface { UpdateView(http.ResponseWriter, *http.Request) DeleteView(http.ResponseWriter, *http.Request) + + GetSystemDashboard(http.ResponseWriter, *http.Request) } diff --git a/pkg/modules/dashboard/impldashboard/fs/definitions/ai-o11y-overview.json b/pkg/modules/dashboard/impldashboard/fs/definitions/ai-o11y-overview.json new file mode 100644 index 00000000000..3a2d8e7aad4 --- /dev/null +++ b/pkg/modules/dashboard/impldashboard/fs/definitions/ai-o11y-overview.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "definition": { + "schemaVersion": "v6", + "name": "signoz---ai-o11y-overview", + "tags": [], + "spec": { + "display": { + "name": "AI Observability Overview", + "description": "Overview of LLM traffic. Panels ship in an upcoming release." + }, + "variables": [], + "panels": {}, + "layouts": [] + } + } +} diff --git a/pkg/modules/dashboard/impldashboard/module.go b/pkg/modules/dashboard/impldashboard/module.go index 2e29754e0ab..3b322026709 100644 --- a/pkg/modules/dashboard/impldashboard/module.go +++ b/pkg/modules/dashboard/impldashboard/module.go @@ -21,23 +21,25 @@ import ( ) type module struct { - store dashboardtypes.Store - settings factory.ScopedProviderSettings - analytics analytics.Analytics - orgGetter organization.Getter - queryParser queryparser.QueryParser - tagModule tag.Module + store dashboardtypes.Store + settings factory.ScopedProviderSettings + analytics analytics.Analytics + orgGetter organization.Getter + queryParser queryparser.QueryParser + tagModule tag.Module + systemDashboardRegistry dashboardtypes.SystemDashboardRegistry } -func NewModule(store dashboardtypes.Store, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, tagModule tag.Module) dashboard.Module { +func NewModule(store dashboardtypes.Store, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, tagModule tag.Module, systemDashboardRegistry dashboardtypes.SystemDashboardRegistry) dashboard.Module { scopedProviderSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard") return &module{ - store: store, - settings: scopedProviderSettings, - analytics: analytics, - orgGetter: orgGetter, - queryParser: queryParser, - tagModule: tagModule, + store: store, + settings: scopedProviderSettings, + analytics: analytics, + orgGetter: orgGetter, + queryParser: queryParser, + tagModule: tagModule, + systemDashboardRegistry: systemDashboardRegistry, } } diff --git a/pkg/modules/dashboard/impldashboard/store.go b/pkg/modules/dashboard/impldashboard/store.go index 5a7391fc6b6..056e0f7894a 100644 --- a/pkg/modules/dashboard/impldashboard/store.go +++ b/pkg/modules/dashboard/impldashboard/store.go @@ -3,6 +3,7 @@ package impldashboard import ( "context" "strings" + "time" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/sqlstore" @@ -64,6 +65,23 @@ func (store *store) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID) return storableDashboard, nil } +func (store *store) GetByName(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.StorableDashboard, error) { + storableDashboard := new(dashboardtypes.StorableDashboard) + err := store. + sqlstore. + BunDB(). + NewSelect(). + Model(storableDashboard). + Where("name = ?", name). + Where("org_id = ?", orgID). + Scan(ctx) + if err != nil { + return nil, store.sqlstore.WrapNotFoundErrf(err, errors.CodeNotFound, "dashboard with name %s doesn't exist", name) + } + + return storableDashboard, nil +} + // ListForUser emits the joined dashboard ⨝ user_dashboard_preference query the // spec calls for. Aliases: // @@ -613,3 +631,60 @@ func (store *store) DeleteDashboardView(ctx context.Context, orgID valuer.UUID, } return nil } + +func (store *store) CreateSystemDashboard(ctx context.Context, storable *dashboardtypes.StorableSystemDashboard) error { + _, err := store. + sqlstore. + BunDBCtx(ctx). + NewInsert(). + Model(storable). + Exec(ctx) + if err != nil { + return store.sqlstore.WrapAlreadyExistsErrf(err, dashboardtypes.ErrCodeSystemDashboardAlreadyProvisioned, "system dashboard %s is already provisioned", storable.Name) + } + + return nil +} + +func (store *store) GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.StorableSystemDashboard, error) { + storable := new(dashboardtypes.StorableSystemDashboard) + err := store. + sqlstore. + BunDBCtx(ctx). + NewSelect(). + Model(storable). + Where("org_id = ?", orgID). + Where("name = ?", name). + Scan(ctx) + if err != nil { + return nil, store.sqlstore.WrapNotFoundErrf(err, dashboardtypes.ErrCodeSystemDashboardNotFound, "system dashboard %s is not provisioned", name) + } + + return storable, nil +} + +func (store *store) UpdateSystemDashboardVersion(ctx context.Context, orgID valuer.UUID, name string, version int) error { + result, err := store. + sqlstore. + BunDBCtx(ctx). + NewUpdate(). + Model(new(dashboardtypes.StorableSystemDashboard)). + Set("version = ?", version). + Set("updated_at = ?", time.Now()). + Where("org_id = ?", orgID). + Where("name = ?", name). + Exec(ctx) + if err != nil { + return err + } + + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return errors.Newf(errors.TypeNotFound, dashboardtypes.ErrCodeSystemDashboardNotFound, "system dashboard %s is not provisioned", name) + } + + return nil +} diff --git a/pkg/modules/dashboard/impldashboard/system_dashboard_definitions.go b/pkg/modules/dashboard/impldashboard/system_dashboard_definitions.go new file mode 100644 index 00000000000..216732577ca --- /dev/null +++ b/pkg/modules/dashboard/impldashboard/system_dashboard_definitions.go @@ -0,0 +1,46 @@ +package impldashboard + +import ( + "embed" + "io/fs" + "path" + + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" +) + +const definitionsRoot = "fs/definitions" + +//go:embed fs/definitions/*.json +var definitionFiles embed.FS + +// NewSystemDashboardRegistry parses every embedded definition. Definitions are +// build-time assets validated by a test, so a failure here means the binary +// shipped broken JSON. +func NewSystemDashboardRegistry() (dashboardtypes.SystemDashboardRegistry, error) { + entries, err := fs.ReadDir(definitionFiles, definitionsRoot) + if err != nil { + return dashboardtypes.SystemDashboardRegistry{}, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read system dashboard definitions") + } + + definitions := make([]dashboardtypes.SystemDashboardDefinition, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + + file := path.Join(definitionsRoot, entry.Name()) + raw, err := definitionFiles.ReadFile(file) + if err != nil { + return dashboardtypes.SystemDashboardRegistry{}, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read %s", file) + } + + definition, err := dashboardtypes.NewSystemDashboardDefinition(raw) + if err != nil { + return dashboardtypes.SystemDashboardRegistry{}, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "couldn't parse %s", file) + } + definitions = append(definitions, definition) + } + + return dashboardtypes.NewSystemDashboardRegistry(definitions) +} diff --git a/pkg/modules/dashboard/impldashboard/system_dashboard_service.go b/pkg/modules/dashboard/impldashboard/system_dashboard_service.go new file mode 100644 index 00000000000..9ea0df23262 --- /dev/null +++ b/pkg/modules/dashboard/impldashboard/system_dashboard_service.go @@ -0,0 +1,81 @@ +package impldashboard + +import ( + "context" + "log/slog" + "time" + + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/modules/dashboard" + "github.com/SigNoz/signoz/pkg/modules/organization" +) + +const reconcileRetryInterval = 30 * time.Second + +type service struct { + settings factory.ScopedProviderSettings + module dashboard.Module + orgGetter organization.Getter + stopC chan struct{} + healthyC chan struct{} +} + +// NewService reconciles every org's system dashboards once at startup. Orgs +// created later are reconciled by the organization setter instead. +func NewService(providerSettings factory.ProviderSettings, module dashboard.Module, orgGetter organization.Getter) factory.Service { + return &service{ + settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"), + module: module, + orgGetter: orgGetter, + stopC: make(chan struct{}), + healthyC: make(chan struct{}), + } +} + +func (service *service) Start(ctx context.Context) error { + ticker := time.NewTicker(reconcileRetryInterval) + defer ticker.Stop() + + for { + err := service.reconcile(ctx) + if err == nil { + close(service.healthyC) + <-service.stopC + return nil + } + + service.settings.Logger().WarnContext(ctx, "system dashboard reconciliation failed, retrying", errors.Attr(err)) + + select { + case <-service.stopC: + return nil + case <-ticker.C: + } + } +} + +func (service *service) Healthy() <-chan struct{} { + return service.healthyC +} + +func (service *service) Stop(_ context.Context) error { + close(service.stopC) + return nil +} + +func (service *service) reconcile(ctx context.Context) error { + orgs, err := service.orgGetter.ListByOwnedKeyRange(ctx) + if err != nil { + return err + } + + for _, org := range orgs { + if err := service.module.ReconcileSystemDashboards(ctx, org.ID); err != nil { + return errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "couldn't reconcile system dashboards for org %s", org.ID.StringValue()) + } + } + + service.settings.Logger().InfoContext(ctx, "system dashboard reconciliation completed", slog.Int("orgs", len(orgs))) + return nil +} diff --git a/pkg/modules/dashboard/impldashboard/v2_handler.go b/pkg/modules/dashboard/impldashboard/v2_handler.go index 5d99ac3d848..83034daa970 100644 --- a/pkg/modules/dashboard/impldashboard/v2_handler.go +++ b/pkg/modules/dashboard/impldashboard/v2_handler.go @@ -502,3 +502,28 @@ func (handler *handler) GetPublicWidgetQueryRangeV2(rw http.ResponseWriter, r *h render.Success(rw, http.StatusOK, queryRangeResults) } + +func (handler *handler) GetSystemDashboard(rw http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + render.Error(rw, err) + return + } + + name := mux.Vars(r)["name"] + if name == "" { + render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "name is missing in the path")) + return + } + + systemDashboard, err := handler.module.GetSystemDashboard(ctx, valuer.MustNewUUID(claims.OrgID), name) + if err != nil { + render.Error(rw, err) + return + } + + render.Success(rw, http.StatusOK, systemDashboard.ToGettableSystemDashboard()) +} diff --git a/pkg/modules/dashboard/impldashboard/v2_module.go b/pkg/modules/dashboard/impldashboard/v2_module.go index 95d3e97ce6f..74b43be5147 100644 --- a/pkg/modules/dashboard/impldashboard/v2_module.go +++ b/pkg/modules/dashboard/impldashboard/v2_module.go @@ -2,6 +2,8 @@ package impldashboard import ( "context" + "log/slog" + "strings" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/transition" @@ -19,9 +21,12 @@ func (m *module) CreateV2(ctx context.Context, orgID valuer.UUID, createdBy stri return nil, err } - dashboard := postable.NewDashboardV2(orgID, createdBy, source) + dashboard, err := postable.NewDashboardV2(orgID, createdBy, source) + if err != nil { + return nil, err + } - err := m.store.RunInTx(ctx, func(ctx context.Context) error { + err = m.store.RunInTx(ctx, func(ctx context.Context) error { resolvedTags, err := m.tagModule.SyncTags(ctx, orgID, coretypes.KindDashboard, dashboard.ID, postable.Tags) if err != nil { return err @@ -120,6 +125,20 @@ func (module *module) GetV2(ctx context.Context, orgID valuer.UUID, id valuer.UU return storable.ToDashboardV2(tags) } +func (module *module) getByNameV2(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.DashboardV2, error) { + storable, err := module.store.GetByName(ctx, orgID, name) + if err != nil { + return nil, err + } + + tags, err := module.tagModule.ListForResource(ctx, orgID, coretypes.KindDashboard, storable.ID) + if err != nil { + return nil, err + } + + return storable.ToDashboardV2(tags) +} + // MigrateV2 retries the v1→v2 migration on a dashboard still stored as v1 (one the // bulk 103 migration skipped or failed). Idempotent: an already-v2 one is unchanged. func (module *module) MigrateV2(ctx context.Context, orgID valuer.UUID, id valuer.UUID) (*dashboardtypes.DashboardV2, error) { @@ -179,13 +198,33 @@ func (module *module) UpdateV2(ctx context.Context, orgID valuer.UUID, id valuer return nil, err } - err = module.store.RunInTx(ctx, func(ctx context.Context) error { - resolvedTags, err := module.tagModule.SyncTags(ctx, orgID, coretypes.KindDashboard, id, updatable.Tags) + return module.updateV2(ctx, orgID, existing, updatedBy, updatable, existing.Update) +} + +// updateUnsafeV2 updates a dashboard bypassing the guards. Intended for internal system callers. +func (module *module) updateUnsafeV2(ctx context.Context, orgID valuer.UUID, id valuer.UUID, updatedBy string, updatable dashboardtypes.UpdatableDashboardV2) (*dashboardtypes.DashboardV2, error) { + if err := updatable.Validate(); err != nil { + return nil, err + } + + existing, err := module.GetV2(ctx, orgID, id) + if err != nil { + return nil, err + } + + return module.updateV2(ctx, orgID, existing, updatedBy, updatable, existing.UpdateUnsafe) +} + +// apply is existing.Update or existing.UpdateUnsafe, so the gated path keeps its +// in-transaction checks and only updateUnsafeV2 skips them. +func (module *module) updateV2(ctx context.Context, orgID valuer.UUID, existing *dashboardtypes.DashboardV2, updatedBy string, updatable dashboardtypes.UpdatableDashboardV2, apply func(dashboardtypes.UpdatableDashboardV2, string, []*tagtypes.Tag) error) (*dashboardtypes.DashboardV2, error) { + err := module.store.RunInTx(ctx, func(ctx context.Context) error { + resolvedTags, err := module.tagModule.SyncTags(ctx, orgID, coretypes.KindDashboard, existing.ID, updatable.Tags) if err != nil { return err } - err = existing.Update(updatable, updatedBy, resolvedTags) + err = apply(updatable, updatedBy, resolvedTags) if err != nil { return err } @@ -296,3 +335,98 @@ func (module *module) UnpinV2(ctx context.Context, orgID valuer.UUID, userID val func (module *module) DeletePreferencesForUser(ctx context.Context, orgID valuer.UUID, userID valuer.UUID) error { return module.store.DeletePreferencesForUser(ctx, orgID, userID) } + +func (m *module) ReconcileSystemDashboards(ctx context.Context, orgID valuer.UUID) error { + for _, definition := range m.systemDashboardRegistry.List() { + if err := m.reconcileSystemDashboard(ctx, orgID, definition); err != nil { + return err + } + } + + return nil +} + +func (m *module) reconcileSystemDashboard(ctx context.Context, orgID valuer.UUID, definition dashboardtypes.SystemDashboardDefinition) error { + existing, err := m.getByNameV2(ctx, orgID, definition.Name()) + if err != nil { + if !errors.Ast(err, errors.TypeNotFound) { + return err + } + return m.provisionSystemDashboard(ctx, orgID, definition) + } + + state, err := m.store.GetSystemDashboard(ctx, orgID, definition.Name()) + if err != nil { + return err + } + // Only ever move forward: a downgrade must not rewrite the newer content. + if state.Version >= definition.Version { + return nil + } + + return m.upgradeSystemDashboard(ctx, orgID, existing.ID, definition) +} + +// provisionSystemDashboard creates the dashboard and its state row in one transaction, +// so a system dashboard can never exist without the version it was provisioned at. +// A concurrent provisioner (another replica, or the org-creation hook racing the +// startup sweep) loses on the state row's unique (org_id, name) index and rolls back. +func (m *module) provisionSystemDashboard(ctx context.Context, orgID valuer.UUID, definition dashboardtypes.SystemDashboardDefinition) error { + err := m.store.RunInTx(ctx, func(ctx context.Context) error { + created, err := m.CreateV2( + ctx, + orgID, + dashboardtypes.ProvisionerIdentity, + valuer.UUID{}, + dashboardtypes.SourceSystem, + definition.Dashboard, + ) + if err != nil { + return err + } + + return m.store.CreateSystemDashboard(ctx, dashboardtypes.NewStorableSystemDashboard(orgID, created.ID, definition.Name(), definition.Version)) + }) + if err != nil { + if errors.Ast(err, errors.TypeAlreadyExists) { + m.settings.Logger().DebugContext(ctx, "system dashboard already provisioned concurrently", slog.String("name", definition.Name()), slog.String("org_id", orgID.StringValue())) + return nil + } + return err + } + + m.settings.Logger().InfoContext(ctx, "provisioned system dashboard", slog.String("name", definition.Name()), slog.Int("version", definition.Version), slog.String("org_id", orgID.StringValue())) + return nil +} + +func (m *module) upgradeSystemDashboard(ctx context.Context, orgID valuer.UUID, id valuer.UUID, definition dashboardtypes.SystemDashboardDefinition) error { + err := m.store.RunInTx(ctx, func(ctx context.Context) error { + if _, err := m.updateUnsafeV2(ctx, orgID, id, dashboardtypes.ProvisionerIdentity, definition.ToUpdatable()); err != nil { + return err + } + + return m.store.UpdateSystemDashboardVersion(ctx, orgID, definition.Name(), definition.Version) + }) + if err != nil { + return err + } + + m.settings.Logger().InfoContext(ctx, "upgraded system dashboard", slog.String("name", definition.Name()), slog.Int("version", definition.Version), slog.String("org_id", orgID.StringValue())) + return nil +} + +func (m *module) GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*dashboardtypes.DashboardV2, error) { + if strings.HasPrefix(name, dashboardtypes.SystemDashboardNamePrefix) { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "name must not carry the %q prefix", dashboardtypes.SystemDashboardNamePrefix) + } + + existing, err := m.getByNameV2(ctx, orgID, dashboardtypes.SystemDashboardNamePrefix+name) + if err != nil { + return nil, err + } + if err := existing.ErrIfNotSystem(); err != nil { + return nil, err + } + + return existing, nil +} diff --git a/pkg/modules/dashboard/impldashboard/v2_module_test.go b/pkg/modules/dashboard/impldashboard/v2_module_test.go new file mode 100644 index 00000000000..1c020dd2baa --- /dev/null +++ b/pkg/modules/dashboard/impldashboard/v2_module_test.go @@ -0,0 +1,191 @@ +package impldashboard + +import ( + "context" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/SigNoz/signoz/pkg/analytics/analyticstest" + "github.com/SigNoz/signoz/pkg/factory/factorytest" + "github.com/SigNoz/signoz/pkg/modules/tag/impltag" + "github.com/SigNoz/signoz/pkg/queryparser" + "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" + "github.com/SigNoz/signoz/pkg/types/tagtypes" + "github.com/SigNoz/signoz/pkg/valuer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testDashboardName = "test-overview" + +func newTestSQLStore(t *testing.T) sqlstore.SQLStore { + t.Helper() + + store, err := sqlitesqlstore.New(context.Background(), factorytest.NewSettings(), sqlstore.Config{ + Provider: "sqlite", + Connection: sqlstore.ConnectionConfig{MaxOpenConns: 10}, + Sqlite: sqlstore.SqliteConfig{ + Path: filepath.Join(t.TempDir(), "test.db"), + Mode: "wal", + BusyTimeout: 5 * time.Second, + TransactionMode: "deferred", + }, + }) + require.NoError(t, err) + + for _, model := range []any{ + (*dashboardtypes.StorableDashboard)(nil), + (*tagtypes.Tag)(nil), + (*tagtypes.TagRelation)(nil), + (*dashboardtypes.StorableSystemDashboard)(nil), + } { + _, err := store.BunDB().NewCreateTable().Model(model).IfNotExists().Exec(context.Background()) + require.NoError(t, err) + } + + _, err = store.BunDB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_system_dashboard_org_name ON system_dashboard (org_id, name)`) + require.NoError(t, err) + + return store +} + +func newTestModule(t *testing.T, sqlStore sqlstore.SQLStore, definitions ...dashboardtypes.SystemDashboardDefinition) *module { + t.Helper() + + registry, err := dashboardtypes.NewSystemDashboardRegistry(definitions) + require.NoError(t, err) + + providerSettings := factorytest.NewSettings() + return NewModule( + NewStore(sqlStore), + providerSettings, + analyticstest.New(), + nil, + queryparser.New(providerSettings), + impltag.NewModule(impltag.NewStore(sqlStore)), + registry, + ).(*module) +} + +func newTestDefinition(t *testing.T, version int, displayName string) dashboardtypes.SystemDashboardDefinition { + t.Helper() + + raw := `{ + "version": ` + strconv.Itoa(version) + `, + "definition": { + "schemaVersion": "` + dashboardtypes.SchemaVersion + `", + "name": "` + dashboardtypes.SystemDashboardNamePrefix + testDashboardName + `", + "tags": [], + "spec": {"display": {"name": "` + displayName + `"}, "variables": [], "panels": {}, "layouts": []} + } + }` + + definition, err := dashboardtypes.NewSystemDashboardDefinition([]byte(raw)) + require.NoError(t, err) + + return definition +} + +func TestReconcileProvisionsThenUpgrades(t *testing.T) { + ctx := context.Background() + sqlStore := newTestSQLStore(t) + orgID := valuer.GenerateUUID() + + dashboardModule := newTestModule(t, sqlStore, newTestDefinition(t, 1, "v1")) + require.NoError(t, dashboardModule.ReconcileSystemDashboards(ctx, orgID)) + + provisioned, err := dashboardModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + assert.Equal(t, dashboardtypes.SourceSystem, provisioned.Source) + assert.Equal(t, dashboardtypes.ProvisionerIdentity, provisioned.CreatedBy) + assert.Equal(t, "v1", provisioned.Spec.Display.Name) + assert.Equal(t, 1, stateVersion(t, dashboardModule, ctx, orgID)) + + // Reconciling the same version again is a no-op. + require.NoError(t, dashboardModule.ReconcileSystemDashboards(ctx, orgID)) + unchanged, err := dashboardModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + assert.Equal(t, provisioned.UpdatedAt, unchanged.UpdatedAt) + + // An unmodified copy is upgraded in place, keeping its id. + upgradingModule := newTestModule(t, sqlStore, newTestDefinition(t, 2, "v2")) + require.NoError(t, upgradingModule.ReconcileSystemDashboards(ctx, orgID)) + + upgraded, err := upgradingModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + assert.Equal(t, provisioned.ID, upgraded.ID) + assert.Equal(t, "v2", upgraded.Spec.Display.Name) + assert.Equal(t, 2, stateVersion(t, upgradingModule, ctx, orgID)) +} + +func stateVersion(t *testing.T, module *module, ctx context.Context, orgID valuer.UUID) int { + t.Helper() + + state, err := module.store.GetSystemDashboard(ctx, orgID, dashboardtypes.SystemDashboardNamePrefix+testDashboardName) + require.NoError(t, err) + + return state.Version +} + +func TestSystemDashboardsAreImmutableToUsers(t *testing.T) { + ctx := context.Background() + sqlStore := newTestSQLStore(t) + orgID := valuer.GenerateUUID() + + dashboardModule := newTestModule(t, sqlStore, newTestDefinition(t, 1, "v1")) + require.NoError(t, dashboardModule.ReconcileSystemDashboards(ctx, orgID)) + + provisioned, err := dashboardModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + + _, err = dashboardModule.UpdateV2(ctx, orgID, provisioned.ID, "user@signoz.io", newTestDefinition(t, 1, "edited").ToUpdatable()) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be modified") +} + +func TestReconcileDoesNotDowngrade(t *testing.T) { + ctx := context.Background() + sqlStore := newTestSQLStore(t) + orgID := valuer.GenerateUUID() + + newerModule := newTestModule(t, sqlStore, newTestDefinition(t, 3, "v3")) + require.NoError(t, newerModule.ReconcileSystemDashboards(ctx, orgID)) + + olderModule := newTestModule(t, sqlStore, newTestDefinition(t, 2, "v2")) + require.NoError(t, olderModule.ReconcileSystemDashboards(ctx, orgID)) + + got, err := newerModule.GetSystemDashboard(ctx, orgID, testDashboardName) + require.NoError(t, err) + assert.Equal(t, "v3", got.Spec.Display.Name) + assert.Equal(t, 3, stateVersion(t, newerModule, ctx, orgID)) +} + +func TestGetRejectsANonSystemDashboard(t *testing.T) { + ctx := context.Background() + sqlStore := newTestSQLStore(t) + orgID := valuer.GenerateUUID() + + dashboardModule := newTestModule(t, sqlStore) + + var postable dashboardtypes.PostableDashboardV2 + require.NoError(t, postable.UnmarshalJSON([]byte(`{ + "schemaVersion": "`+dashboardtypes.SchemaVersion+`", + "name": "a-user-dashboard", + "tags": [], + "spec": {"display": {"name": "user"}, "variables": [], "panels": {}, "layouts": []} + }`))) + _, err := dashboardModule.CreateV2(ctx, orgID, "user@signoz.io", valuer.GenerateUUID(), dashboardtypes.SourceUser, postable) + require.NoError(t, err) + + // The server-side prefix makes user names structurally unreachable here. + _, err = dashboardModule.GetSystemDashboard(ctx, orgID, "a-user-dashboard") + require.Error(t, err) + + _, err = dashboardModule.GetSystemDashboard(ctx, orgID, dashboardtypes.SystemDashboardNamePrefix+testDashboardName) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not carry") +} diff --git a/pkg/modules/organization/implorganization/setter.go b/pkg/modules/organization/implorganization/setter.go index 33e24c1d4c1..56e7805021f 100644 --- a/pkg/modules/organization/implorganization/setter.go +++ b/pkg/modules/organization/implorganization/setter.go @@ -4,6 +4,7 @@ import ( "context" "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/modules/dashboard" "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/modules/quickfilter" "github.com/SigNoz/signoz/pkg/types" @@ -14,10 +15,11 @@ type setter struct { store types.OrganizationStore alertmanager alertmanager.Alertmanager quickfilter quickfilter.Module + dashboard dashboard.Module } -func NewSetter(store types.OrganizationStore, alertmanager alertmanager.Alertmanager, quickfilter quickfilter.Module) organization.Setter { - return &setter{store: store, alertmanager: alertmanager, quickfilter: quickfilter} +func NewSetter(store types.OrganizationStore, alertmanager alertmanager.Alertmanager, quickfilter quickfilter.Module, dashboard dashboard.Module) organization.Setter { + return &setter{store: store, alertmanager: alertmanager, quickfilter: quickfilter, dashboard: dashboard} } func (module *setter) Create(ctx context.Context, organization *types.Organization, createManagedRoles func(context.Context, valuer.UUID) error) error { @@ -37,6 +39,10 @@ func (module *setter) Create(ctx context.Context, organization *types.Organizati return err } + if err := module.dashboard.ReconcileSystemDashboards(ctx, organization.ID); err != nil { + return err + } + return nil } diff --git a/pkg/signoz/handler_test.go b/pkg/signoz/handler_test.go index 5d88a12b9e0..9b53f3dedd5 100644 --- a/pkg/signoz/handler_test.go +++ b/pkg/signoz/handler_test.go @@ -49,7 +49,9 @@ func TestNewHandlers(t *testing.T) { queryParser := queryparser.New(providerSettings) require.NoError(t, err) tagModule := impltag.NewModule(impltag.NewStore(sqlstore)) - dashboardModule := impldashboard.NewModule(impldashboard.NewStore(sqlstore), providerSettings, nil, orgGetter, queryParser, tagModule) + systemDashboardRegistry, err := impldashboard.NewSystemDashboardRegistry() + require.NoError(t, err) + dashboardModule := impldashboard.NewModule(impldashboard.NewStore(sqlstore), providerSettings, nil, orgGetter, queryParser, tagModule, systemDashboardRegistry) flagger, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry()) require.NoError(t, err) diff --git a/pkg/signoz/module.go b/pkg/signoz/module.go index 7baf3c3a1aa..eaa37a68dd9 100644 --- a/pkg/signoz/module.go +++ b/pkg/signoz/module.go @@ -67,35 +67,35 @@ import ( ) type Modules struct { - OrgGetter organization.Getter - OrgSetter organization.Setter - Preference preference.Module - UserSetter user.Setter - UserGetter user.Getter - RetentionGetter retention.Getter - SavedView savedview.Module - Apdex apdex.Module - Dashboard dashboard.Module - QuickFilter quickfilter.Module - TraceFunnel tracefunnel.Module - RawDataExport rawdataexport.Module - AuthDomain authdomain.Module - Session session.Module - Services services.Module - SpanPercentile spanpercentile.Module - MetricsExplorer metricsexplorer.Module - MetricReductionRule metricreductionrule.Module - InfraMonitoring inframonitoring.Module + OrgGetter organization.Getter + OrgSetter organization.Setter + Preference preference.Module + UserSetter user.Setter + UserGetter user.Getter + RetentionGetter retention.Getter + SavedView savedview.Module + Apdex apdex.Module + Dashboard dashboard.Module + QuickFilter quickfilter.Module + TraceFunnel tracefunnel.Module + RawDataExport rawdataexport.Module + AuthDomain authdomain.Module + Session session.Module + Services services.Module + SpanPercentile spanpercentile.Module + MetricsExplorer metricsexplorer.Module + MetricReductionRule metricreductionrule.Module + InfraMonitoring inframonitoring.Module Promote promote.Module ServiceAccount serviceaccount.Module ServiceAccountGetter serviceaccount.Getter CloudIntegration cloudintegration.Module - LogsPipeline logspipeline.Module - RuleStateHistory rulestatehistory.Module - TraceDetail tracedetail.Module - SpanMapper spanmapper.Module - LLMPricingRule llmpricingrule.Module - Tag tag.Module + LogsPipeline logspipeline.Module + RuleStateHistory rulestatehistory.Module + TraceDetail tracedetail.Module + SpanMapper spanmapper.Module + LLMPricingRule llmpricingrule.Module + Tag tag.Module } func NewModules( @@ -126,7 +126,7 @@ func NewModules( metricReductionRule metricreductionrule.Module, ) Modules { quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore)) - orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter) + orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter, dashboard) // Cleanup callbacks from other modules, invoked when a user is deleted. onDeleteUser := []user.OnDeleteUser{ dashboard.DeletePreferencesForUser, @@ -136,34 +136,34 @@ func NewModules( authDomainModule := implauthdomain.NewModule(implauthdomain.NewStore(sqlstore), authNs, authz) return Modules{ - OrgGetter: orgGetter, - OrgSetter: orgSetter, - Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewAvailablePreference()), - SavedView: implsavedview.NewModule(implsavedview.NewStore(sqlstore)), - Apdex: implapdex.NewModule(sqlstore), - Dashboard: dashboard, - UserSetter: userSetter, - UserGetter: userGetter, - RetentionGetter: retentionGetter, - QuickFilter: quickfilter, - TraceFunnel: impltracefunnel.NewModule(impltracefunnel.NewStore(sqlstore)), - RawDataExport: implrawdataexport.NewModule(querier), - AuthDomain: authDomainModule, - Session: implsession.NewModule(providerSettings, authNs, userSetter, userGetter, authDomainModule, tokenizer, orgGetter, authz, config.Global), - SpanPercentile: implspanpercentile.NewModule(querier, providerSettings), - Services: implservices.NewModule(querier, telemetryStore), - MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, fl, providerSettings, config.MetricsExplorer), - MetricReductionRule: metricReductionRule, - InfraMonitoring: implinframonitoring.NewModule(telemetryStore, telemetryMetadataStore, querier, fl, providerSettings, config.InfraMonitoring), - Promote: implpromote.NewModule(telemetryMetadataStore, telemetryStore), + OrgGetter: orgGetter, + OrgSetter: orgSetter, + Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewAvailablePreference()), + SavedView: implsavedview.NewModule(implsavedview.NewStore(sqlstore)), + Apdex: implapdex.NewModule(sqlstore), + Dashboard: dashboard, + UserSetter: userSetter, + UserGetter: userGetter, + RetentionGetter: retentionGetter, + QuickFilter: quickfilter, + TraceFunnel: impltracefunnel.NewModule(impltracefunnel.NewStore(sqlstore)), + RawDataExport: implrawdataexport.NewModule(querier), + AuthDomain: authDomainModule, + Session: implsession.NewModule(providerSettings, authNs, userSetter, userGetter, authDomainModule, tokenizer, orgGetter, authz, config.Global), + SpanPercentile: implspanpercentile.NewModule(querier, providerSettings), + Services: implservices.NewModule(querier, telemetryStore), + MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, fl, providerSettings, config.MetricsExplorer), + MetricReductionRule: metricReductionRule, + InfraMonitoring: implinframonitoring.NewModule(telemetryStore, telemetryMetadataStore, querier, fl, providerSettings, config.InfraMonitoring), + Promote: implpromote.NewModule(telemetryMetadataStore, telemetryStore), ServiceAccount: serviceAccount, ServiceAccountGetter: serviceAccountGetter, - LogsPipeline: impllogspipeline.NewModule(sqlstore), - RuleStateHistory: implrulestatehistory.NewModule(implrulestatehistory.NewStore(telemetryStore, telemetryMetadataStore, providerSettings.Logger), ruleStore), - CloudIntegration: cloudIntegrationModule, - TraceDetail: impltracedetail.NewModule(impltracedetail.NewTraceStore(telemetryStore), providerSettings, config.TraceDetail), - SpanMapper: implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), fl), - LLMPricingRule: impllmpricingrule.NewModule(impllmpricingrule.NewStore(sqlstore), fl, querier), - Tag: tagModule, + LogsPipeline: impllogspipeline.NewModule(sqlstore), + RuleStateHistory: implrulestatehistory.NewModule(implrulestatehistory.NewStore(telemetryStore, telemetryMetadataStore, providerSettings.Logger), ruleStore), + CloudIntegration: cloudIntegrationModule, + TraceDetail: impltracedetail.NewModule(impltracedetail.NewTraceStore(telemetryStore), providerSettings, config.TraceDetail), + SpanMapper: implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), fl), + LLMPricingRule: impllmpricingrule.NewModule(impllmpricingrule.NewStore(sqlstore), fl, querier), + Tag: tagModule, } } diff --git a/pkg/signoz/module_test.go b/pkg/signoz/module_test.go index 3a147716d87..82e843664c7 100644 --- a/pkg/signoz/module_test.go +++ b/pkg/signoz/module_test.go @@ -51,7 +51,9 @@ func TestNewModules(t *testing.T) { queryParser := queryparser.New(providerSettings) require.NoError(t, err) tagModule := impltag.NewModule(impltag.NewStore(sqlstore)) - dashboardModule := impldashboard.NewModule(impldashboard.NewStore(sqlstore), providerSettings, nil, orgGetter, queryParser, tagModule) + systemDashboardRegistry, err := impldashboard.NewSystemDashboardRegistry() + require.NoError(t, err) + dashboardModule := impldashboard.NewModule(impldashboard.NewStore(sqlstore), providerSettings, nil, orgGetter, queryParser, tagModule, systemDashboardRegistry) flagger, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry()) require.NoError(t, err) diff --git a/pkg/signoz/provider.go b/pkg/signoz/provider.go index e5dcbe0f093..878d1debb40 100644 --- a/pkg/signoz/provider.go +++ b/pkg/signoz/provider.go @@ -245,6 +245,7 @@ func NewSQLMigrationProviderFactories( sqlmigration.NewMigrateLambdaDashboardsFactory(), sqlmigration.NewAddAuthDomainTuplesFactory(sqlstore), sqlmigration.NewAddDeploymentHostTuplesFactory(sqlstore), + sqlmigration.NewAddSystemDashboardFactory(sqlstore, sqlschema), ) } diff --git a/pkg/signoz/signoz.go b/pkg/signoz/signoz.go index 95f4a74487b..b11e3f74d39 100644 --- a/pkg/signoz/signoz.go +++ b/pkg/signoz/signoz.go @@ -60,6 +60,7 @@ import ( "github.com/SigNoz/signoz/pkg/telemetrystore" pkgtokenizer "github.com/SigNoz/signoz/pkg/tokenizer" "github.com/SigNoz/signoz/pkg/types/authtypes" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/version" @@ -175,7 +176,7 @@ func New( telemetrystoreProviderFactories factory.NamedMap[factory.ProviderFactory[telemetrystore.TelemetryStore, telemetrystore.Config]], authNsCallback func(ctx context.Context, providerSettings factory.ProviderSettings, store authtypes.AuthNStore, licensing licensing.Licensing) (map[authtypes.AuthNProvider]authn.AuthN, error), authzCallback func(context.Context, sqlstore.SQLStore, authz.Config, licensing.Licensing, []authz.OnBeforeRoleDelete) (factory.ProviderFactory[authz.AuthZ, authz.Config], error), - dashboardModuleCallback func(sqlstore.SQLStore, factory.ProviderSettings, analytics.Analytics, organization.Getter, queryparser.QueryParser, querier.Querier, licensing.Licensing, tag.Module) dashboard.Module, + dashboardModuleCallback func(sqlstore.SQLStore, factory.ProviderSettings, analytics.Analytics, organization.Getter, queryparser.QueryParser, querier.Querier, licensing.Licensing, tag.Module, dashboardtypes.SystemDashboardRegistry) dashboard.Module, gatewayProviderFactory func(licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config], auditorProviderFactories func(licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]], meterReporterProviderFactories func(context.Context, factory.ProviderSettings, flagger.Flagger, licensing.Licensing, telemetrystore.TelemetryStore, retention.Getter, organization.Getter, zeus.Zeus) (factory.NamedMap[factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config]], string), @@ -440,8 +441,13 @@ func New( // Initialize query parser (needed for dashboard module) queryParser := queryparser.New(providerSettings) - // Initialize dashboard module - dashboard := dashboardModuleCallback(sqlstore, providerSettings, analytics, orgGetter, queryParser, querier, licensing, tagModule) + // Initialize dashboard module. The system dashboard registry is parsed here so + // a malformed embedded definition fails startup instead of a request. + systemDashboardRegistry, err := impldashboard.NewSystemDashboardRegistry() + if err != nil { + return nil, err + } + dashboard := dashboardModuleCallback(sqlstore, providerSettings, analytics, orgGetter, queryParser, querier, licensing, tagModule, systemDashboardRegistry) // Initialize user getter userGetter := impluser.NewGetter(userStore, userRoleStore, flagger) @@ -610,6 +616,7 @@ func New( factory.NewNamedService(factory.MustNewName("auditor"), auditor), factory.NewNamedService(factory.MustNewName("meterreporter"), meterReporter, factory.MustNewName("licensing")), factory.NewNamedService(factory.MustNewName("ruler"), rulerInstance), + factory.NewNamedService(factory.MustNewName("systemdashboard"), impldashboard.NewService(providerSettings, dashboard, orgGetter)), ) if err != nil { return nil, err diff --git a/pkg/sqlmigration/119_add_system_dashboard.go b/pkg/sqlmigration/119_add_system_dashboard.go new file mode 100644 index 00000000000..b59c0a5d2b8 --- /dev/null +++ b/pkg/sqlmigration/119_add_system_dashboard.go @@ -0,0 +1,93 @@ +package sqlmigration + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/sqlschema" + "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/uptrace/bun" + "github.com/uptrace/bun/migrate" +) + +type addSystemDashboard struct { + sqlstore sqlstore.SQLStore + sqlschema sqlschema.SQLSchema +} + +func NewAddSystemDashboardFactory(sqlstore sqlstore.SQLStore, sqlschema sqlschema.SQLSchema) factory.ProviderFactory[SQLMigration, Config] { + return factory.NewProviderFactory( + factory.MustNewName("add_system_dashboard"), + func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) { + return &addSystemDashboard{sqlstore: sqlstore, sqlschema: sqlschema}, nil + }, + ) +} + +func (migration *addSystemDashboard) Register(migrations *migrate.Migrations) error { + return migrations.Register(migration.Up, migration.Down) +} + +func (migration *addSystemDashboard) Up(ctx context.Context, db *bun.DB) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + sqls := migration.sqlschema.Operator().CreateTable(&sqlschema.Table{ + Name: "system_dashboard", + Columns: []*sqlschema.Column{ + {Name: "id", DataType: sqlschema.DataTypeText, Nullable: false}, + {Name: "org_id", DataType: sqlschema.DataTypeText, Nullable: false}, + {Name: "dashboard_id", DataType: sqlschema.DataTypeText, Nullable: false}, + {Name: "name", DataType: sqlschema.DataTypeText, Nullable: false}, + {Name: "version", DataType: sqlschema.DataTypeBigInt, Nullable: false}, + {Name: "created_at", DataType: sqlschema.DataTypeTimestamp, Nullable: false}, + {Name: "updated_at", DataType: sqlschema.DataTypeTimestamp, Nullable: false}, + }, + PrimaryKeyConstraint: &sqlschema.PrimaryKeyConstraint{ + ColumnNames: []sqlschema.ColumnName{"id"}, + }, + ForeignKeyConstraints: []*sqlschema.ForeignKeyConstraint{ + { + ReferencingColumnName: sqlschema.ColumnName("org_id"), + ReferencedTableName: sqlschema.TableName("organizations"), + ReferencedColumnName: sqlschema.ColumnName("id"), + }, + { + ReferencingColumnName: sqlschema.ColumnName("dashboard_id"), + ReferencedTableName: sqlschema.TableName("dashboard"), + ReferencedColumnName: sqlschema.ColumnName("id"), + }, + }, + }) + + // (org_id, name) is what makes provisioning safe across replicas: the state + // row is written in the same transaction as the dashboard, so a losing racer + // rolls back its dashboard too. + sqls = append(sqls, migration.sqlschema.Operator().CreateIndex( + &sqlschema.UniqueIndex{ + TableName: "system_dashboard", + ColumnNames: []sqlschema.ColumnName{"org_id", "name"}, + }, + )...) + sqls = append(sqls, migration.sqlschema.Operator().CreateIndex( + &sqlschema.UniqueIndex{ + TableName: "system_dashboard", + ColumnNames: []sqlschema.ColumnName{"dashboard_id"}, + }, + )...) + + for _, sql := range sqls { + if _, err := tx.ExecContext(ctx, string(sql)); err != nil { + return err + } + } + + return tx.Commit() +} + +func (migration *addSystemDashboard) Down(context.Context, *bun.DB) error { + return nil +} diff --git a/pkg/types/dashboardtypes/perses_dashboard.go b/pkg/types/dashboardtypes/perses_dashboard.go index f72116b6b40..d640bdc2916 100644 --- a/pkg/types/dashboardtypes/perses_dashboard.go +++ b/pkg/types/dashboardtypes/perses_dashboard.go @@ -25,6 +25,10 @@ const ( dashboardNameSuffixLen = 8 ) +// SystemDashboardNamePrefix is reserved for dashboards SigNoz ships and owns. Generated +// names never contain consecutive hyphens, so only a typed name can carry it — create rejects that. +const SystemDashboardNamePrefix = "signoz---" + const ( dashboardIconPathPrefix = "/assets/Icons/" dashboardLogoPathPrefix = "/assets/Logos/" @@ -75,8 +79,8 @@ type DashboardV2 struct { } func (d *DashboardV2) ErrIfNotMutable() error { - if d.Source == SourceIntegration { - return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "integration dashboards cannot be modified") + if d.Source != SourceUser { + return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "%s dashboards cannot be modified", d.Source) } return nil } @@ -95,6 +99,11 @@ func (d *DashboardV2) Update(updatable UpdatableDashboardV2, updatedBy string, r if err := d.ErrIfNotUpdatable(); err != nil { return err } + return d.UpdateUnsafe(updatable, updatedBy, resolvedTags) +} + +// UpdateUnsafe applies the update without the source/lock gate. Intended for internal system callers. +func (d *DashboardV2) UpdateUnsafe(updatable UpdatableDashboardV2, updatedBy string, resolvedTags []*tagtypes.Tag) error { if updatable.Name != d.Name { return errors.NewInvalidInputf(ErrCodeDashboardImmutable, "name is immutable; cannot change from %q to %q", d.Name, updatable.Name) } @@ -129,6 +138,13 @@ func (d *DashboardV2) LockUnlock(lock bool, isAdmin bool, updatedBy string) erro return nil } +func (d *DashboardV2) ErrIfNotSystem() error { + if d.Source != SourceSystem { + return errors.Newf(errors.TypeNotFound, ErrCodeDashboardNotFound, "dashboard %q is not a system dashboard", d.Name) + } + return nil +} + func (d *DashboardV2) ErrIfNotClonable() error { if !d.Source.isClonable() { return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "%s dashboards cannot be cloned", d.Source) @@ -205,13 +221,21 @@ type PostableDashboardV2 struct { Spec DashboardSpec `json:"spec" required:"true"` } -func (postable PostableDashboardV2) NewDashboardV2(orgID valuer.UUID, createdBy string, source Source) *DashboardV2 { +func (postable PostableDashboardV2) NewDashboardV2(orgID valuer.UUID, createdBy string, source Source) (*DashboardV2, error) { now := time.Now() name := postable.Name if postable.GenerateName { name = generateDashboardName(postable.Spec.Display.Name) } + // Checked on the final name, here rather than in validateName, because only + // the constructor knows the source. + if source != SourceSystem && strings.HasPrefix(name, SystemDashboardNamePrefix) { + return nil, errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "name %q is invalid: the %q prefix is reserved for system dashboards", name, SystemDashboardNamePrefix) + } + if source == SourceSystem && !strings.HasPrefix(name, SystemDashboardNamePrefix) { + return nil, errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "name %q is invalid: system dashboard names must start with the %q prefix", name, SystemDashboardNamePrefix) + } return &DashboardV2{ Identifiable: types.Identifiable{ID: valuer.GenerateUUID()}, @@ -224,7 +248,7 @@ func (postable PostableDashboardV2) NewDashboardV2(orgID valuer.UUID, createdBy Name: name, Tags: tagtypes.NewTagsFromPostableTags(orgID, coretypes.KindDashboard, postable.Tags), Spec: postable.Spec, - } + }, nil } func (p *PostableDashboardV2) UnmarshalJSON(data []byte) error { @@ -365,6 +389,36 @@ func (d DashboardV2) ToGettableDashboardV2() GettableDashboardV2 { } } +// GettableSystemDashboard is the system-dashboard endpoint's response. System +// dashboards are addressed by their stable definition name, so it carries no id. +type GettableSystemDashboard struct { + types.TimeAuditable + types.UserAuditable + + OrgID valuer.UUID `json:"orgId" required:"true"` + Locked bool `json:"locked" required:"true"` + Source Source `json:"source" required:"true"` + + DashboardV2MetadataBase + Name string `json:"name" required:"true"` + Tags []*tagtypes.GettableTag `json:"tags" required:"true"` + Spec DashboardSpec `json:"spec" required:"true"` +} + +func (d DashboardV2) ToGettableSystemDashboard() GettableSystemDashboard { + return GettableSystemDashboard{ + TimeAuditable: d.TimeAuditable, + UserAuditable: d.UserAuditable, + OrgID: d.OrgID, + Locked: d.Locked, + Source: d.Source, + DashboardV2MetadataBase: d.DashboardV2MetadataBase, + Name: d.Name, + Tags: tagtypes.NewGettableTagsFromTags(d.Tags), + Spec: d.Spec, + } +} + // ════════════════════════════════════════════════════════════════════════ // Storable // ════════════════════════════════════════════════════════════════════════ diff --git a/pkg/types/dashboardtypes/perses_dashboard_convertors_test.go b/pkg/types/dashboardtypes/perses_dashboard_convertors_test.go index bc17c854a4a..0faf2d029ff 100644 --- a/pkg/types/dashboardtypes/perses_dashboard_convertors_test.go +++ b/pkg/types/dashboardtypes/perses_dashboard_convertors_test.go @@ -89,21 +89,25 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { cases := []struct { scenario string source Source + name string expectedLocked bool }{ { scenario: "user source is not locked", source: SourceUser, + name: "my-dashboard", expectedLocked: false, }, { scenario: "system source is not locked", source: SourceSystem, + name: SystemDashboardNamePrefix + "my-dashboard", expectedLocked: false, }, { scenario: "integration source is locked", source: SourceIntegration, + name: "my-dashboard", expectedLocked: true, }, } @@ -115,7 +119,7 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { SchemaVersion: SchemaVersion, Image: "img", }, - Name: "my-dashboard", + Name: tc.name, Tags: []tagtypes.PostableTag{ {Key: "team", Value: "platform"}, {Key: "env", Value: "prod"}, @@ -124,7 +128,8 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { } before := time.Now() - dashboard := postable.NewDashboardV2(orgID, "alice", tc.source) + dashboard, err := postable.NewDashboardV2(orgID, "alice", tc.source) + require.NoError(t, err) after := time.Now() require.NotNil(t, dashboard) @@ -160,8 +165,10 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { Spec: DashboardSpec{}, } - first := postable.NewDashboardV2(orgID, "alice", SourceUser) - second := postable.NewDashboardV2(orgID, "alice", SourceUser) + first, err := postable.NewDashboardV2(orgID, "alice", SourceUser) + require.NoError(t, err) + second, err := postable.NewDashboardV2(orgID, "alice", SourceUser) + require.NoError(t, err) assert.NotEqual(t, first.ID, second.ID, "expected distinct UUIDs across invocations") }) @@ -174,7 +181,8 @@ func TestPostableDashboardV2NewDashboardV2(t *testing.T) { }, } - dashboard := postable.NewDashboardV2(orgID, "alice", SourceUser) + dashboard, err := postable.NewDashboardV2(orgID, "alice", SourceUser) + require.NoError(t, err) assert.True(t, strings.HasPrefix(dashboard.Name, "my-dashboard-"), "expected slug prefix, got %q", dashboard.Name) assert.Len(t, dashboard.Name, len("my-dashboard-")+dashboardNameSuffixLen) }) diff --git a/pkg/types/dashboardtypes/perses_dashboard_patch_test.go b/pkg/types/dashboardtypes/perses_dashboard_patch_test.go index 3cd7ba421f0..84d91f7b9d4 100644 --- a/pkg/types/dashboardtypes/perses_dashboard_patch_test.go +++ b/pkg/types/dashboardtypes/perses_dashboard_patch_test.go @@ -109,7 +109,8 @@ func TestPatchableDashboardV2_Apply(t *testing.T) { var p PostableDashboardV2 require.NoError(t, json.Unmarshal([]byte(basePostableJSON), &p), "base postable JSON must validate") testOrgID := valuer.GenerateUUID() - base := p.NewDashboardV2(testOrgID, "somecreatedthisiguess@signoz.io", SourceUser) + base, err := p.NewDashboardV2(testOrgID, "somecreatedthisiguess@signoz.io", SourceUser) + require.NoError(t, err) base.Tags = []*tagtypes.Tag{ {Key: "team", Value: "alpha"}, {Key: "env", Value: "prod"}, diff --git a/pkg/types/dashboardtypes/perses_dashboard_test.go b/pkg/types/dashboardtypes/perses_dashboard_test.go index fd48b4e143e..cdcbe62ee44 100644 --- a/pkg/types/dashboardtypes/perses_dashboard_test.go +++ b/pkg/types/dashboardtypes/perses_dashboard_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/valuer" "github.com/perses/spec/go/dashboard" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1928,3 +1929,37 @@ func TestEnsureSingleExpressionAggregation(t *testing.T) { }) } } + +// Guards the constant: a prefixed name must stay a valid DNS-1123 label. +func TestSystemDashboardNamePrefix(t *testing.T) { + require.NoError(t, validateDashboardName(SystemDashboardNamePrefix+"ai-o11y-overview")) +} + +func TestNewDashboardV2RejectsReservedName(t *testing.T) { + testCases := []struct { + description string + name string + source Source + errContains string + }{ + {description: "reserved name for a system dashboard", name: SystemDashboardNamePrefix + "overview", source: SourceSystem}, + {description: "reserved name for a user dashboard", name: SystemDashboardNamePrefix + "overview", source: SourceUser, errContains: "reserved for system dashboards"}, + {description: "reserved name for an integration dashboard", name: SystemDashboardNamePrefix + "overview", source: SourceIntegration, errContains: "reserved for system dashboards"}, + {description: "unprefixed name for a system dashboard", name: "overview", source: SourceSystem, errContains: "must start with"}, + {description: "ordinary name for a user dashboard", name: "overview", source: SourceUser}, + {description: "fewer hyphens than the prefix for a user dashboard", name: "signoz--overview", source: SourceUser}, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + postable := PostableDashboardV2{Name: testCase.name} + _, err := postable.NewDashboardV2(valuer.GenerateUUID(), "user@signoz.io", testCase.source) + if testCase.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), testCase.errContains) + return + } + require.NoError(t, err) + }) + } +} diff --git a/pkg/types/dashboardtypes/store.go b/pkg/types/dashboardtypes/store.go index 3de1503f0e0..9ef59670471 100644 --- a/pkg/types/dashboardtypes/store.go +++ b/pkg/types/dashboardtypes/store.go @@ -13,6 +13,9 @@ type Store interface { Get(context.Context, valuer.UUID, valuer.UUID) (*StorableDashboard, error) + // GetByName resolves a dashboard by its per-org unique name. + GetByName(ctx context.Context, orgID valuer.UUID, name string) (*StorableDashboard, error) + GetPublic(context.Context, string) (*StorablePublicDashboard, error) GetDashboardByOrgsAndPublicID(context.Context, []string, string) (*StorableDashboard, error) @@ -72,4 +75,13 @@ type Store interface { UpdateDashboardView(ctx context.Context, view *DashboardView) error DeleteDashboardView(ctx context.Context, orgID valuer.UUID, id valuer.UUID) error + + // ════════════════════════════════════════════════════════════════════════ + // System dashboard methods + // ════════════════════════════════════════════════════════════════════════ + CreateSystemDashboard(ctx context.Context, storable *StorableSystemDashboard) error + + GetSystemDashboard(ctx context.Context, orgID valuer.UUID, name string) (*StorableSystemDashboard, error) + + UpdateSystemDashboardVersion(ctx context.Context, orgID valuer.UUID, name string, version int) error } diff --git a/pkg/types/dashboardtypes/system_dashboard.go b/pkg/types/dashboardtypes/system_dashboard.go new file mode 100644 index 00000000000..0a43d30c981 --- /dev/null +++ b/pkg/types/dashboardtypes/system_dashboard.go @@ -0,0 +1,45 @@ +package dashboardtypes + +import ( + "time" + + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/valuer" + "github.com/uptrace/bun" +) + +var ( + ErrCodeSystemDashboardNotFound = errors.MustNewCode("system_dashboard_not_found") + ErrCodeSystemDashboardDefinitionInvalid = errors.MustNewCode("system_dashboard_definition_invalid") + ErrCodeSystemDashboardAlreadyProvisioned = errors.MustNewCode("system_dashboard_already_provisioned") +) + +// ProvisionerIdentity is stamped into created_by/updated_by by the reconciler. +const ProvisionerIdentity = "signoz" + +// StorableSystemDashboard records the shipped version each org's copy of a system +// dashboard was last provisioned at. That version is the only thing the dashboard +// row cannot answer, since the binary only embeds the latest definition. +type StorableSystemDashboard struct { + bun.BaseModel `bun:"table:system_dashboard"` + + types.Identifiable + types.TimeAuditable + OrgID valuer.UUID `bun:"org_id,type:text,notnull"` + DashboardID valuer.UUID `bun:"dashboard_id,type:text,notnull"` + Name string `bun:"name,type:text,notnull"` + Version int `bun:"version,notnull"` +} + +func NewStorableSystemDashboard(orgID valuer.UUID, dashboardID valuer.UUID, name string, version int) *StorableSystemDashboard { + now := time.Now() + return &StorableSystemDashboard{ + Identifiable: types.Identifiable{ID: valuer.GenerateUUID()}, + TimeAuditable: types.TimeAuditable{CreatedAt: now, UpdatedAt: now}, + OrgID: orgID, + DashboardID: dashboardID, + Name: name, + Version: version, + } +} diff --git a/pkg/types/dashboardtypes/system_dashboard_definition.go b/pkg/types/dashboardtypes/system_dashboard_definition.go new file mode 100644 index 00000000000..371df1bea01 --- /dev/null +++ b/pkg/types/dashboardtypes/system_dashboard_definition.go @@ -0,0 +1,95 @@ +package dashboardtypes + +import ( + "bytes" + "encoding/json" + "slices" + "strings" + + "github.com/SigNoz/signoz/pkg/errors" +) + +// SystemDashboardDefinition is one shipped system dashboard. Version is bumped on +// every content change and drives upgrade detection; the name is the stable key +// and never changes. +type SystemDashboardDefinition struct { + Version int `json:"version"` + Dashboard PostableDashboardV2 `json:"definition"` +} + +func (definition SystemDashboardDefinition) Name() string { + return definition.Dashboard.Name +} + +func NewSystemDashboardDefinition(raw []byte) (SystemDashboardDefinition, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + + var definition SystemDashboardDefinition + if err := decoder.Decode(&definition); err != nil { + return SystemDashboardDefinition{}, errors.WrapInvalidInputf(err, ErrCodeSystemDashboardDefinitionInvalid, "%s", err.Error()) + } + if err := definition.validate(); err != nil { + return SystemDashboardDefinition{}, err + } + + return definition, nil +} + +func (definition SystemDashboardDefinition) validate() error { + if definition.Version < 1 { + return errors.NewInvalidInputf(ErrCodeSystemDashboardDefinitionInvalid, "version must be at least 1, got %d", definition.Version) + } + if !strings.HasPrefix(definition.Name(), SystemDashboardNamePrefix) { + return errors.NewInvalidInputf(ErrCodeSystemDashboardDefinitionInvalid, "name %q must start with %q", definition.Name(), SystemDashboardNamePrefix) + } + if definition.Dashboard.GenerateName { + return errors.NewInvalidInputf(ErrCodeSystemDashboardDefinitionInvalid, "%s: generateName is not allowed, the name is the stable key", definition.Name()) + } + + return nil +} + +// ToUpdatable is how an upgrade re-applies a definition onto an existing row: +// everything but the dashboard's identity comes from the shipped definition. +func (definition SystemDashboardDefinition) ToUpdatable() UpdatableDashboardV2 { + return UpdatableDashboardV2{ + DashboardV2MetadataBase: definition.Dashboard.DashboardV2MetadataBase, + Name: definition.Dashboard.Name, + Tags: definition.Dashboard.Tags, + Spec: definition.Dashboard.Spec, + } +} + +// SystemDashboardRegistry holds every definition embedded in the binary, keyed by name. +type SystemDashboardRegistry struct { + definitions map[string]SystemDashboardDefinition +} + +func NewSystemDashboardRegistry(definitions []SystemDashboardDefinition) (SystemDashboardRegistry, error) { + byName := make(map[string]SystemDashboardDefinition, len(definitions)) + for _, definition := range definitions { + if _, duplicate := byName[definition.Name()]; duplicate { + return SystemDashboardRegistry{}, errors.NewInvalidInputf(ErrCodeSystemDashboardDefinitionInvalid, "duplicate system dashboard name %q", definition.Name()) + } + byName[definition.Name()] = definition + } + + return SystemDashboardRegistry{definitions: byName}, nil +} + +func (registry SystemDashboardRegistry) Get(name string) (SystemDashboardDefinition, bool) { + definition, ok := registry.definitions[name] + return definition, ok +} + +// List returns the definitions sorted by name so provisioning order is stable. +func (registry SystemDashboardRegistry) List() []SystemDashboardDefinition { + definitions := make([]SystemDashboardDefinition, 0, len(registry.definitions)) + for _, definition := range registry.definitions { + definitions = append(definitions, definition) + } + slices.SortFunc(definitions, func(a, b SystemDashboardDefinition) int { return strings.Compare(a.Name(), b.Name()) }) + + return definitions +} diff --git a/tests/integration/tests/dashboard/07_system_dashboard.py b/tests/integration/tests/dashboard/07_system_dashboard.py new file mode 100644 index 00000000000..a2f1b4587ca --- /dev/null +++ b/tests/integration/tests/dashboard/07_system_dashboard.py @@ -0,0 +1,192 @@ +from collections.abc import Callable +from http import HTTPStatus + +import requests +from sqlalchemy import sql + +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.dashboards import DASHBOARDS_BASE_URL, MAX_LIST_LIMIT +from fixtures.types import Operation, SigNoz + +SYSTEM_BASE_URL = "/api/v2/dashboards/system" + +# Provisioned for every org by the reconciler; the path segment is the bare +# definition name, the stored name carries the reserved prefix. +SYSTEM_DASHBOARD_NAME = "ai-o11y-overview" +SYSTEM_DASHBOARD_PREFIX = "signoz---" + + +def test_get_system_dashboard( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{SYSTEM_BASE_URL}/{SYSTEM_DASHBOARD_NAME}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + + assert response.status_code == HTTPStatus.OK, response.text + dashboard = response.json()["data"] + assert dashboard["name"] == SYSTEM_DASHBOARD_PREFIX + SYSTEM_DASHBOARD_NAME + assert dashboard["source"] == "system" + assert dashboard["createdBy"] == "signoz" + assert dashboard["schemaVersion"] == "v6" + assert "id" not in dashboard + + +def test_get_system_dashboard_rejects_prefixed_name( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{SYSTEM_BASE_URL}/{SYSTEM_DASHBOARD_PREFIX}{SYSTEM_DASHBOARD_NAME}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert "must not carry" in response.json()["error"]["message"] + + +def test_get_missing_system_dashboard_returns_not_found( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{SYSTEM_BASE_URL}/no-such-dashboard"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + + assert response.status_code == HTTPStatus.NOT_FOUND, response.text + + +def test_system_dashboard_hidden_from_list_but_gettable_by_id( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + # The API never exposes a system dashboard's id; read it from the state row. + with signoz.sqlstore.conn.connect() as conn: + dashboard_id = conn.execute( + sql.text("SELECT dashboard_id FROM system_dashboard WHERE name = :name"), + {"name": SYSTEM_DASHBOARD_PREFIX + SYSTEM_DASHBOARD_NAME}, + ).scalar_one() + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}?limit={MAX_LIST_LIMIT}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.OK, response.text + listed = response.json()["data"]["dashboards"] or [] + assert all(dashboard["source"] != "system" for dashboard in listed) + assert all(dashboard["id"] != dashboard_id for dashboard in listed) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["data"]["source"] == "system" + + +def test_system_dashboard_is_immutable( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.get( + signoz.self.host_configs["8080"].get(f"{SYSTEM_BASE_URL}/{SYSTEM_DASHBOARD_NAME}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.OK, response.text + dashboard = response.json()["data"] + + with signoz.sqlstore.conn.connect() as conn: + dashboard_id = conn.execute( + sql.text("SELECT dashboard_id FROM system_dashboard WHERE name = :name"), + {"name": SYSTEM_DASHBOARD_PREFIX + SYSTEM_DASHBOARD_NAME}, + ).scalar_one() + + response = requests.put( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}"), + json={ + "schemaVersion": dashboard["schemaVersion"], + "name": dashboard["name"], + "tags": [], + "spec": dashboard["spec"], + }, + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.json()["error"]["code"] == "dashboard_immutable" + + response = requests.delete( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.json()["error"]["code"] == "dashboard_immutable" + + response = requests.put( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}/lock"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.json()["error"]["code"] == "dashboard_immutable" + + response = requests.post( + signoz.self.host_configs["8080"].get(f"{DASHBOARDS_BASE_URL}/{dashboard_id}/clone"), + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.json()["error"]["code"] == "dashboard_immutable" + + +def test_create_rejects_reserved_prefix_name( + signoz: SigNoz, + create_user_admin: Operation, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], +): + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = requests.post( + signoz.self.host_configs["8080"].get(DASHBOARDS_BASE_URL), + json={ + "schemaVersion": "v6", + "name": f"{SYSTEM_DASHBOARD_PREFIX}custom", + "tags": [], + "spec": { + "display": {"name": "Custom"}, + "variables": [], + "panels": {}, + "layouts": [], + }, + }, + headers={"Authorization": f"Bearer {token}"}, + timeout=5, + ) + + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert "reserved for system dashboards" in response.json()["error"]["message"]