From 0ef48ab74a37dba56f71e9b1111a9cebc37a54e9 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 29 Oct 2026 17:16:19 -0300 Subject: [PATCH 1/2] feat(CI): Set up CodSpeed Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/codspeed.yml | 35 ++++++++ .gitmodules | 4 + Makefile | 9 ++ benchmarks/sdk_benchmark_test.go | 148 +++++++++++++++++++++++++++++++ flagengine/sdk-benchmarks | 1 + 5 files changed, 197 insertions(+) create mode 100644 .github/workflows/codspeed.yml create mode 100644 benchmarks/sdk_benchmark_test.go create mode 160000 flagengine/sdk-benchmarks diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000..6098f334 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,35 @@ +name: CodSpeed + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read # Check out the repository and its submodules + id-token: write # Authenticate result uploads to CodSpeed via OIDC + +jobs: + benchmarks: + name: Run benchmarks + runs-on: codspeed-macro + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + submodules: recursive + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: stable + + - name: Run benchmarks + uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5.6.0 + with: + mode: walltime + run: make benchmark diff --git a/.gitmodules b/.gitmodules index 5e406b4e..bb5614c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = flagengine/engine-test-data url = git@github.com:Flagsmith/engine-test-data.git branch = v3.10.0 +[submodule "flagengine/sdk-benchmarks"] + path = flagengine/sdk-benchmarks + url = https://github.com/Flagsmith/sdk-benchmarks.git + branch = 0.0.2 diff --git a/Makefile b/Makefile index 21f55231..7f6006f3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ .EXPORT_ALL_VARIABLES: EVALUATION_CONTEXT_SCHEMA_URL ?= https://raw.githubusercontent.com/Flagsmith/flagsmith/main/sdk/evaluation-context.json +SDK_BENCHMARKS := flagengine/sdk-benchmarks +SDK_BENCHMARKS_ENVIRONMENTS := $(abspath $(SDK_BENCHMARKS)/environments) .PHONY: generate-evaluation-context @@ -13,3 +15,10 @@ generate-evaluation-context: --just-types-and-package \ --top-level EvaluationContext \ -o evaluationcontext.go + +$(SDK_BENCHMARKS_ENVIRONMENTS): + git submodule update --init -- $(SDK_BENCHMARKS) + +.PHONY: benchmark +benchmark: | $(SDK_BENCHMARKS_ENVIRONMENTS) + go test -bench=. ./benchmarks/ diff --git a/benchmarks/sdk_benchmark_test.go b/benchmarks/sdk_benchmark_test.go new file mode 100644 index 00000000..8694c535 --- /dev/null +++ b/benchmarks/sdk_benchmark_test.go @@ -0,0 +1,148 @@ +package benchmarks + +import ( + "bytes" + "context" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + flagsmith "github.com/Flagsmith/flagsmith-go-client/v5" +) + +const ( + identityOverridesIdentifier = "identity-override-1" + identifier = "user-1" +) + +var ( + environments = []string{"small", "medium", "large"} + traits = []*flagsmith.Trait{ + {TraitKey: "plan", TraitValue: "plan-7"}, + {TraitKey: "age", TraitValue: 30}, + {TraitKey: "country", TraitValue: "GB"}, + {TraitKey: "version", TraitValue: "2.5.1"}, + {TraitKey: "email", TraitValue: "jane@example.com"}, + {TraitKey: "user_id", TraitValue: 12}, + {TraitKey: "beta", TraitValue: true}, + {TraitKey: "score", TraitValue: 95}, + {TraitKey: "tier", TraitValue: "pro"}, + {TraitKey: "device", TraitValue: "iPhone 15"}, + } +) + +type environmentDocumentTransport []byte + +func (environmentDocument environmentDocumentTransport) RoundTrip(request *http.Request) (*http.Response, error) { + return &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(bytes.NewReader(environmentDocument)), + ContentLength: int64(len(environmentDocument)), + Request: request, + }, nil +} + +// runPerEnvironment runs a benchmark against a locally evaluating client per environment document size. +func runPerEnvironment(b *testing.B, benchmark func(b *testing.B, client *flagsmith.Client)) { + for _, environment := range environments { + b.Run(environment, func(b *testing.B) { + environmentDocument, err := os.ReadFile(filepath.Join(os.Getenv("SDK_BENCHMARKS_ENVIRONMENTS"), environment+".json")) + if err != nil { + b.Fatal(err) + } + ctx, closeClient := context.WithCancel(context.Background()) + b.Cleanup(closeClient) + client := flagsmith.NewClient("ser.benchmark", + flagsmith.WithBaseURL("http://flagsmith.benchmark/api/v1/"), + flagsmith.WithHTTPClient(&http.Client{Transport: environmentDocumentTransport(environmentDocument)}), + flagsmith.WithLocalEvaluation(ctx), + flagsmith.WithEnvironmentRefreshInterval(24*time.Hour), + flagsmith.WithSlogLogger(slog.New(slog.DiscardHandler)), + ) + if err := client.UpdateEnvironment(ctx); err != nil { + b.Fatal(err) + } + benchmark(b, client) + }) + } +} + +// BenchmarkGetEnvironmentFlags gets a feature value from the environment flags. +func BenchmarkGetEnvironmentFlags(b *testing.B) { + runPerEnvironment(b, func(b *testing.B, client *flagsmith.Client) { + for b.Loop() { + flags, err := client.GetEnvironmentFlags(context.Background()) + if err != nil { + b.Fatal(err) + } + if _, err := flags.GetFeatureValue("feature_1"); err != nil { + b.Fatal(err) + } + } + }) +} + +// BenchmarkGetIdentityFlagsWithIdentityOverrides checks a feature for an identity with identity overrides. +func BenchmarkGetIdentityFlagsWithIdentityOverrides(b *testing.B) { + runPerEnvironment(b, func(b *testing.B, client *flagsmith.Client) { + for b.Loop() { + flags, err := client.GetIdentityFlags(context.Background(), identityOverridesIdentifier, nil) + if err != nil { + b.Fatal(err) + } + if _, err := flags.IsFeatureEnabled("feature_8"); err != nil { + b.Fatal(err) + } + } + }) +} + +// BenchmarkGetIdentityFlagsNoTraits gets a feature value for an identity without traits. +func BenchmarkGetIdentityFlagsNoTraits(b *testing.B) { + runPerEnvironment(b, func(b *testing.B, client *flagsmith.Client) { + for b.Loop() { + flags, err := client.GetIdentityFlags(context.Background(), identifier, nil) + if err != nil { + b.Fatal(err) + } + if _, err := flags.GetFeatureValue("feature_1"); err != nil { + b.Fatal(err) + } + } + }) +} + +// BenchmarkGetIdentityFlagsWithTraits gets a feature value for an identity with traits. +func BenchmarkGetIdentityFlagsWithTraits(b *testing.B) { + runPerEnvironment(b, func(b *testing.B, client *flagsmith.Client) { + for b.Loop() { + flags, err := client.GetIdentityFlags(context.Background(), identifier, traits) + if err != nil { + b.Fatal(err) + } + if _, err := flags.GetFeatureValue("feature_1"); err != nil { + b.Fatal(err) + } + } + }) +} + +// BenchmarkGetIdentitySegmentsWithTraits lists the segments of an identity with traits. +func BenchmarkGetIdentitySegmentsWithTraits(b *testing.B) { + runPerEnvironment(b, func(b *testing.B, client *flagsmith.Client) { + for b.Loop() { + if _, err := client.GetIdentitySegments(identifier, traits); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/flagengine/sdk-benchmarks b/flagengine/sdk-benchmarks new file mode 160000 index 00000000..7da2be42 --- /dev/null +++ b/flagengine/sdk-benchmarks @@ -0,0 +1 @@ +Subproject commit 7da2be42ec7c58a597892e41141d531bd1dd100c From 4ce1a855ab16fe0d02a9e62ca42c2b59d9f0fafd Mon Sep 17 00:00:00 2001 From: Evandro Myller <22429+emyller@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:53:26 -0300 Subject: [PATCH 2/2] does this work Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 6098f334..44966087 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: stable + go-version-file: go.mod - name: Run benchmarks uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5.6.0