From aa70be952d55df407ab02ec570b6114b7ba7620d Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 29 Oct 2026 18:27:04 -0300 Subject: [PATCH 1/8] ci: Benchmark the SDK with CodSpeed Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/codspeed.yml | 40 ++++++++++++++++ .gitmodules | 4 ++ Makefile | 12 +++++ benchmarks/run.rb | 19 ++++++++ benchmarks/sdk_benchmark.rb | 88 ++++++++++++++++++++++++++++++++++ codspeed.yml | 33 +++++++++++++ flagsmith.gemspec | 2 +- spec/sdk-benchmarks | 1 + 8 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/codspeed.yml create mode 100644 Makefile create mode 100644 benchmarks/run.rb create mode 100644 benchmarks/sdk_benchmark.rb create mode 100644 codspeed.yml create mode 160000 spec/sdk-benchmarks diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000..93e949f --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,40 @@ +name: CodSpeed + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read # Check out the repository and its submodules + id-token: write # Authenticate result uploads to CodSpeed via OIDC + +jobs: + benchmarks: + name: Run benchmarks + runs-on: codspeed-macro + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Ruby + uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f # v1.335.0 + with: + ruby-version: '3.3' + + - name: Install CodSpeed CLI + env: + CODSPEED_VERSION: 5.6.0 + CODSPEED_INSTALLER_SHA256: bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40 + run: | + curl -fsSL -o "$RUNNER_TEMP/codspeed-install.sh" "https://codspeed.io/v$CODSPEED_VERSION/install.sh" + echo "$CODSPEED_INSTALLER_SHA256 $RUNNER_TEMP/codspeed-install.sh" | sha256sum --check --strict + bash "$RUNNER_TEMP/codspeed-install.sh" --quiet + + - name: Run benchmarks + run: make benchmark diff --git a/.gitmodules b/.gitmodules index b2bee4c..0c86e07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = spec/engine-test-data url = git@github.com:Flagsmith/engine-test-data.git branch = v3.7.0 +[submodule "spec/sdk-benchmarks"] + path = spec/sdk-benchmarks + url = https://github.com/Flagsmith/sdk-benchmarks.git + branch = 0.0.2 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..939a0c1 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +SDK_BENCHMARKS := spec/sdk-benchmarks + +$(SDK_BENCHMARKS)/environments: + git submodule update --init -- $(SDK_BENCHMARKS) + +.PHONY: benchmark-deps +benchmark-deps: $(SDK_BENCHMARKS)/environments + bundle install + +.PHONY: benchmark +benchmark: benchmark-deps + SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode walltime --config codspeed.yml diff --git a/benchmarks/run.rb b/benchmarks/run.rb new file mode 100644 index 0000000..bc9c457 --- /dev/null +++ b/benchmarks/run.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'bundler/setup' +require_relative 'sdk_benchmark' + +REVOLUTIONS = 1000 + +benchmark, environment = ARGV +sdk_benchmarks_dir = ENV.fetch('SDK_BENCHMARKS', '') + +unless !sdk_benchmarks_dir.empty? && SdkBenchmark::BENCHMARKS.key?(benchmark) && SdkBenchmark::ENVIRONMENTS.include?(environment) + warn "Usage: SDK_BENCHMARKS= ruby #{$PROGRAM_NAME} " \ + "<#{SdkBenchmark::BENCHMARKS.keys.join('|')}> <#{SdkBenchmark::ENVIRONMENTS.join('|')}>" + exit 2 +end + +sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment) +REVOLUTIONS.times { sdk_benchmark.public_send(SdkBenchmark::BENCHMARKS.fetch(benchmark)) } +sdk_benchmark.close diff --git a/benchmarks/sdk_benchmark.rb b/benchmarks/sdk_benchmark.rb new file mode 100644 index 0000000..190a384 --- /dev/null +++ b/benchmarks/sdk_benchmark.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'flagsmith' + +# Benchmarks a locally evaluating SDK client against environment documents tiered by size. +class SdkBenchmark + ENVIRONMENTS = %w[small medium large].freeze + BENCHMARKS = { + 'getEnvironmentFlags' => :get_environment_flags, + 'getIdentityFlagsWithIdentityOverrides' => :get_identity_flags_with_identity_overrides, + 'getIdentityFlagsNoTraits' => :get_identity_flags_no_traits, + 'getIdentityFlagsWithTraits' => :get_identity_flags_with_traits, + 'getIdentitySegmentsWithTraits' => :get_identity_segments_with_traits + }.freeze + + IDENTITY_OVERRIDES_IDENTIFIER = 'identity-override-1' + IDENTIFIER = 'user-1' + TRAITS = { + plan: 'plan-7', + age: 30, + country: 'GB', + version: '2.5.1', + email: 'jane@example.com', + user_id: 12, + beta: true, + score: 95, + tier: 'pro', + device: 'iPhone 15' + }.freeze + + # Build a locally evaluating client serving the environment document from memory. + def initialize(sdk_benchmarks_dir, environment) + environment_document = File.read(File.join(sdk_benchmarks_dir, 'environments', "#{environment}.json")) + Faraday.default_adapter = serve_environment_document(environment_document) + @client = Flagsmith::Client.new( + environment_key: 'ser.benchmark', + api_url: 'http://flagsmith.benchmark/api/v1/', + enable_local_evaluation: true, + environment_refresh_interval_seconds: 86_400, # Refreshed by benchmarks only + logger: Logger.new(File::NULL) + ) + end + + def close + @client.environment_data_polling_manager.stop + end + + # rubocop:disable Naming/AccessorMethodName + + # Get a feature value from the environment flags. + def get_environment_flags + @client.get_environment_flags.feature_value('feature_1') + end + + # Check a feature for an identity with identity overrides. + def get_identity_flags_with_identity_overrides + @client.get_identity_flags(IDENTITY_OVERRIDES_IDENTIFIER).feature_enabled?('feature_8') + end + + # Get a feature value for an identity without traits. + def get_identity_flags_no_traits + @client.get_identity_flags(IDENTIFIER).feature_value('feature_1') + end + + # Get a feature value for an identity with traits. + def get_identity_flags_with_traits + @client.get_identity_flags(IDENTIFIER, **TRAITS).feature_value('feature_1') + end + + # List the segments of an identity with traits. + def get_identity_segments_with_traits + @client.get_identity_segments(IDENTIFIER, TRAITS) + end + + # rubocop:enable Naming/AccessorMethodName + + private + + def serve_environment_document(environment_document) + Class.new(Faraday::Adapter) do + define_method(:call) do |env| + super(env) + save_response(env, 200, environment_document, { 'Content-Type' => 'application/json' }) + @app.call(env) + end + end + end +end diff --git a/codspeed.yml b/codspeed.yml new file mode 100644 index 0000000..6400170 --- /dev/null +++ b/codspeed.yml @@ -0,0 +1,33 @@ +$schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json + +benchmarks: + - name: getEnvironmentFlags[small] + exec: ruby benchmarks/run.rb getEnvironmentFlags small + - name: getEnvironmentFlags[medium] + exec: ruby benchmarks/run.rb getEnvironmentFlags medium + - name: getEnvironmentFlags[large] + exec: ruby benchmarks/run.rb getEnvironmentFlags large + - name: getIdentityFlagsWithIdentityOverrides[small] + exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides small + - name: getIdentityFlagsWithIdentityOverrides[medium] + exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides medium + - name: getIdentityFlagsWithIdentityOverrides[large] + exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides large + - name: getIdentityFlagsNoTraits[small] + exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits small + - name: getIdentityFlagsNoTraits[medium] + exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits medium + - name: getIdentityFlagsNoTraits[large] + exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits large + - name: getIdentityFlagsWithTraits[small] + exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits small + - name: getIdentityFlagsWithTraits[medium] + exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits medium + - name: getIdentityFlagsWithTraits[large] + exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits large + - name: getIdentitySegmentsWithTraits[small] + exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits small + - name: getIdentitySegmentsWithTraits[medium] + exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits medium + - name: getIdentitySegmentsWithTraits[large] + exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits large diff --git a/flagsmith.gemspec b/flagsmith.gemspec index adbe62a..77953be 100644 --- a/flagsmith.gemspec +++ b/flagsmith.gemspec @@ -12,7 +12,7 @@ Gem::Specification.new do |spec| # The `git ls-files -z` loads the files in the RubyGem that have been added into git. spec.files = Dir.chdir(__dir__) do `git ls-files -z`.split("\x0").reject do |f| - (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|sdk-spec|features)/|\.(?:git|travis|circleci)|appveyor)}) + (f == __FILE__) || f.match(%r{\A(?:(?:benchmarks|bin|test|spec|sdk-spec|features)/|\.(?:git|travis|circleci)|appveyor)}) end end diff --git a/spec/sdk-benchmarks b/spec/sdk-benchmarks new file mode 160000 index 0000000..7da2be4 --- /dev/null +++ b/spec/sdk-benchmarks @@ -0,0 +1 @@ +Subproject commit 7da2be42ec7c58a597892e41141d531bd1dd100c From 01ef0e7ee7e3c012c55bcb496189f7cfcd2bdcf0 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 30 Oct 2026 10:50:38 -0300 Subject: [PATCH 2/8] ci: Use the benchmark document's environment key Co-Authored-By: Claude Opus 5.5 (1M context) --- benchmarks/sdk_benchmark.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/sdk_benchmark.rb b/benchmarks/sdk_benchmark.rb index 190a384..2cabb17 100644 --- a/benchmarks/sdk_benchmark.rb +++ b/benchmarks/sdk_benchmark.rb @@ -33,7 +33,7 @@ def initialize(sdk_benchmarks_dir, environment) environment_document = File.read(File.join(sdk_benchmarks_dir, 'environments', "#{environment}.json")) Faraday.default_adapter = serve_environment_document(environment_document) @client = Flagsmith::Client.new( - environment_key: 'ser.benchmark', + environment_key: JSON.parse(environment_document).fetch('api_key'), api_url: 'http://flagsmith.benchmark/api/v1/', enable_local_evaluation: true, environment_refresh_interval_seconds: 86_400, # Refreshed by benchmarks only From eca3a1fb5a6639bd058dd495dcde48575e98bbae Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 30 Oct 2026 12:37:15 -0300 Subject: [PATCH 3/8] ci: Run CodSpeed benchmarks in simulation on GitHub-hosted runners Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/codspeed.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 93e949f..7bd9c63 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -14,7 +14,7 @@ permissions: jobs: benchmarks: name: Run benchmarks - runs-on: codspeed-macro + runs-on: ubuntu-latest steps: - name: Checkout repository @@ -27,14 +27,12 @@ jobs: with: ruby-version: '3.3' - - name: Install CodSpeed CLI - env: - CODSPEED_VERSION: 5.6.0 - CODSPEED_INSTALLER_SHA256: bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40 - run: | - curl -fsSL -o "$RUNNER_TEMP/codspeed-install.sh" "https://codspeed.io/v$CODSPEED_VERSION/install.sh" - echo "$CODSPEED_INSTALLER_SHA256 $RUNNER_TEMP/codspeed-install.sh" | sha256sum --check --strict - bash "$RUNNER_TEMP/codspeed-install.sh" --quiet + - name: Install benchmark dependencies + run: make benchmark-deps - name: Run benchmarks - run: make benchmark + uses: CodSpeedHQ/action@61efe07e57e55bedeab750c434504de272a56e8c # v5.6.1 + env: + SDK_BENCHMARKS: spec/sdk-benchmarks + with: + mode: simulation From 6ff9115db5b1eb2004837e4155d0c06c53d351d4 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 30 Oct 2026 13:37:33 -0300 Subject: [PATCH 4/8] ci: Run CodSpeed benchmarks on macro runners, installing the CLI via make Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/codspeed.yml | 11 ++--------- Makefile | 12 +++++++++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 7bd9c63..84b6bec 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -14,7 +14,7 @@ permissions: jobs: benchmarks: name: Run benchmarks - runs-on: ubuntu-latest + runs-on: codspeed-macro steps: - name: Checkout repository @@ -27,12 +27,5 @@ jobs: with: ruby-version: '3.3' - - name: Install benchmark dependencies - run: make benchmark-deps - - name: Run benchmarks - uses: CodSpeedHQ/action@61efe07e57e55bedeab750c434504de272a56e8c # v5.6.1 - env: - SDK_BENCHMARKS: spec/sdk-benchmarks - with: - mode: simulation + run: make benchmark diff --git a/Makefile b/Makefile index 939a0c1..a75936e 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,18 @@ SDK_BENCHMARKS := spec/sdk-benchmarks +CODSPEED_VERSION := 5.6.0 +CODSPEED_INSTALLER_SHA256 := bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40 $(SDK_BENCHMARKS)/environments: git submodule update --init -- $(SDK_BENCHMARKS) -.PHONY: benchmark-deps -benchmark-deps: $(SDK_BENCHMARKS)/environments +.PHONY: install-codspeed +install-codspeed: $(SDK_BENCHMARKS)/environments bundle install + installer=$$(mktemp) && \ + curl -fsSL -o "$$installer" "https://codspeed.io/v$(CODSPEED_VERSION)/install.sh" && \ + echo "$(CODSPEED_INSTALLER_SHA256) $$installer" | sha256sum --check --strict && \ + bash "$$installer" --quiet .PHONY: benchmark -benchmark: benchmark-deps +benchmark: install-codspeed SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode walltime --config codspeed.yml From d0257fa9f436ebd0eac78f94274f61dc5bc1a905 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 30 Oct 2026 13:42:47 -0300 Subject: [PATCH 5/8] ci: Run CodSpeed benchmarks on GitHub-hosted runners Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 84b6bec..22c64c4 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -14,7 +14,7 @@ permissions: jobs: benchmarks: name: Run benchmarks - runs-on: codspeed-macro + runs-on: ubuntu-latest steps: - name: Checkout repository From 65967ee2a4cce6ecfeb081ba20d152c6b1f0e0bb Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 30 Oct 2026 13:49:42 -0300 Subject: [PATCH 6/8] ci: Benchmark in simulation mode with fewer iterations Co-Authored-By: Claude Opus 5.5 (1M context) --- Makefile | 2 +- benchmarks/run.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a75936e..8689072 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,4 @@ install-codspeed: $(SDK_BENCHMARKS)/environments .PHONY: benchmark benchmark: install-codspeed - SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode walltime --config codspeed.yml + SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode simulation --config codspeed.yml diff --git a/benchmarks/run.rb b/benchmarks/run.rb index bc9c457..b5b3a4e 100644 --- a/benchmarks/run.rb +++ b/benchmarks/run.rb @@ -3,7 +3,7 @@ require 'bundler/setup' require_relative 'sdk_benchmark' -REVOLUTIONS = 1000 +REVOLUTIONS = 10 benchmark, environment = ARGV sdk_benchmarks_dir = ENV.fetch('SDK_BENCHMARKS', '') From 867c9bb8648451a5312acf50768e1945fd97a10f Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Fri, 30 Oct 2026 17:23:40 +0000 Subject: [PATCH 7/8] ci: Measure only SDK calls in CodSpeed using instrument-hooks (#119) --- .gitmodules | 3 ++ Makefile | 12 +++++- benchmarks/codspeed/codspeed.c | 60 ++++++++++++++++++++++++++++ benchmarks/codspeed/extconf.rb | 22 ++++++++++ benchmarks/codspeed/instrument-hooks | 1 + benchmarks/run.rb | 40 ++++++++++++++----- codspeed.yml | 32 +-------------- 7 files changed, 130 insertions(+), 40 deletions(-) create mode 100644 benchmarks/codspeed/codspeed.c create mode 100644 benchmarks/codspeed/extconf.rb create mode 160000 benchmarks/codspeed/instrument-hooks diff --git a/.gitmodules b/.gitmodules index 0c86e07..72eee3a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = spec/sdk-benchmarks url = https://github.com/Flagsmith/sdk-benchmarks.git branch = 0.0.2 +[submodule "benchmarks/codspeed/instrument-hooks"] + path = benchmarks/codspeed/instrument-hooks + url = https://github.com/CodSpeedHQ/instrument-hooks.git diff --git a/Makefile b/Makefile index 8689072..776d4d4 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,19 @@ SDK_BENCHMARKS := spec/sdk-benchmarks CODSPEED_VERSION := 5.6.0 CODSPEED_INSTALLER_SHA256 := bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40 +INSTRUMENT_HOOKS := benchmarks/codspeed/instrument-hooks +CODSPEED_EXT := tmp/codspeed/codspeed.$(shell ruby -e 'print RbConfig::CONFIG["DLEXT"]') + $(SDK_BENCHMARKS)/environments: git submodule update --init -- $(SDK_BENCHMARKS) +$(INSTRUMENT_HOOKS)/dist/core.c: + git submodule update --init -- $(INSTRUMENT_HOOKS) + +$(CODSPEED_EXT): benchmarks/codspeed/codspeed.c benchmarks/codspeed/extconf.rb $(INSTRUMENT_HOOKS)/dist/core.c + mkdir -p $(@D) + cd $(@D) && ruby $(CURDIR)/benchmarks/codspeed/extconf.rb && $(MAKE) + .PHONY: install-codspeed install-codspeed: $(SDK_BENCHMARKS)/environments bundle install @@ -14,5 +24,5 @@ install-codspeed: $(SDK_BENCHMARKS)/environments bash "$$installer" --quiet .PHONY: benchmark -benchmark: install-codspeed +benchmark: install-codspeed $(CODSPEED_EXT) SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode simulation --config codspeed.yml diff --git a/benchmarks/codspeed/codspeed.c b/benchmarks/codspeed/codspeed.c new file mode 100644 index 0000000..b9000b0 --- /dev/null +++ b/benchmarks/codspeed/codspeed.c @@ -0,0 +1,60 @@ +// Ruby bindings for CodSpeed's instrument-hooks, measuring a block in-process. +#include +#include + +#include "core.h" + +static InstrumentHooks *hooks; + +// Root frame of the benchmark flame graph: everything measured runs inside it. +__attribute__((noinline)) static VALUE __codspeed_root_frame__measure(VALUE _) { + return rb_yield(Qnil); +} + +static VALUE stop_measuring(VALUE uri) { + callgrind_stop_instrumentation(); + if (instrument_hooks_stop_benchmark(hooks) != 0) + rb_warn("CodSpeed: failed to stop the benchmark"); + if (instrument_hooks_set_executed_benchmark(hooks, getpid(), StringValueCStr(uri)) != 0) + rb_warn("CodSpeed: failed to report the executed benchmark"); + return Qnil; +} + +// CodSpeed.measure(uri) { ... }: measure the block once as the benchmark at uri. +static VALUE codspeed_measure(VALUE self, VALUE uri) { + rb_need_block(); + StringValueCStr(uri); + if (instrument_hooks_start_benchmark(hooks) != 0) + rb_raise(rb_eRuntimeError, "CodSpeed: failed to start the benchmark"); + // Toggle callgrind here, rather than in start_benchmark, to measure as little harness as possible. + callgrind_start_instrumentation(); + return rb_ensure(__codspeed_root_frame__measure, Qnil, stop_measuring, uri); +} + +// CodSpeed.instrumented?: whether the process runs under the CodSpeed runner. +static VALUE codspeed_instrumented_p(VALUE self) { + return instrument_hooks_is_instrumented(hooks) ? Qtrue : Qfalse; +} + +// CodSpeed.setup(integration, version, skipped_objects): register the harness with the runner. +static VALUE codspeed_setup(VALUE self, VALUE integration, VALUE version, VALUE skipped_objects) { + if (instrument_hooks_set_integration(hooks, StringValueCStr(integration), StringValueCStr(version)) != 0) + rb_warn("CodSpeed: failed to register the integration"); + // Attribute interpreter costs to the root frame, as pytest-codspeed does for libpython. + for (long i = 0; i < RARRAY_LEN(skipped_objects); i++) { + VALUE path = rb_ary_entry(skipped_objects, i); + instrument_hooks_callgrind_add_obj_skip(StringValueCStr(path)); + } + return Qnil; +} + +void Init_codspeed(void) { + hooks = instrument_hooks_init(); + if (!hooks) rb_raise(rb_eRuntimeError, "CodSpeed: failed to initialise instrument-hooks"); + instrument_hooks_set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, true); + + VALUE codspeed = rb_define_module("CodSpeed"); + rb_define_module_function(codspeed, "measure", codspeed_measure, 1); + rb_define_module_function(codspeed, "instrumented?", codspeed_instrumented_p, 0); + rb_define_module_function(codspeed, "setup", codspeed_setup, 3); +} diff --git a/benchmarks/codspeed/extconf.rb b/benchmarks/codspeed/extconf.rb new file mode 100644 index 0000000..b0a84c5 --- /dev/null +++ b/benchmarks/codspeed/extconf.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Builds the CodSpeed extension from instrument-hooks' single-file distribution. +require 'mkmf' + +instrument_hooks = File.join(__dir__, 'instrument-hooks') +abort "Missing #{instrument_hooks}: run `git submodule update --init`" unless File.exist?(File.join(instrument_hooks, 'dist', 'core.c')) + +# rubocop:disable Style/GlobalVars +$INCFLAGS << " -I#{File.join(instrument_hooks, 'includes')}" +$VPATH << File.join(instrument_hooks, 'dist') +$srcs = %w[codspeed.c core.c] +# Compile as C11 like pytest-codspeed: GCC 15 defaults to C23, which breaks zig.h's noreturn. +$CFLAGS << ' -std=gnu11' +# Silence warnings from the generated core.c, as instrument-hooks recommends. +$CFLAGS << %w[ + -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter -Wno-unused-but-set-variable + -Wno-type-limits -Wno-format -Wno-format-security -Wno-old-style-definition -Wno-unterminated-string-initialization +].join(' ').prepend(' ') +# rubocop:enable Style/GlobalVars + +create_makefile('codspeed') diff --git a/benchmarks/codspeed/instrument-hooks b/benchmarks/codspeed/instrument-hooks new file mode 160000 index 0000000..8fd5b87 --- /dev/null +++ b/benchmarks/codspeed/instrument-hooks @@ -0,0 +1 @@ +Subproject commit 8fd5b878499f819561b33219cdafaa6f8c28b4e4 diff --git a/benchmarks/run.rb b/benchmarks/run.rb index b5b3a4e..dfc969a 100644 --- a/benchmarks/run.rb +++ b/benchmarks/run.rb @@ -1,19 +1,41 @@ # frozen_string_literal: true +# Runs every SDK benchmark in a single process, measuring only the benchmarked calls. +# +# Under the CodSpeed runner, each call is measured with instrument-hooks, so Ruby start-up, +# loading the SDK and building the client stay out of the results. Elsewhere, it prints timings. require 'bundler/setup' +require 'benchmark' +require 'rbconfig' require_relative 'sdk_benchmark' -REVOLUTIONS = 10 +LOCAL_REVOLUTIONS = 100 -benchmark, environment = ARGV sdk_benchmarks_dir = ENV.fetch('SDK_BENCHMARKS', '') - -unless !sdk_benchmarks_dir.empty? && SdkBenchmark::BENCHMARKS.key?(benchmark) && SdkBenchmark::ENVIRONMENTS.include?(environment) - warn "Usage: SDK_BENCHMARKS= ruby #{$PROGRAM_NAME} " \ - "<#{SdkBenchmark::BENCHMARKS.keys.join('|')}> <#{SdkBenchmark::ENVIRONMENTS.join('|')}>" +if sdk_benchmarks_dir.empty? + warn "Usage: SDK_BENCHMARKS= ruby #{$PROGRAM_NAME}" exit 2 end -sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment) -REVOLUTIONS.times { sdk_benchmark.public_send(SdkBenchmark::BENCHMARKS.fetch(benchmark)) } -sdk_benchmark.close +codspeed = !ENV.fetch('CODSPEED_ENV', '').empty? +if codspeed + require_relative '../tmp/codspeed/codspeed' + abort 'CodSpeed: not running under instrumentation' unless CodSpeed.instrumented? + libruby = File.join(RbConfig::CONFIG['libdir'], RbConfig::CONFIG['LIBRUBY_SO']) + CodSpeed.setup('flagsmith-ruby-client', Flagsmith::VERSION, [RbConfig.ruby, libruby].select { File.exist?(_1) }) +end + +SdkBenchmark::ENVIRONMENTS.each do |environment| + sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment) + SdkBenchmark::BENCHMARKS.each do |name, method| + sdk_benchmark.public_send(method) # Warm up caches + if codspeed + GC.start + CodSpeed.measure("benchmarks/sdk_benchmark.rb::#{name}[#{environment}]") { sdk_benchmark.public_send(method) } + else + seconds = Benchmark.realtime { LOCAL_REVOLUTIONS.times { sdk_benchmark.public_send(method) } } + puts format('%-50s %10.3f ms', name: "#{name}[#{environment}]", ms: seconds * 1000 / LOCAL_REVOLUTIONS) + end + end + sdk_benchmark.close +end diff --git a/codspeed.yml b/codspeed.yml index 6400170..7fb6f72 100644 --- a/codspeed.yml +++ b/codspeed.yml @@ -1,33 +1,5 @@ $schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json benchmarks: - - name: getEnvironmentFlags[small] - exec: ruby benchmarks/run.rb getEnvironmentFlags small - - name: getEnvironmentFlags[medium] - exec: ruby benchmarks/run.rb getEnvironmentFlags medium - - name: getEnvironmentFlags[large] - exec: ruby benchmarks/run.rb getEnvironmentFlags large - - name: getIdentityFlagsWithIdentityOverrides[small] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides small - - name: getIdentityFlagsWithIdentityOverrides[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides medium - - name: getIdentityFlagsWithIdentityOverrides[large] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides large - - name: getIdentityFlagsNoTraits[small] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits small - - name: getIdentityFlagsNoTraits[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits medium - - name: getIdentityFlagsNoTraits[large] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits large - - name: getIdentityFlagsWithTraits[small] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits small - - name: getIdentityFlagsWithTraits[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits medium - - name: getIdentityFlagsWithTraits[large] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits large - - name: getIdentitySegmentsWithTraits[small] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits small - - name: getIdentitySegmentsWithTraits[medium] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits medium - - name: getIdentitySegmentsWithTraits[large] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits large + - name: SDK benchmarks + entrypoint: ruby benchmarks/run.rb From cbb2a43b1548f9a410312acdcb99e489a8d2e61f Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 30 Oct 2026 14:27:43 -0300 Subject: [PATCH 8/8] hola que pasa