diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000..22c64c4 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,31 @@ +name: CodSpeed + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read # Check out the repository and its submodules + id-token: write # Authenticate result uploads to CodSpeed via OIDC + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Ruby + uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f # v1.335.0 + with: + ruby-version: '3.3' + + - name: Run benchmarks + run: make benchmark diff --git a/.gitmodules b/.gitmodules index b2bee4c..72eee3a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,10 @@ path = spec/engine-test-data url = git@github.com:Flagsmith/engine-test-data.git branch = v3.7.0 +[submodule "spec/sdk-benchmarks"] + path = spec/sdk-benchmarks + url = https://github.com/Flagsmith/sdk-benchmarks.git + branch = 0.0.2 +[submodule "benchmarks/codspeed/instrument-hooks"] + path = benchmarks/codspeed/instrument-hooks + url = https://github.com/CodSpeedHQ/instrument-hooks.git diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..776d4d4 --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +SDK_BENCHMARKS := spec/sdk-benchmarks +CODSPEED_VERSION := 5.6.0 +CODSPEED_INSTALLER_SHA256 := bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40 + +INSTRUMENT_HOOKS := benchmarks/codspeed/instrument-hooks +CODSPEED_EXT := tmp/codspeed/codspeed.$(shell ruby -e 'print RbConfig::CONFIG["DLEXT"]') + +$(SDK_BENCHMARKS)/environments: + git submodule update --init -- $(SDK_BENCHMARKS) + +$(INSTRUMENT_HOOKS)/dist/core.c: + git submodule update --init -- $(INSTRUMENT_HOOKS) + +$(CODSPEED_EXT): benchmarks/codspeed/codspeed.c benchmarks/codspeed/extconf.rb $(INSTRUMENT_HOOKS)/dist/core.c + mkdir -p $(@D) + cd $(@D) && ruby $(CURDIR)/benchmarks/codspeed/extconf.rb && $(MAKE) + +.PHONY: install-codspeed +install-codspeed: $(SDK_BENCHMARKS)/environments + bundle install + installer=$$(mktemp) && \ + curl -fsSL -o "$$installer" "https://codspeed.io/v$(CODSPEED_VERSION)/install.sh" && \ + echo "$(CODSPEED_INSTALLER_SHA256) $$installer" | sha256sum --check --strict && \ + bash "$$installer" --quiet + +.PHONY: benchmark +benchmark: install-codspeed $(CODSPEED_EXT) + SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode simulation --config codspeed.yml diff --git a/benchmarks/codspeed/codspeed.c b/benchmarks/codspeed/codspeed.c new file mode 100644 index 0000000..b9000b0 --- /dev/null +++ b/benchmarks/codspeed/codspeed.c @@ -0,0 +1,60 @@ +// Ruby bindings for CodSpeed's instrument-hooks, measuring a block in-process. +#include +#include + +#include "core.h" + +static InstrumentHooks *hooks; + +// Root frame of the benchmark flame graph: everything measured runs inside it. +__attribute__((noinline)) static VALUE __codspeed_root_frame__measure(VALUE _) { + return rb_yield(Qnil); +} + +static VALUE stop_measuring(VALUE uri) { + callgrind_stop_instrumentation(); + if (instrument_hooks_stop_benchmark(hooks) != 0) + rb_warn("CodSpeed: failed to stop the benchmark"); + if (instrument_hooks_set_executed_benchmark(hooks, getpid(), StringValueCStr(uri)) != 0) + rb_warn("CodSpeed: failed to report the executed benchmark"); + return Qnil; +} + +// CodSpeed.measure(uri) { ... }: measure the block once as the benchmark at uri. +static VALUE codspeed_measure(VALUE self, VALUE uri) { + rb_need_block(); + StringValueCStr(uri); + if (instrument_hooks_start_benchmark(hooks) != 0) + rb_raise(rb_eRuntimeError, "CodSpeed: failed to start the benchmark"); + // Toggle callgrind here, rather than in start_benchmark, to measure as little harness as possible. + callgrind_start_instrumentation(); + return rb_ensure(__codspeed_root_frame__measure, Qnil, stop_measuring, uri); +} + +// CodSpeed.instrumented?: whether the process runs under the CodSpeed runner. +static VALUE codspeed_instrumented_p(VALUE self) { + return instrument_hooks_is_instrumented(hooks) ? Qtrue : Qfalse; +} + +// CodSpeed.setup(integration, version, skipped_objects): register the harness with the runner. +static VALUE codspeed_setup(VALUE self, VALUE integration, VALUE version, VALUE skipped_objects) { + if (instrument_hooks_set_integration(hooks, StringValueCStr(integration), StringValueCStr(version)) != 0) + rb_warn("CodSpeed: failed to register the integration"); + // Attribute interpreter costs to the root frame, as pytest-codspeed does for libpython. + for (long i = 0; i < RARRAY_LEN(skipped_objects); i++) { + VALUE path = rb_ary_entry(skipped_objects, i); + instrument_hooks_callgrind_add_obj_skip(StringValueCStr(path)); + } + return Qnil; +} + +void Init_codspeed(void) { + hooks = instrument_hooks_init(); + if (!hooks) rb_raise(rb_eRuntimeError, "CodSpeed: failed to initialise instrument-hooks"); + instrument_hooks_set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, true); + + VALUE codspeed = rb_define_module("CodSpeed"); + rb_define_module_function(codspeed, "measure", codspeed_measure, 1); + rb_define_module_function(codspeed, "instrumented?", codspeed_instrumented_p, 0); + rb_define_module_function(codspeed, "setup", codspeed_setup, 3); +} diff --git a/benchmarks/codspeed/extconf.rb b/benchmarks/codspeed/extconf.rb new file mode 100644 index 0000000..b0a84c5 --- /dev/null +++ b/benchmarks/codspeed/extconf.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Builds the CodSpeed extension from instrument-hooks' single-file distribution. +require 'mkmf' + +instrument_hooks = File.join(__dir__, 'instrument-hooks') +abort "Missing #{instrument_hooks}: run `git submodule update --init`" unless File.exist?(File.join(instrument_hooks, 'dist', 'core.c')) + +# rubocop:disable Style/GlobalVars +$INCFLAGS << " -I#{File.join(instrument_hooks, 'includes')}" +$VPATH << File.join(instrument_hooks, 'dist') +$srcs = %w[codspeed.c core.c] +# Compile as C11 like pytest-codspeed: GCC 15 defaults to C23, which breaks zig.h's noreturn. +$CFLAGS << ' -std=gnu11' +# Silence warnings from the generated core.c, as instrument-hooks recommends. +$CFLAGS << %w[ + -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter -Wno-unused-but-set-variable + -Wno-type-limits -Wno-format -Wno-format-security -Wno-old-style-definition -Wno-unterminated-string-initialization +].join(' ').prepend(' ') +# rubocop:enable Style/GlobalVars + +create_makefile('codspeed') diff --git a/benchmarks/codspeed/instrument-hooks b/benchmarks/codspeed/instrument-hooks new file mode 160000 index 0000000..8fd5b87 --- /dev/null +++ b/benchmarks/codspeed/instrument-hooks @@ -0,0 +1 @@ +Subproject commit 8fd5b878499f819561b33219cdafaa6f8c28b4e4 diff --git a/benchmarks/run.rb b/benchmarks/run.rb new file mode 100644 index 0000000..dfc969a --- /dev/null +++ b/benchmarks/run.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Runs every SDK benchmark in a single process, measuring only the benchmarked calls. +# +# Under the CodSpeed runner, each call is measured with instrument-hooks, so Ruby start-up, +# loading the SDK and building the client stay out of the results. Elsewhere, it prints timings. +require 'bundler/setup' +require 'benchmark' +require 'rbconfig' +require_relative 'sdk_benchmark' + +LOCAL_REVOLUTIONS = 100 + +sdk_benchmarks_dir = ENV.fetch('SDK_BENCHMARKS', '') +if sdk_benchmarks_dir.empty? + warn "Usage: SDK_BENCHMARKS= ruby #{$PROGRAM_NAME}" + exit 2 +end + +codspeed = !ENV.fetch('CODSPEED_ENV', '').empty? +if codspeed + require_relative '../tmp/codspeed/codspeed' + abort 'CodSpeed: not running under instrumentation' unless CodSpeed.instrumented? + libruby = File.join(RbConfig::CONFIG['libdir'], RbConfig::CONFIG['LIBRUBY_SO']) + CodSpeed.setup('flagsmith-ruby-client', Flagsmith::VERSION, [RbConfig.ruby, libruby].select { File.exist?(_1) }) +end + +SdkBenchmark::ENVIRONMENTS.each do |environment| + sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment) + SdkBenchmark::BENCHMARKS.each do |name, method| + sdk_benchmark.public_send(method) # Warm up caches + if codspeed + GC.start + CodSpeed.measure("benchmarks/sdk_benchmark.rb::#{name}[#{environment}]") { sdk_benchmark.public_send(method) } + else + seconds = Benchmark.realtime { LOCAL_REVOLUTIONS.times { sdk_benchmark.public_send(method) } } + puts format('%-50s %10.3f ms', name: "#{name}[#{environment}]", ms: seconds * 1000 / LOCAL_REVOLUTIONS) + end + end + sdk_benchmark.close +end diff --git a/benchmarks/sdk_benchmark.rb b/benchmarks/sdk_benchmark.rb new file mode 100644 index 0000000..2cabb17 --- /dev/null +++ b/benchmarks/sdk_benchmark.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'flagsmith' + +# Benchmarks a locally evaluating SDK client against environment documents tiered by size. +class SdkBenchmark + ENVIRONMENTS = %w[small medium large].freeze + BENCHMARKS = { + 'getEnvironmentFlags' => :get_environment_flags, + 'getIdentityFlagsWithIdentityOverrides' => :get_identity_flags_with_identity_overrides, + 'getIdentityFlagsNoTraits' => :get_identity_flags_no_traits, + 'getIdentityFlagsWithTraits' => :get_identity_flags_with_traits, + 'getIdentitySegmentsWithTraits' => :get_identity_segments_with_traits + }.freeze + + IDENTITY_OVERRIDES_IDENTIFIER = 'identity-override-1' + IDENTIFIER = 'user-1' + TRAITS = { + plan: 'plan-7', + age: 30, + country: 'GB', + version: '2.5.1', + email: 'jane@example.com', + user_id: 12, + beta: true, + score: 95, + tier: 'pro', + device: 'iPhone 15' + }.freeze + + # Build a locally evaluating client serving the environment document from memory. + def initialize(sdk_benchmarks_dir, environment) + environment_document = File.read(File.join(sdk_benchmarks_dir, 'environments', "#{environment}.json")) + Faraday.default_adapter = serve_environment_document(environment_document) + @client = Flagsmith::Client.new( + environment_key: JSON.parse(environment_document).fetch('api_key'), + api_url: 'http://flagsmith.benchmark/api/v1/', + enable_local_evaluation: true, + environment_refresh_interval_seconds: 86_400, # Refreshed by benchmarks only + logger: Logger.new(File::NULL) + ) + end + + def close + @client.environment_data_polling_manager.stop + end + + # rubocop:disable Naming/AccessorMethodName + + # Get a feature value from the environment flags. + def get_environment_flags + @client.get_environment_flags.feature_value('feature_1') + end + + # Check a feature for an identity with identity overrides. + def get_identity_flags_with_identity_overrides + @client.get_identity_flags(IDENTITY_OVERRIDES_IDENTIFIER).feature_enabled?('feature_8') + end + + # Get a feature value for an identity without traits. + def get_identity_flags_no_traits + @client.get_identity_flags(IDENTIFIER).feature_value('feature_1') + end + + # Get a feature value for an identity with traits. + def get_identity_flags_with_traits + @client.get_identity_flags(IDENTIFIER, **TRAITS).feature_value('feature_1') + end + + # List the segments of an identity with traits. + def get_identity_segments_with_traits + @client.get_identity_segments(IDENTIFIER, TRAITS) + end + + # rubocop:enable Naming/AccessorMethodName + + private + + def serve_environment_document(environment_document) + Class.new(Faraday::Adapter) do + define_method(:call) do |env| + super(env) + save_response(env, 200, environment_document, { 'Content-Type' => 'application/json' }) + @app.call(env) + end + end + end +end diff --git a/codspeed.yml b/codspeed.yml new file mode 100644 index 0000000..7fb6f72 --- /dev/null +++ b/codspeed.yml @@ -0,0 +1,5 @@ +$schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json + +benchmarks: + - name: SDK benchmarks + entrypoint: ruby benchmarks/run.rb diff --git a/flagsmith.gemspec b/flagsmith.gemspec index 224ed9e..8dd2713 100644 --- a/flagsmith.gemspec +++ b/flagsmith.gemspec @@ -12,7 +12,7 @@ Gem::Specification.new do |spec| # The `git ls-files -z` loads the files in the RubyGem that have been added into git. spec.files = Dir.chdir(__dir__) do `git ls-files -z`.split("\x0").reject do |f| - (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|sdk-spec|features)/|\.(?:git|travis|circleci)|appveyor)}) + (f == __FILE__) || f.match(%r{\A(?:(?:benchmarks|bin|test|spec|sdk-spec|features)/|\.(?:git|travis|circleci)|appveyor)}) end end diff --git a/spec/sdk-benchmarks b/spec/sdk-benchmarks new file mode 160000 index 0000000..7da2be4 --- /dev/null +++ b/spec/sdk-benchmarks @@ -0,0 +1 @@ +Subproject commit 7da2be42ec7c58a597892e41141d531bd1dd100c