ci: Benchmark the SDK with CodSpeed

.github/workflows/codspeed.yml

@@ -0,0 +1,31 @@
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read  # Check out the repository and its submodules
+  id-token: write  # Authenticate result uploads to CodSpeed via OIDC
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f  # v1.335.0
+        with:
+          ruby-version: '3.3'
+
+      - name: Run benchmarks
+        run: make benchmark

.gitmodules

@@ -2,3 +2,10 @@
 	path = spec/engine-test-data
 	url = git@github.com:Flagsmith/engine-test-data.git
 	branch = v3.7.0
+[submodule "spec/sdk-benchmarks"]
+	path = spec/sdk-benchmarks
+	url = https://github.com/Flagsmith/sdk-benchmarks.git
+	branch = 0.0.2
+[submodule "benchmarks/codspeed/instrument-hooks"]
+	path = benchmarks/codspeed/instrument-hooks
+	url = https://github.com/CodSpeedHQ/instrument-hooks.git

Makefile

@@ -0,0 +1,28 @@
+SDK_BENCHMARKS := spec/sdk-benchmarks
+CODSPEED_VERSION := 5.6.0
+CODSPEED_INSTALLER_SHA256 := bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40
+
+INSTRUMENT_HOOKS := benchmarks/codspeed/instrument-hooks
+CODSPEED_EXT := tmp/codspeed/codspeed.$(shell ruby -e 'print RbConfig::CONFIG["DLEXT"]')
+
+$(SDK_BENCHMARKS)/environments:
+	git submodule update --init -- $(SDK_BENCHMARKS)
+
+$(INSTRUMENT_HOOKS)/dist/core.c:
+	git submodule update --init -- $(INSTRUMENT_HOOKS)
+
+$(CODSPEED_EXT): benchmarks/codspeed/codspeed.c benchmarks/codspeed/extconf.rb $(INSTRUMENT_HOOKS)/dist/core.c
+	mkdir -p $(@D)
+	cd $(@D) && ruby $(CURDIR)/benchmarks/codspeed/extconf.rb && $(MAKE)
+
+.PHONY: install-codspeed
+install-codspeed: $(SDK_BENCHMARKS)/environments
+	bundle install
+	installer=$$(mktemp) && \
+	curl -fsSL -o "$$installer" "https://codspeed.io/v$(CODSPEED_VERSION)/install.sh" && \
+	echo "$(CODSPEED_INSTALLER_SHA256)  $$installer" | sha256sum --check --strict && \
+	bash "$$installer" --quiet
+
+.PHONY: benchmark
+benchmark: install-codspeed $(CODSPEED_EXT)
+	SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode simulation --config codspeed.yml

benchmarks/codspeed/codspeed.c

@@ -0,0 +1,60 @@
+// Ruby bindings for CodSpeed's instrument-hooks, measuring a block in-process.
+#include <ruby.h>
+#include <unistd.h>
+
+#include "core.h"
+
+static InstrumentHooks *hooks;
+
+// Root frame of the benchmark flame graph: everything measured runs inside it.
+__attribute__((noinline)) static VALUE __codspeed_root_frame__measure(VALUE _) {
+  return rb_yield(Qnil);
+}
+
+static VALUE stop_measuring(VALUE uri) {
+  callgrind_stop_instrumentation();
+  if (instrument_hooks_stop_benchmark(hooks) != 0)
+    rb_warn("CodSpeed: failed to stop the benchmark");
+  if (instrument_hooks_set_executed_benchmark(hooks, getpid(), StringValueCStr(uri)) != 0)
+    rb_warn("CodSpeed: failed to report the executed benchmark");
+  return Qnil;
+}
+
+// CodSpeed.measure(uri) { ... }: measure the block once as the benchmark at uri.
+static VALUE codspeed_measure(VALUE self, VALUE uri) {
+  rb_need_block();
+  StringValueCStr(uri);
+  if (instrument_hooks_start_benchmark(hooks) != 0)
+    rb_raise(rb_eRuntimeError, "CodSpeed: failed to start the benchmark");
+  // Toggle callgrind here, rather than in start_benchmark, to measure as little harness as possible.
+  callgrind_start_instrumentation();
+  return rb_ensure(__codspeed_root_frame__measure, Qnil, stop_measuring, uri);
+}
+
+// CodSpeed.instrumented?: whether the process runs under the CodSpeed runner.
+static VALUE codspeed_instrumented_p(VALUE self) {
+  return instrument_hooks_is_instrumented(hooks) ? Qtrue : Qfalse;
+}
+
+// CodSpeed.setup(integration, version, skipped_objects): register the harness with the runner.
+static VALUE codspeed_setup(VALUE self, VALUE integration, VALUE version, VALUE skipped_objects) {
+  if (instrument_hooks_set_integration(hooks, StringValueCStr(integration), StringValueCStr(version)) != 0)
+    rb_warn("CodSpeed: failed to register the integration");
+  // Attribute interpreter costs to the root frame, as pytest-codspeed does for libpython.
+  for (long i = 0; i < RARRAY_LEN(skipped_objects); i++) {
+    VALUE path = rb_ary_entry(skipped_objects, i);
+    instrument_hooks_callgrind_add_obj_skip(StringValueCStr(path));
+  }
+  return Qnil;
+}
+
+void Init_codspeed(void) {
+  hooks = instrument_hooks_init();
+  if (!hooks) rb_raise(rb_eRuntimeError, "CodSpeed: failed to initialise instrument-hooks");
+  instrument_hooks_set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, true);
+
+  VALUE codspeed = rb_define_module("CodSpeed");
+  rb_define_module_function(codspeed, "measure", codspeed_measure, 1);
+  rb_define_module_function(codspeed, "instrumented?", codspeed_instrumented_p, 0);
+  rb_define_module_function(codspeed, "setup", codspeed_setup, 3);
+}

benchmarks/codspeed/extconf.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# Builds the CodSpeed extension from instrument-hooks' single-file distribution.
+require 'mkmf'
+
+instrument_hooks = File.join(__dir__, 'instrument-hooks')
+abort "Missing #{instrument_hooks}: run `git submodule update --init`" unless File.exist?(File.join(instrument_hooks, 'dist', 'core.c'))
+
+# rubocop:disable Style/GlobalVars
+$INCFLAGS << " -I#{File.join(instrument_hooks, 'includes')}"
+$VPATH << File.join(instrument_hooks, 'dist')
+$srcs = %w[codspeed.c core.c]
+# Compile as C11 like pytest-codspeed: GCC 15 defaults to C23, which breaks zig.h's noreturn.
+$CFLAGS << ' -std=gnu11'
+# Silence warnings from the generated core.c, as instrument-hooks recommends.
+$CFLAGS << %w[
+  -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter -Wno-unused-but-set-variable
+  -Wno-type-limits -Wno-format -Wno-format-security -Wno-old-style-definition -Wno-unterminated-string-initialization
+].join(' ').prepend(' ')
+# rubocop:enable Style/GlobalVars
+
+create_makefile('codspeed')

benchmarks/run.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Runs every SDK benchmark in a single process, measuring only the benchmarked calls.
+#
+# Under the CodSpeed runner, each call is measured with instrument-hooks, so Ruby start-up,
+# loading the SDK and building the client stay out of the results. Elsewhere, it prints timings.
+require 'bundler/setup'
+require 'benchmark'
+require 'rbconfig'
+require_relative 'sdk_benchmark'
+
+LOCAL_REVOLUTIONS = 100
+
+sdk_benchmarks_dir = ENV.fetch('SDK_BENCHMARKS', '')
+if sdk_benchmarks_dir.empty?
+  warn "Usage: SDK_BENCHMARKS=<dir> ruby #{$PROGRAM_NAME}"
+  exit 2
+end
+
+codspeed = !ENV.fetch('CODSPEED_ENV', '').empty?
+if codspeed
+  require_relative '../tmp/codspeed/codspeed'
+  abort 'CodSpeed: not running under instrumentation' unless CodSpeed.instrumented?
+  libruby = File.join(RbConfig::CONFIG['libdir'], RbConfig::CONFIG['LIBRUBY_SO'])
+  CodSpeed.setup('flagsmith-ruby-client', Flagsmith::VERSION, [RbConfig.ruby, libruby].select { File.exist?(_1) })
+end
+
+SdkBenchmark::ENVIRONMENTS.each do |environment|
+  sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment)
+  SdkBenchmark::BENCHMARKS.each do |name, method|
+    sdk_benchmark.public_send(method) # Warm up caches
+    if codspeed
+      GC.start
+      CodSpeed.measure("benchmarks/sdk_benchmark.rb::#{name}[#{environment}]") { sdk_benchmark.public_send(method) }
+    else
+      seconds = Benchmark.realtime { LOCAL_REVOLUTIONS.times { sdk_benchmark.public_send(method) } }
+      puts format('%<name>-50s %<ms>10.3f ms', name: "#{name}[#{environment}]", ms: seconds * 1000 / LOCAL_REVOLUTIONS)
+    end
+  end
+  sdk_benchmark.close
+end

benchmarks/sdk_benchmark.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'flagsmith'
+
+# Benchmarks a locally evaluating SDK client against environment documents tiered by size.
+class SdkBenchmark
+  ENVIRONMENTS = %w[small medium large].freeze
+  BENCHMARKS = {
+    'getEnvironmentFlags' => :get_environment_flags,
+    'getIdentityFlagsWithIdentityOverrides' => :get_identity_flags_with_identity_overrides,
+    'getIdentityFlagsNoTraits' => :get_identity_flags_no_traits,
+    'getIdentityFlagsWithTraits' => :get_identity_flags_with_traits,
+    'getIdentitySegmentsWithTraits' => :get_identity_segments_with_traits
+  }.freeze
+
+  IDENTITY_OVERRIDES_IDENTIFIER = 'identity-override-1'
+  IDENTIFIER = 'user-1'
+  TRAITS = {
+    plan: 'plan-7',
+    age: 30,
+    country: 'GB',
+    version: '2.5.1',
+    email: 'jane@example.com',
+    user_id: 12,
+    beta: true,
+    score: 95,
+    tier: 'pro',
+    device: 'iPhone 15'
+  }.freeze
+
+  # Build a locally evaluating client serving the environment document from memory.
+  def initialize(sdk_benchmarks_dir, environment)
+    environment_document = File.read(File.join(sdk_benchmarks_dir, 'environments', "#{environment}.json"))
+    Faraday.default_adapter = serve_environment_document(environment_document)
+    @client = Flagsmith::Client.new(
+      environment_key: JSON.parse(environment_document).fetch('api_key'),
+      api_url: 'http://flagsmith.benchmark/api/v1/',
+      enable_local_evaluation: true,
+      environment_refresh_interval_seconds: 86_400, # Refreshed by benchmarks only
+      logger: Logger.new(File::NULL)
+    )
+  end
+
+  def close
+    @client.environment_data_polling_manager.stop
+  end
+
+  # rubocop:disable Naming/AccessorMethodName
+
+  # Get a feature value from the environment flags.
+  def get_environment_flags
+    @client.get_environment_flags.feature_value('feature_1')
+  end
+
+  # Check a feature for an identity with identity overrides.
+  def get_identity_flags_with_identity_overrides
+    @client.get_identity_flags(IDENTITY_OVERRIDES_IDENTIFIER).feature_enabled?('feature_8')
+  end
+
+  # Get a feature value for an identity without traits.
+  def get_identity_flags_no_traits
+    @client.get_identity_flags(IDENTIFIER).feature_value('feature_1')
+  end
+
+  # Get a feature value for an identity with traits.
+  def get_identity_flags_with_traits
+    @client.get_identity_flags(IDENTIFIER, **TRAITS).feature_value('feature_1')
+  end
+
+  # List the segments of an identity with traits.
+  def get_identity_segments_with_traits
+    @client.get_identity_segments(IDENTIFIER, TRAITS)
+  end
+
+  # rubocop:enable Naming/AccessorMethodName
+
+  private
+
+  def serve_environment_document(environment_document)
+    Class.new(Faraday::Adapter) do
+      define_method(:call) do |env|
+        super(env)
+        save_response(env, 200, environment_document, { 'Content-Type' => 'application/json' })
+        @app.call(env)
+      end
+    end
+  end
+end

codspeed.yml

@@ -0,0 +1,5 @@
+$schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json
+
+benchmarks:
+  - name: SDK benchmarks
+    entrypoint: ruby benchmarks/run.rb

flagsmith.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(__dir__) do
     `git ls-files -z`.split("\x0").reject do |f|
-      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|sdk-spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+      (f == __FILE__) || f.match(%r{\A(?:(?:benchmarks|bin|test|spec|sdk-spec|features)/|\.(?:git|travis|circleci)|appveyor)})
     end
   end