ci: Measure only SDK calls in CodSpeed using instrument-hooks

.gitmodules

@@ -6,3 +6,6 @@
 	path = spec/sdk-benchmarks
 	url = https://github.com/Flagsmith/sdk-benchmarks.git
 	branch = 0.0.2
+[submodule "benchmarks/codspeed/instrument-hooks"]
+	path = benchmarks/codspeed/instrument-hooks
+	url = https://github.com/CodSpeedHQ/instrument-hooks.git

Makefile

@@ -2,9 +2,19 @@ SDK_BENCHMARKS := spec/sdk-benchmarks
 CODSPEED_VERSION := 5.6.0
 CODSPEED_INSTALLER_SHA256 := bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40
 
+INSTRUMENT_HOOKS := benchmarks/codspeed/instrument-hooks
+CODSPEED_EXT := tmp/codspeed/codspeed.$(shell ruby -e 'print RbConfig::CONFIG["DLEXT"]')
+
 $(SDK_BENCHMARKS)/environments:
 	git submodule update --init -- $(SDK_BENCHMARKS)
 
+$(INSTRUMENT_HOOKS)/dist/core.c:
+	git submodule update --init -- $(INSTRUMENT_HOOKS)
+
+$(CODSPEED_EXT): benchmarks/codspeed/codspeed.c benchmarks/codspeed/extconf.rb $(INSTRUMENT_HOOKS)/dist/core.c
+	mkdir -p $(@D)
+	cd $(@D) && ruby $(CURDIR)/benchmarks/codspeed/extconf.rb && $(MAKE)
+
 .PHONY: install-codspeed
 install-codspeed: $(SDK_BENCHMARKS)/environments
 	bundle install
@@ -14,5 +24,5 @@ install-codspeed: $(SDK_BENCHMARKS)/environments
 	bash "$$installer" --quiet
 
 .PHONY: benchmark
-benchmark: install-codspeed
+benchmark: install-codspeed $(CODSPEED_EXT)
 	SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode simulation --config codspeed.yml

benchmarks/codspeed/codspeed.c

@@ -0,0 +1,60 @@
+// Ruby bindings for CodSpeed's instrument-hooks, measuring a block in-process.
+#include <ruby.h>
+#include <unistd.h>
+
+#include "core.h"
+
+static InstrumentHooks *hooks;
+
+// Root frame of the benchmark flame graph: everything measured runs inside it.
+__attribute__((noinline)) static VALUE __codspeed_root_frame__measure(VALUE _) {
+  return rb_yield(Qnil);
+}
+
+static VALUE stop_measuring(VALUE uri) {
+  callgrind_stop_instrumentation();
+  if (instrument_hooks_stop_benchmark(hooks) != 0)
+    rb_warn("CodSpeed: failed to stop the benchmark");
+  if (instrument_hooks_set_executed_benchmark(hooks, getpid(), StringValueCStr(uri)) != 0)
+    rb_warn("CodSpeed: failed to report the executed benchmark");
+  return Qnil;
+}
+
+// CodSpeed.measure(uri) { ... }: measure the block once as the benchmark at uri.
+static VALUE codspeed_measure(VALUE self, VALUE uri) {
+  rb_need_block();
+  StringValueCStr(uri);
+  if (instrument_hooks_start_benchmark(hooks) != 0)
+    rb_raise(rb_eRuntimeError, "CodSpeed: failed to start the benchmark");
+  // Toggle callgrind here, rather than in start_benchmark, to measure as little harness as possible.
+  callgrind_start_instrumentation();
+  return rb_ensure(__codspeed_root_frame__measure, Qnil, stop_measuring, uri);
+}
+
+// CodSpeed.instrumented?: whether the process runs under the CodSpeed runner.
+static VALUE codspeed_instrumented_p(VALUE self) {
+  return instrument_hooks_is_instrumented(hooks) ? Qtrue : Qfalse;
+}
+
+// CodSpeed.setup(integration, version, skipped_objects): register the harness with the runner.
+static VALUE codspeed_setup(VALUE self, VALUE integration, VALUE version, VALUE skipped_objects) {
+  if (instrument_hooks_set_integration(hooks, StringValueCStr(integration), StringValueCStr(version)) != 0)
+    rb_warn("CodSpeed: failed to register the integration");
+  // Attribute interpreter costs to the root frame, as pytest-codspeed does for libpython.
+  for (long i = 0; i < RARRAY_LEN(skipped_objects); i++) {
+    VALUE path = rb_ary_entry(skipped_objects, i);
+    instrument_hooks_callgrind_add_obj_skip(StringValueCStr(path));
+  }
+  return Qnil;
+}
+
+void Init_codspeed(void) {
+  hooks = instrument_hooks_init();
+  if (!hooks) rb_raise(rb_eRuntimeError, "CodSpeed: failed to initialise instrument-hooks");
+  instrument_hooks_set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, true);
+
+  VALUE codspeed = rb_define_module("CodSpeed");
+  rb_define_module_function(codspeed, "measure", codspeed_measure, 1);
+  rb_define_module_function(codspeed, "instrumented?", codspeed_instrumented_p, 0);
+  rb_define_module_function(codspeed, "setup", codspeed_setup, 3);
+}

benchmarks/codspeed/extconf.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# Builds the CodSpeed extension from instrument-hooks' single-file distribution.
+require 'mkmf'
+
+instrument_hooks = File.join(__dir__, 'instrument-hooks')
+abort "Missing #{instrument_hooks}: run `git submodule update --init`" unless File.exist?(File.join(instrument_hooks, 'dist', 'core.c'))
+
+# rubocop:disable Style/GlobalVars
+$INCFLAGS << " -I#{File.join(instrument_hooks, 'includes')}"
+$VPATH << File.join(instrument_hooks, 'dist')
+$srcs = %w[codspeed.c core.c]
+# Compile as C11 like pytest-codspeed: GCC 15 defaults to C23, which breaks zig.h's noreturn.
+$CFLAGS << ' -std=gnu11'
+# Silence warnings from the generated core.c, as instrument-hooks recommends.
+$CFLAGS << %w[
+  -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter -Wno-unused-but-set-variable
+  -Wno-type-limits -Wno-format -Wno-format-security -Wno-old-style-definition -Wno-unterminated-string-initialization
+].join(' ').prepend(' ')
+# rubocop:enable Style/GlobalVars
+
+create_makefile('codspeed')

benchmarks/run.rb

@@ -1,19 +1,41 @@
 # frozen_string_literal: true
 
+# Runs every SDK benchmark in a single process, measuring only the benchmarked calls.
+#
+# Under the CodSpeed runner, each call is measured with instrument-hooks, so Ruby start-up,
+# loading the SDK and building the client stay out of the results. Elsewhere, it prints timings.
 require 'bundler/setup'
+require 'benchmark'
+require 'rbconfig'
 require_relative 'sdk_benchmark'
 
-REVOLUTIONS = 10
+LOCAL_REVOLUTIONS = 100
 
-benchmark, environment = ARGV
 sdk_benchmarks_dir = ENV.fetch('SDK_BENCHMARKS', '')
-
-unless !sdk_benchmarks_dir.empty? && SdkBenchmark::BENCHMARKS.key?(benchmark) && SdkBenchmark::ENVIRONMENTS.include?(environment)
-  warn "Usage: SDK_BENCHMARKS=<dir> ruby #{$PROGRAM_NAME} " \
-       "<#{SdkBenchmark::BENCHMARKS.keys.join('|')}> <#{SdkBenchmark::ENVIRONMENTS.join('|')}>"
+if sdk_benchmarks_dir.empty?
+  warn "Usage: SDK_BENCHMARKS=<dir> ruby #{$PROGRAM_NAME}"
   exit 2
 end
 
-sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment)
-REVOLUTIONS.times { sdk_benchmark.public_send(SdkBenchmark::BENCHMARKS.fetch(benchmark)) }
-sdk_benchmark.close
+codspeed = !ENV.fetch('CODSPEED_ENV', '').empty?
+if codspeed
+  require_relative '../tmp/codspeed/codspeed'
+  abort 'CodSpeed: not running under instrumentation' unless CodSpeed.instrumented?
+  libruby = File.join(RbConfig::CONFIG['libdir'], RbConfig::CONFIG['LIBRUBY_SO'])
+  CodSpeed.setup('flagsmith-ruby-client', Flagsmith::VERSION, [RbConfig.ruby, libruby].select { File.exist?(_1) })
+end
+
+SdkBenchmark::ENVIRONMENTS.each do |environment|
+  sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment)
+  SdkBenchmark::BENCHMARKS.each do |name, method|
+    sdk_benchmark.public_send(method) # Warm up caches
+    if codspeed
+      GC.start
+      CodSpeed.measure("benchmarks/sdk_benchmark.rb::#{name}[#{environment}]") { sdk_benchmark.public_send(method) }
+    else
+      seconds = Benchmark.realtime { LOCAL_REVOLUTIONS.times { sdk_benchmark.public_send(method) } }
+      puts format('%<name>-50s %<ms>10.3f ms', name: "#{name}[#{environment}]", ms: seconds * 1000 / LOCAL_REVOLUTIONS)
+    end
+  end
+  sdk_benchmark.close
+end

codspeed.yml

@@ -1,33 +1,5 @@
 $schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json
 
 benchmarks:
-  - name: getEnvironmentFlags[small]
-    exec: ruby benchmarks/run.rb getEnvironmentFlags small
-  - name: getEnvironmentFlags[medium]
-    exec: ruby benchmarks/run.rb getEnvironmentFlags medium
-  - name: getEnvironmentFlags[large]
-    exec: ruby benchmarks/run.rb getEnvironmentFlags large
-  - name: getIdentityFlagsWithIdentityOverrides[small]
-    exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides small
-  - name: getIdentityFlagsWithIdentityOverrides[medium]
-    exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides medium
-  - name: getIdentityFlagsWithIdentityOverrides[large]
-    exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides large
-  - name: getIdentityFlagsNoTraits[small]
-    exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits small
-  - name: getIdentityFlagsNoTraits[medium]
-    exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits medium
-  - name: getIdentityFlagsNoTraits[large]
-    exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits large
-  - name: getIdentityFlagsWithTraits[small]
-    exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits small
-  - name: getIdentityFlagsWithTraits[medium]
-    exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits medium
-  - name: getIdentityFlagsWithTraits[large]
-    exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits large
-  - name: getIdentitySegmentsWithTraits[small]
-    exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits small
-  - name: getIdentitySegmentsWithTraits[medium]
-    exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits medium
-  - name: getIdentitySegmentsWithTraits[large]
-    exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits large
+  - name: SDK benchmarks
+    entrypoint: ruby benchmarks/run.rb