diff --git a/.gitmodules b/.gitmodules index 0c86e07..72eee3a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = spec/sdk-benchmarks url = https://github.com/Flagsmith/sdk-benchmarks.git branch = 0.0.2 +[submodule "benchmarks/codspeed/instrument-hooks"] + path = benchmarks/codspeed/instrument-hooks + url = https://github.com/CodSpeedHQ/instrument-hooks.git diff --git a/Makefile b/Makefile index 8689072..776d4d4 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,19 @@ SDK_BENCHMARKS := spec/sdk-benchmarks CODSPEED_VERSION := 5.6.0 CODSPEED_INSTALLER_SHA256 := bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40 +INSTRUMENT_HOOKS := benchmarks/codspeed/instrument-hooks +CODSPEED_EXT := tmp/codspeed/codspeed.$(shell ruby -e 'print RbConfig::CONFIG["DLEXT"]') + $(SDK_BENCHMARKS)/environments: git submodule update --init -- $(SDK_BENCHMARKS) +$(INSTRUMENT_HOOKS)/dist/core.c: + git submodule update --init -- $(INSTRUMENT_HOOKS) + +$(CODSPEED_EXT): benchmarks/codspeed/codspeed.c benchmarks/codspeed/extconf.rb $(INSTRUMENT_HOOKS)/dist/core.c + mkdir -p $(@D) + cd $(@D) && ruby $(CURDIR)/benchmarks/codspeed/extconf.rb && $(MAKE) + .PHONY: install-codspeed install-codspeed: $(SDK_BENCHMARKS)/environments bundle install @@ -14,5 +24,5 @@ install-codspeed: $(SDK_BENCHMARKS)/environments bash "$$installer" --quiet .PHONY: benchmark -benchmark: install-codspeed +benchmark: install-codspeed $(CODSPEED_EXT) SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode simulation --config codspeed.yml diff --git a/benchmarks/codspeed/codspeed.c b/benchmarks/codspeed/codspeed.c new file mode 100644 index 0000000..b9000b0 --- /dev/null +++ b/benchmarks/codspeed/codspeed.c @@ -0,0 +1,60 @@ +// Ruby bindings for CodSpeed's instrument-hooks, measuring a block in-process. +#include +#include + +#include "core.h" + +static InstrumentHooks *hooks; + +// Root frame of the benchmark flame graph: everything measured runs inside it. +__attribute__((noinline)) static VALUE __codspeed_root_frame__measure(VALUE _) { + return rb_yield(Qnil); +} + +static VALUE stop_measuring(VALUE uri) { + callgrind_stop_instrumentation(); + if (instrument_hooks_stop_benchmark(hooks) != 0) + rb_warn("CodSpeed: failed to stop the benchmark"); + if (instrument_hooks_set_executed_benchmark(hooks, getpid(), StringValueCStr(uri)) != 0) + rb_warn("CodSpeed: failed to report the executed benchmark"); + return Qnil; +} + +// CodSpeed.measure(uri) { ... }: measure the block once as the benchmark at uri. +static VALUE codspeed_measure(VALUE self, VALUE uri) { + rb_need_block(); + StringValueCStr(uri); + if (instrument_hooks_start_benchmark(hooks) != 0) + rb_raise(rb_eRuntimeError, "CodSpeed: failed to start the benchmark"); + // Toggle callgrind here, rather than in start_benchmark, to measure as little harness as possible. + callgrind_start_instrumentation(); + return rb_ensure(__codspeed_root_frame__measure, Qnil, stop_measuring, uri); +} + +// CodSpeed.instrumented?: whether the process runs under the CodSpeed runner. +static VALUE codspeed_instrumented_p(VALUE self) { + return instrument_hooks_is_instrumented(hooks) ? Qtrue : Qfalse; +} + +// CodSpeed.setup(integration, version, skipped_objects): register the harness with the runner. +static VALUE codspeed_setup(VALUE self, VALUE integration, VALUE version, VALUE skipped_objects) { + if (instrument_hooks_set_integration(hooks, StringValueCStr(integration), StringValueCStr(version)) != 0) + rb_warn("CodSpeed: failed to register the integration"); + // Attribute interpreter costs to the root frame, as pytest-codspeed does for libpython. + for (long i = 0; i < RARRAY_LEN(skipped_objects); i++) { + VALUE path = rb_ary_entry(skipped_objects, i); + instrument_hooks_callgrind_add_obj_skip(StringValueCStr(path)); + } + return Qnil; +} + +void Init_codspeed(void) { + hooks = instrument_hooks_init(); + if (!hooks) rb_raise(rb_eRuntimeError, "CodSpeed: failed to initialise instrument-hooks"); + instrument_hooks_set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, true); + + VALUE codspeed = rb_define_module("CodSpeed"); + rb_define_module_function(codspeed, "measure", codspeed_measure, 1); + rb_define_module_function(codspeed, "instrumented?", codspeed_instrumented_p, 0); + rb_define_module_function(codspeed, "setup", codspeed_setup, 3); +} diff --git a/benchmarks/codspeed/extconf.rb b/benchmarks/codspeed/extconf.rb new file mode 100644 index 0000000..b0a84c5 --- /dev/null +++ b/benchmarks/codspeed/extconf.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Builds the CodSpeed extension from instrument-hooks' single-file distribution. +require 'mkmf' + +instrument_hooks = File.join(__dir__, 'instrument-hooks') +abort "Missing #{instrument_hooks}: run `git submodule update --init`" unless File.exist?(File.join(instrument_hooks, 'dist', 'core.c')) + +# rubocop:disable Style/GlobalVars +$INCFLAGS << " -I#{File.join(instrument_hooks, 'includes')}" +$VPATH << File.join(instrument_hooks, 'dist') +$srcs = %w[codspeed.c core.c] +# Compile as C11 like pytest-codspeed: GCC 15 defaults to C23, which breaks zig.h's noreturn. +$CFLAGS << ' -std=gnu11' +# Silence warnings from the generated core.c, as instrument-hooks recommends. +$CFLAGS << %w[ + -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter -Wno-unused-but-set-variable + -Wno-type-limits -Wno-format -Wno-format-security -Wno-old-style-definition -Wno-unterminated-string-initialization +].join(' ').prepend(' ') +# rubocop:enable Style/GlobalVars + +create_makefile('codspeed') diff --git a/benchmarks/codspeed/instrument-hooks b/benchmarks/codspeed/instrument-hooks new file mode 160000 index 0000000..8fd5b87 --- /dev/null +++ b/benchmarks/codspeed/instrument-hooks @@ -0,0 +1 @@ +Subproject commit 8fd5b878499f819561b33219cdafaa6f8c28b4e4 diff --git a/benchmarks/run.rb b/benchmarks/run.rb index b5b3a4e..dfc969a 100644 --- a/benchmarks/run.rb +++ b/benchmarks/run.rb @@ -1,19 +1,41 @@ # frozen_string_literal: true +# Runs every SDK benchmark in a single process, measuring only the benchmarked calls. +# +# Under the CodSpeed runner, each call is measured with instrument-hooks, so Ruby start-up, +# loading the SDK and building the client stay out of the results. Elsewhere, it prints timings. require 'bundler/setup' +require 'benchmark' +require 'rbconfig' require_relative 'sdk_benchmark' -REVOLUTIONS = 10 +LOCAL_REVOLUTIONS = 100 -benchmark, environment = ARGV sdk_benchmarks_dir = ENV.fetch('SDK_BENCHMARKS', '') - -unless !sdk_benchmarks_dir.empty? && SdkBenchmark::BENCHMARKS.key?(benchmark) && SdkBenchmark::ENVIRONMENTS.include?(environment) - warn "Usage: SDK_BENCHMARKS= ruby #{$PROGRAM_NAME} " \ - "<#{SdkBenchmark::BENCHMARKS.keys.join('|')}> <#{SdkBenchmark::ENVIRONMENTS.join('|')}>" +if sdk_benchmarks_dir.empty? + warn "Usage: SDK_BENCHMARKS= ruby #{$PROGRAM_NAME}" exit 2 end -sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment) -REVOLUTIONS.times { sdk_benchmark.public_send(SdkBenchmark::BENCHMARKS.fetch(benchmark)) } -sdk_benchmark.close +codspeed = !ENV.fetch('CODSPEED_ENV', '').empty? +if codspeed + require_relative '../tmp/codspeed/codspeed' + abort 'CodSpeed: not running under instrumentation' unless CodSpeed.instrumented? + libruby = File.join(RbConfig::CONFIG['libdir'], RbConfig::CONFIG['LIBRUBY_SO']) + CodSpeed.setup('flagsmith-ruby-client', Flagsmith::VERSION, [RbConfig.ruby, libruby].select { File.exist?(_1) }) +end + +SdkBenchmark::ENVIRONMENTS.each do |environment| + sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment) + SdkBenchmark::BENCHMARKS.each do |name, method| + sdk_benchmark.public_send(method) # Warm up caches + if codspeed + GC.start + CodSpeed.measure("benchmarks/sdk_benchmark.rb::#{name}[#{environment}]") { sdk_benchmark.public_send(method) } + else + seconds = Benchmark.realtime { LOCAL_REVOLUTIONS.times { sdk_benchmark.public_send(method) } } + puts format('%-50s %10.3f ms', name: "#{name}[#{environment}]", ms: seconds * 1000 / LOCAL_REVOLUTIONS) + end + end + sdk_benchmark.close +end diff --git a/codspeed.yml b/codspeed.yml index 6400170..7fb6f72 100644 --- a/codspeed.yml +++ b/codspeed.yml @@ -1,33 +1,5 @@ $schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json benchmarks: - - name: getEnvironmentFlags[small] - exec: ruby benchmarks/run.rb getEnvironmentFlags small - - name: getEnvironmentFlags[medium] - exec: ruby benchmarks/run.rb getEnvironmentFlags medium - - name: getEnvironmentFlags[large] - exec: ruby benchmarks/run.rb getEnvironmentFlags large - - name: getIdentityFlagsWithIdentityOverrides[small] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides small - - name: getIdentityFlagsWithIdentityOverrides[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides medium - - name: getIdentityFlagsWithIdentityOverrides[large] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides large - - name: getIdentityFlagsNoTraits[small] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits small - - name: getIdentityFlagsNoTraits[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits medium - - name: getIdentityFlagsNoTraits[large] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits large - - name: getIdentityFlagsWithTraits[small] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits small - - name: getIdentityFlagsWithTraits[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits medium - - name: getIdentityFlagsWithTraits[large] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits large - - name: getIdentitySegmentsWithTraits[small] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits small - - name: getIdentitySegmentsWithTraits[medium] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits medium - - name: getIdentitySegmentsWithTraits[large] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits large + - name: SDK benchmarks + entrypoint: ruby benchmarks/run.rb