From c2ab4123ae677670d410e6da3cc7557e72c7206e Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 29 Oct 2026 17:16:23 -0300 Subject: [PATCH] feat(CI): Set up CodSpeed Co-Authored-By: Claude Opus 5.5 (1M context) --- .formatter.exs | 2 +- .github/workflows/codspeed.yml | 45 ++++++++ .gitmodules | 7 ++ Makefile | 34 ++++++ bench/codspeed.exs | 205 +++++++++++++++++++++++++++++++++ bench/instrument-hooks | 1 + bench/instrument_hooks_nif.c | 110 ++++++++++++++++++ bench/sdk_benchmark.exs | 171 +++++++++++++++++++++++++++ test/support/sdk-benchmarks | 1 + 9 files changed, 575 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/codspeed.yml create mode 100644 Makefile create mode 100644 bench/codspeed.exs create mode 160000 bench/instrument-hooks create mode 100644 bench/instrument_hooks_nif.c create mode 100644 bench/sdk_benchmark.exs create mode 160000 test/support/sdk-benchmarks diff --git a/.formatter.exs b/.formatter.exs index 76c5fb3..dd48143 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,4 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}", "sdk-spec/*.exs"] + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}", "{bench,sdk-spec}/*.exs"] ] diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000..039aa15 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,45 @@ +name: CodSpeed + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read # Check out the repository and its submodules + id-token: write # Authenticate result uploads to CodSpeed via OIDC + +jobs: + benchmarks: + name: Run benchmarks + runs-on: codspeed-macro + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + submodules: recursive + persist-credentials: false + + - name: Set up Elixir + uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 + env: + ImageOS: ubuntu22-arm64 # codspeed-macro is an arm64 Ubuntu 22.04 runner + with: + version-file: .tool-versions + version-type: strict + + - name: Install CodSpeed CLI + env: + CODSPEED_VERSION: 5.6.0 + CODSPEED_INSTALLER_SHA256: bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40 + run: | + installer=$(mktemp) + curl -fsSL -o "$installer" "https://codspeed.io/v$CODSPEED_VERSION/install.sh" + echo "$CODSPEED_INSTALLER_SHA256 $installer" | sha256sum --check --strict + bash "$installer" --quiet + + - name: Run benchmarks + run: make benchmark diff --git a/.gitmodules b/.gitmodules index 2f5331f..ded3c47 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,10 @@ path = test/support/engine-test-data url = git@github.com:Flagsmith/engine-test-data.git branch = v1.0.0 +[submodule "test/support/sdk-benchmarks"] + path = test/support/sdk-benchmarks + url = https://github.com/Flagsmith/sdk-benchmarks.git + branch = 0.0.2 +[submodule "bench/instrument-hooks"] + path = bench/instrument-hooks + url = https://github.com/CodSpeedHQ/instrument-hooks.git diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b1ba9ba --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +SDK_BENCHMARKS := test/support/sdk-benchmarks +INSTRUMENT_HOOKS := bench/instrument-hooks +INSTRUMENT_HOOKS_NIF := _build/codspeed/instrument_hooks_nif.so + +ERTS_INCLUDE_DIR = $(shell erl -noshell -eval 'io:format("~ts/erts-~ts/include", [code:root_dir(), erlang:system_info(version)]), halt().') +NIF_CFLAGS := -std=c11 -O3 -g -fPIC -Wno-format -Wno-format-security -Wno-unused-but-set-variable -Wno-unused-const-variable -Wno-type-limits -Wno-uninitialized -Wno-incompatible-pointer-types +ifeq ($(shell uname -s),Darwin) +NIF_LDFLAGS := -dynamiclib -undefined dynamic_lookup +else +NIF_LDFLAGS := -shared +BENCHMARK_ERL_FLAGS := +JPperf true +endif + +$(SDK_BENCHMARKS)/environments: + git submodule update --init -- $(SDK_BENCHMARKS) + +$(INSTRUMENT_HOOKS)/dist/core.c: + git submodule update --init -- $(INSTRUMENT_HOOKS) + +$(INSTRUMENT_HOOKS_NIF): bench/instrument_hooks_nif.c | $(INSTRUMENT_HOOKS)/dist/core.c + mkdir -p $(@D) + $(CC) $(NIF_CFLAGS) -I$(ERTS_INCLUDE_DIR) -I$(INSTRUMENT_HOOKS)/includes $(NIF_LDFLAGS) -o $@ $< $(INSTRUMENT_HOOKS)/dist/core.c + +.PHONY: benchmark-build +benchmark-build: $(INSTRUMENT_HOOKS_NIF) | $(SDK_BENCHMARKS)/environments + mix deps.get + mix compile + +.PHONY: benchmark +benchmark: benchmark-build + SDK_BENCHMARKS_ENVIRONMENTS_DIR=$(abspath $(SDK_BENCHMARKS)/environments) \ + INSTRUMENT_HOOKS_NIF=$(abspath $(basename $(INSTRUMENT_HOOKS_NIF))) \ + ERL_FLAGS="$(BENCHMARK_ERL_FLAGS)" \ + codspeed run --mode walltime -- mix run --no-compile --no-deps-check bench/sdk_benchmark.exs diff --git a/bench/codspeed.exs b/bench/codspeed.exs new file mode 100644 index 0000000..4313292 --- /dev/null +++ b/bench/codspeed.exs @@ -0,0 +1,205 @@ +defmodule Flagsmith.Bench.InstrumentHooks do + @moduledoc """ + Bindings to CodSpeed's instrument-hooks library, loaded from the NIF at `INSTRUMENT_HOOKS_NIF`. + """ + @on_load :load_nif + + def load_nif do + "INSTRUMENT_HOOKS_NIF" + |> System.fetch_env!() + |> String.to_charlist() + |> :erlang.load_nif(0) + end + + def is_instrumented, do: :erlang.nif_error(:not_loaded) + def set_integration(_name, _version), do: :erlang.nif_error(:not_loaded) + def start_benchmark, do: :erlang.nif_error(:not_loaded) + def stop_benchmark, do: :erlang.nif_error(:not_loaded) + def set_executed_benchmark(_uri), do: :erlang.nif_error(:not_loaded) + def add_benchmark_timestamps(_start, _end), do: :erlang.nif_error(:not_loaded) + def current_timestamp, do: :erlang.nif_error(:not_loaded) +end + +defmodule Flagsmith.Bench.CodSpeed do + @moduledoc """ + Measures benchmarks for CodSpeed's walltime instrument, following pytest-codspeed's + warmup and round sizing, and reports them as CodSpeed walltime results. + """ + alias Flagsmith.Bench.InstrumentHooks + + @integration_name "flagsmith-elixir-client-benchmarks" + @integration_version "0.1.0" + @warmup_time_ns 1_000_000_000 + @max_time_ns 3_000_000_000 + @min_round_time_ns 1_000_000 + + @doc """ + Register this harness with the CodSpeed runner. + """ + def init do + IO.puts("CodSpeed instrumentation: #{InstrumentHooks.is_instrumented()}") + :ok = InstrumentHooks.set_integration(@integration_name, @integration_version) + end + + @doc """ + Measure a function, returning its walltime benchmark result. + """ + def bench(name, uri, function) do + __codspeed_root_frame__(function) + + {warmup_times_ns, warmup_iters} = warmup(function) + warmup_mean_ns = Enum.sum(warmup_times_ns) / warmup_iters + + iter_per_round = + if warmup_mean_ns <= @min_round_time_ns, + do: ceil(@min_round_time_ns / warmup_mean_ns), + else: 1 + + rounds = max(1, trunc(@max_time_ns / (warmup_mean_ns * iter_per_round))) + + :ok = InstrumentHooks.start_benchmark() + run_start = :erlang.monotonic_time(:nanosecond) + times_per_round_ns = run_rounds(function, iter_per_round, rounds, run_start, []) + :ok = InstrumentHooks.stop_benchmark() + :ok = InstrumentHooks.set_executed_benchmark(uri) + + %{ + name: name, + uri: uri, + config: %{ + warmup_time_ns: @warmup_time_ns, + min_round_time_ns: @min_round_time_ns, + max_time_ns: @max_time_ns, + max_rounds: nil + }, + stats: stats(times_per_round_ns, iter_per_round, warmup_iters) + } + end + + @doc """ + Write walltime results where the CodSpeed runner collects them. + """ + def write_results(benchmarks) do + case System.get_env("CODSPEED_PROFILE_FOLDER") do + nil -> + :ok + + profile_folder -> + pid = System.pid() + results_dir = Path.join(profile_folder, "results") + File.mkdir_p!(results_dir) + + results = %{ + creator: %{ + name: @integration_name, + version: @integration_version, + pid: String.to_integer(pid) + }, + instrument: %{type: "walltime"}, + benchmarks: benchmarks + } + + File.write!(Path.join(results_dir, "#{pid}.json"), Jason.encode_to_iodata!(results)) + end + end + + def __codspeed_root_frame__(function), do: function.() + + defp warmup(function) do + warmup_start = :erlang.monotonic_time(:nanosecond) + warmup(function, warmup_start, [], 0) + end + + defp warmup(function, warmup_start, times_ns, iters) do + start = :erlang.monotonic_time(:nanosecond) + __codspeed_root_frame__(function) + finish = :erlang.monotonic_time(:nanosecond) + times_ns = [finish - start | times_ns] + + if finish - warmup_start > @warmup_time_ns, + do: {times_ns, iters + 1}, + else: warmup(function, warmup_start, times_ns, iters + 1) + end + + defp run_rounds(_function, _iter_per_round, 0, _run_start, times_ns), + do: Enum.reverse(times_ns) + + defp run_rounds(function, iter_per_round, rounds_left, run_start, times_ns) do + hooks_start = InstrumentHooks.current_timestamp() + start = :erlang.monotonic_time(:nanosecond) + run_iterations(function, iter_per_round) + finish = :erlang.monotonic_time(:nanosecond) + + :ok = + InstrumentHooks.add_benchmark_timestamps(hooks_start, InstrumentHooks.current_timestamp()) + + times_ns = [finish - start | times_ns] + + if finish - run_start > @max_time_ns, + do: Enum.reverse(times_ns), + else: run_rounds(function, iter_per_round, rounds_left - 1, run_start, times_ns) + end + + defp run_iterations(_function, 0), do: :ok + + defp run_iterations(function, iterations) do + __codspeed_root_frame__(function) + run_iterations(function, iterations - 1) + end + + # Mirrors `WalltimeBenchmark::from_runtime_data` in CodSpeed's runner-shared crate. + defp stats(times_per_round_ns, iter_per_round, warmup_iters) do + sorted = times_per_round_ns |> Enum.map(&(div(&1, iter_per_round) * 1.0)) |> Enum.sort() + rounds = length(sorted) + mean_ns = Enum.sum(sorted) / rounds + q1_ns = quantile(sorted, 0.25) + q3_ns = quantile(sorted, 0.75) + stdev_ns = sample_stdev(sorted, mean_ns) + iqr = q3_ns - q1_ns + + %{ + min_ns: List.first(sorted), + max_ns: List.last(sorted), + mean_ns: mean_ns, + stdev_ns: stdev_ns, + q1_ns: q1_ns, + median_ns: median(sorted), + q3_ns: q3_ns, + rounds: rounds, + total_time: Enum.sum(times_per_round_ns) / 1_000_000_000, + iqr_outlier_rounds: Enum.count(sorted, &(&1 < q1_ns - 1.5 * iqr or &1 > q3_ns + 1.5 * iqr)), + stdev_outlier_rounds: Enum.count(sorted, &(abs(&1 - mean_ns) > 2 * stdev_ns)), + iter_per_round: iter_per_round, + warmup_iters: warmup_iters + } + end + + defp median(sorted) do + middle = div(length(sorted), 2) + + if rem(length(sorted), 2) == 0, + do: (Enum.at(sorted, middle - 1) + Enum.at(sorted, middle)) / 2, + else: Enum.at(sorted, middle) + end + + defp quantile([only], _p), do: only + defp quantile([first, second], p), do: first * (1 - p) + second * p + + defp quantile(sorted, p) do + n = length(sorted) + position = p * (n + 1) - 1 + index = floor(position) + fraction = position - index + + if index + 1 < n, + do: Enum.at(sorted, index) * (1 - fraction) + Enum.at(sorted, index + 1) * fraction, + else: Enum.at(sorted, min(index, n - 1)) + end + + defp sample_stdev([_], _mean), do: 0.0 + + defp sample_stdev(data, mean) do + variance = Enum.sum(Enum.map(data, &((&1 - mean) ** 2))) / (length(data) - 1) + :math.sqrt(variance) + end +end diff --git a/bench/instrument-hooks b/bench/instrument-hooks new file mode 160000 index 0000000..8fd5b87 --- /dev/null +++ b/bench/instrument-hooks @@ -0,0 +1 @@ +Subproject commit 8fd5b878499f819561b33219cdafaa6f8c28b4e4 diff --git a/bench/instrument_hooks_nif.c b/bench/instrument_hooks_nif.c new file mode 100644 index 0000000..4450011 --- /dev/null +++ b/bench/instrument_hooks_nif.c @@ -0,0 +1,110 @@ +#include +#include +#include + +#include "core.h" + +static InstrumentHooks *hooks = NULL; + +static ERL_NIF_TERM make_result(ErlNifEnv *env, uint8_t status) { + if (status == 0) { + return enif_make_atom(env, "ok"); + } + return enif_make_tuple2(env, enif_make_atom(env, "error"), + enif_make_uint(env, status)); +} + +static int read_string(ErlNifEnv *env, ERL_NIF_TERM term, char *buffer, + size_t size) { + ErlNifBinary binary; + if (!enif_inspect_binary(env, term, &binary) || binary.size >= size) { + return 0; + } + memcpy(buffer, binary.data, binary.size); + buffer[binary.size] = '\0'; + return 1; +} + +static ERL_NIF_TERM is_instrumented(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + return enif_make_atom( + env, instrument_hooks_is_instrumented(hooks) ? "true" : "false"); +} + +static ERL_NIF_TERM set_integration(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + char name[256]; + char version[64]; + if (!read_string(env, argv[0], name, sizeof(name)) || + !read_string(env, argv[1], version, sizeof(version))) { + return enif_make_badarg(env); + } + return make_result(env, + instrument_hooks_set_integration(hooks, name, version)); +} + +static ERL_NIF_TERM start_benchmark(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + return make_result(env, instrument_hooks_start_benchmark(hooks)); +} + +static ERL_NIF_TERM stop_benchmark(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + return make_result(env, instrument_hooks_stop_benchmark(hooks)); +} + +static ERL_NIF_TERM set_executed_benchmark(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + char uri[1024]; + if (!read_string(env, argv[0], uri, sizeof(uri))) { + return enif_make_badarg(env); + } + return make_result(env, instrument_hooks_set_executed_benchmark( + hooks, (int32_t)getpid(), uri)); +} + +static ERL_NIF_TERM add_benchmark_timestamps(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + ErlNifUInt64 start; + ErlNifUInt64 end; + if (!enif_get_uint64(env, argv[0], &start) || + !enif_get_uint64(env, argv[1], &end)) { + return enif_make_badarg(env); + } + int32_t pid = (int32_t)getpid(); + uint8_t status = instrument_hooks_add_marker( + hooks, pid, MARKER_TYPE_BENCHMARK_START, start); + if (status == 0) { + status = instrument_hooks_add_marker(hooks, pid, + MARKER_TYPE_BENCHMARK_END, end); + } + return make_result(env, status); +} + +static ERL_NIF_TERM current_timestamp(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + return enif_make_uint64(env, instrument_hooks_current_timestamp()); +} + +static int load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info) { + hooks = instrument_hooks_init(); + return hooks == NULL ? 1 : 0; +} + +static void unload(ErlNifEnv *env, void *priv_data) { + instrument_hooks_deinit(hooks); + hooks = NULL; +} + +static ErlNifFunc nif_funcs[] = { + {"is_instrumented", 0, is_instrumented, 0}, + {"set_integration", 2, set_integration, 0}, + {"start_benchmark", 0, start_benchmark, 0}, + {"stop_benchmark", 0, stop_benchmark, 0}, + {"set_executed_benchmark", 1, set_executed_benchmark, 0}, + {"add_benchmark_timestamps", 2, add_benchmark_timestamps, 0}, + {"current_timestamp", 0, current_timestamp, 0}, +}; + +ERL_NIF_INIT(Elixir.Flagsmith.Bench.InstrumentHooks, nif_funcs, load, NULL, + NULL, unload) diff --git a/bench/sdk_benchmark.exs b/bench/sdk_benchmark.exs new file mode 100644 index 0000000..ae3bdb4 --- /dev/null +++ b/bench/sdk_benchmark.exs @@ -0,0 +1,171 @@ +Code.require_file("codspeed.exs", __DIR__) + +defmodule Flagsmith.SdkBenchmark.EnvironmentDocumentAdapter do + @moduledoc """ + Serves an environment document from memory to any request. + """ + @behaviour Tesla.Adapter + + @impl Tesla.Adapter + def call(env, environment_document: environment_document) do + {:ok, + %{ + env + | status: 200, + headers: [{"content-type", "application/json"}], + body: environment_document + }} + end +end + +defmodule Flagsmith.SdkBenchmark do + @moduledoc """ + Benchmarks a locally evaluating SDK client against environment documents tiered by size. + """ + + @file_uri Path.relative_to(__ENV__.file, Path.expand("..", __DIR__)) + @identity_overrides_identifier "identity-override-1" + @identifier "user-1" + @traits [ + %{trait_key: "plan", trait_value: "plan-7"}, + %{trait_key: "age", trait_value: 30}, + %{trait_key: "country", trait_value: "GB"}, + %{trait_key: "version", trait_value: "2.5.1"}, + %{trait_key: "email", trait_value: "jane@example.com"}, + %{trait_key: "user_id", trait_value: 12}, + %{trait_key: "beta", trait_value: true}, + %{trait_key: "score", trait_value: 95}, + %{trait_key: "tier", trait_value: "pro"}, + %{trait_key: "device", trait_value: "iPhone 15"} + ] + + @benchmarks [ + :get_environment_flags, + :get_identity_flags_with_identity_overrides, + :get_identity_flags_no_traits, + :get_identity_flags_with_traits + ] + @environments ["small", "medium", "large"] + + @doc """ + Run every benchmark against every environment, returning whether all of them succeeded. + """ + def run do + Flagsmith.Bench.CodSpeed.init() + + outcomes = + for environment <- @environments, + outcome <- run_environment(environment), + do: outcome + + outcomes + |> Enum.flat_map(fn + {:ok, benchmark} -> [benchmark] + {:error, _name} -> [] + end) + |> Flagsmith.Bench.CodSpeed.write_results() + + Enum.all?(outcomes, &match?({:ok, _}, &1)) + end + + defp run_environment(environment) do + client = build_client(environment) + + try do + Enum.map(@benchmarks, &run_benchmark(&1, environment, client)) + after + close_client() + end + end + + defp run_benchmark(benchmark, environment, client) do + name = "#{benchmark}[#{environment}]" + + try do + result = + Flagsmith.Bench.CodSpeed.bench(name, "#{@file_uri}::#{name}", fn -> + apply(__MODULE__, benchmark, [client]) + end) + + %{stats: %{mean_ns: mean_ns, rounds: rounds, iter_per_round: iter_per_round}} = result + + IO.puts( + "#{name}: #{Float.round(mean_ns / 1000, 2)}µs (#{rounds} rounds × #{iter_per_round})" + ) + + {:ok, result} + catch + kind, reason -> + IO.puts(:stderr, "#{name}: failed\n" <> Exception.format(kind, reason, __STACKTRACE__)) + {:error, name} + end + end + + @doc """ + Build a locally evaluating client serving the environment document from memory. + """ + def build_client(environment) do + environments_dir = System.fetch_env!("SDK_BENCHMARKS_ENVIRONMENTS_DIR") + environment_document = File.read!(Path.join(environments_dir, "#{environment}.json")) + + Application.put_env( + :tesla, + :adapter, + {Flagsmith.SdkBenchmark.EnvironmentDocumentAdapter, + environment_document: environment_document} + ) + + {:ok, _} = Flagsmith.Supervisor.start_link() + + client = + Flagsmith.Client.new( + environment_key: "ser.benchmark", + api_url: "http://flagsmith.benchmark/api/v1/", + enable_local_evaluation: true, + environment_refresh_interval_milliseconds: 86_400_000 + ) + + {:ok, _} = Flagsmith.Client.get_environment(client) + client + end + + def close_client, do: Supervisor.stop(Flagsmith.Supervisor) + + @doc """ + Get a feature value from the environment flags. + """ + def get_environment_flags(client) do + {:ok, flags} = Flagsmith.Client.get_environment_flags(client) + Flagsmith.Client.get_feature_value(flags, "feature_1") + end + + @doc """ + Check a feature for an identity with identity overrides. + """ + def get_identity_flags_with_identity_overrides(client) do + {:ok, flags} = + Flagsmith.Client.get_identity_flags(client, @identity_overrides_identifier, []) + + Flagsmith.Client.is_feature_enabled(flags, "feature_8") + end + + @doc """ + Get a feature value for an identity without traits. + """ + def get_identity_flags_no_traits(client) do + {:ok, flags} = Flagsmith.Client.get_identity_flags(client, @identifier, []) + Flagsmith.Client.get_feature_value(flags, "feature_1") + end + + @doc """ + Get a feature value for an identity with traits. + """ + def get_identity_flags_with_traits(client) do + {:ok, flags} = Flagsmith.Client.get_identity_flags(client, @identifier, @traits) + Flagsmith.Client.get_feature_value(flags, "feature_1") + end +end + +Logger.configure(level: :warning) + +unless Flagsmith.SdkBenchmark.run(), do: System.halt(1) diff --git a/test/support/sdk-benchmarks b/test/support/sdk-benchmarks new file mode 160000 index 0000000..7da2be4 --- /dev/null +++ b/test/support/sdk-benchmarks @@ -0,0 +1 @@ +Subproject commit 7da2be42ec7c58a597892e41141d531bd1dd100c