feat(CI): Set up CodSpeed

.github/workflows/codspeed.yml

@@ -0,0 +1,43 @@
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read  # Check out the repository and its submodules
+  id-token: write  # Authenticate result uploads to CodSpeed via OIDC
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: codspeed-macro
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          submodules: recursive
+          persist-credentials: false
+
+      - name: Set up PHP
+        uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962  # 2.40.0
+        with:
+          php-version: "8.5"
+          extensions: "bcmath, gmp"
+
+      - name: Install CodSpeed CLI
+        env:
+          CODSPEED_VERSION: 5.6.0
+          CODSPEED_INSTALLER_SHA256: bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40
+        run: |
+          installer=$(mktemp)
+          curl -fsSL -o "$installer" "https://codspeed.io/v$CODSPEED_VERSION/install.sh"
+          echo "$CODSPEED_INSTALLER_SHA256  $installer" | sha256sum --check --strict
+          bash "$installer" --quiet
+
+      - name: Run benchmarks
+        run: make benchmark

.gitmodules

@@ -2,3 +2,7 @@
 	path = tests/Engine/EngineTests/EngineTestData
 	url = git@github.com:Flagsmith/engine-test-data.git
 	branch = v3.7.0
+[submodule "tests/Engine/EngineTests/SdkBenchmarks"]
+	path = tests/Engine/EngineTests/SdkBenchmarks
+	url = https://github.com/Flagsmith/sdk-benchmarks.git
+	branch = 0.0.2

Makefile

@@ -0,0 +1,12 @@
+SDK_BENCHMARKS := tests/Engine/EngineTests/SdkBenchmarks
+
+$(SDK_BENCHMARKS)/environments:
+	git submodule update --init -- $(SDK_BENCHMARKS)
+
+vendor/autoload.php: composer.json
+	composer install --prefer-dist --no-interaction
+	touch $@
+
+.PHONY: benchmark
+benchmark: vendor/autoload.php $(SDK_BENCHMARKS)/environments
+	SDK_BENCHMARKS_ENVIRONMENTS_DIR=$(abspath $(SDK_BENCHMARKS)/environments) codspeed run --mode walltime

benchmarks/SdkBenchmark.php

@@ -0,0 +1,103 @@
+<?php
+
+namespace FlagsmithBenchmark;
+
+use Flagsmith\Flagsmith;
+use Flagsmith\Models\Segment;
+use GuzzleHttp\Client;
+use GuzzleHttp\Promise\Create;
+use GuzzleHttp\Promise\PromiseInterface;
+use GuzzleHttp\Psr7\Response;
+use Psr\Http\Message\RequestInterface;
+
+/**
+ * Benchmarks a locally evaluating SDK client against environment documents tiered by size.
+ */
+class SdkBenchmark
+{
+    public const ENVIRONMENTS = ['small', 'medium', 'large'];
+    public const BENCHMARKS = [
+        'getEnvironmentFlags',
+        'getIdentityFlagsWithIdentityOverrides',
+        'getIdentityFlagsNoTraits',
+        'getIdentityFlagsWithTraits',
+        'getIdentitySegmentsWithTraits',
+    ];
+
+    private const IDENTITY_OVERRIDES_IDENTIFIER = 'identity-override-1';
+    private const IDENTIFIER = 'user-1';
+    private const TRAITS = [
+        'plan' => 'plan-7',
+        'age' => 30,
+        'country' => 'GB',
+        'version' => '2.5.1',
+        'email' => 'jane@example.com',
+        'user_id' => 12,
+        'beta' => true,
+        'score' => 95,
+        'tier' => 'pro',
+        'device' => 'iPhone 15',
+    ];
+
+    private Flagsmith $client;
+
+    /**
+     * Build a locally evaluating client serving the environment document from memory.
+     */
+    public function __construct(string $environmentsDir, string $environment)
+    {
+        $environmentDocument = file_get_contents("{$environmentsDir}/{$environment}.json");
+
+        $this->client = (new Flagsmith(
+            apiKey: 'ser.benchmark',
+            host: 'http://flagsmith.benchmark/api/v1/',
+            environmentTtl: 86400,  // Refreshed by benchmarks only
+        ))->withClient(new Client([
+            'handler' => static fn (RequestInterface $request): PromiseInterface => Create::promiseFor(
+                new Response(200, ['Content-Type' => 'application/json'], $environmentDocument),
+            ),
+        ]));
+        $this->client->updateEnvironment();
+    }
+
+    /**
+     * Get a feature value from the environment flags.
+     */
+    public function getEnvironmentFlags(): mixed
+    {
+        return $this->client->getEnvironmentFlags()->getFeatureValue('feature_1');
+    }
+
+    /**
+     * Check a feature for an identity with identity overrides.
+     */
+    public function getIdentityFlagsWithIdentityOverrides(): bool
+    {
+        return $this->client->getIdentityFlags(self::IDENTITY_OVERRIDES_IDENTIFIER)->isFeatureEnabled('feature_8');
+    }
+
+    /**
+     * Get a feature value for an identity without traits.
+     */
+    public function getIdentityFlagsNoTraits(): mixed
+    {
+        return $this->client->getIdentityFlags(self::IDENTIFIER)->getFeatureValue('feature_1');
+    }
+
+    /**
+     * Get a feature value for an identity with traits.
+     */
+    public function getIdentityFlagsWithTraits(): mixed
+    {
+        return $this->client->getIdentityFlags(self::IDENTIFIER, (object) self::TRAITS)->getFeatureValue('feature_1');
+    }
+
+    /**
+     * List the segments of an identity with traits.
+     * @return array<Segment>
+     */
+    public function getIdentitySegmentsWithTraits(): array
+    {
+        return $this->client->getIdentitySegments(self::IDENTIFIER, (object) self::TRAITS);
+    }
+}

benchmarks/run.php

@@ -0,0 +1,29 @@
+<?php
+
+use FlagsmithBenchmark\SdkBenchmark;
+
+require __DIR__ . '/../vendor/autoload.php';
+
+const REVOLUTIONS = 1000;
+
+[, $benchmark, $environment] = $argv + [null, null, null];
+$environmentsDir = getenv('SDK_BENCHMARKS_ENVIRONMENTS_DIR');
+
+if (
+    $environmentsDir === false
+    || !in_array($benchmark, SdkBenchmark::BENCHMARKS, true)
+    || !in_array($environment, SdkBenchmark::ENVIRONMENTS, true)
+) {
+    fwrite(STDERR, sprintf(
+        "Usage: SDK_BENCHMARKS_ENVIRONMENTS_DIR=<dir> php %s <%s> <%s>\n",
+        $argv[0],
+        implode('|', SdkBenchmark::BENCHMARKS),
+        implode('|', SdkBenchmark::ENVIRONMENTS),
+    ));
+    exit(2);
+}
+
+$sdkBenchmark = new SdkBenchmark($environmentsDir, $environment);
+for ($revolution = 0; $revolution < REVOLUTIONS; $revolution++) {
+    $sdkBenchmark->$benchmark();
+}

codspeed.yml

@@ -0,0 +1,33 @@
+$schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json
+
+benchmarks:
+  - name: getEnvironmentFlags[small]
+    exec: php benchmarks/run.php getEnvironmentFlags small
+  - name: getEnvironmentFlags[medium]
+    exec: php benchmarks/run.php getEnvironmentFlags medium
+  - name: getEnvironmentFlags[large]
+    exec: php benchmarks/run.php getEnvironmentFlags large
+  - name: getIdentityFlagsWithIdentityOverrides[small]
+    exec: php benchmarks/run.php getIdentityFlagsWithIdentityOverrides small
+  - name: getIdentityFlagsWithIdentityOverrides[medium]
+    exec: php benchmarks/run.php getIdentityFlagsWithIdentityOverrides medium
+  - name: getIdentityFlagsWithIdentityOverrides[large]
+    exec: php benchmarks/run.php getIdentityFlagsWithIdentityOverrides large
+  - name: getIdentityFlagsNoTraits[small]
+    exec: php benchmarks/run.php getIdentityFlagsNoTraits small
+  - name: getIdentityFlagsNoTraits[medium]
+    exec: php benchmarks/run.php getIdentityFlagsNoTraits medium
+  - name: getIdentityFlagsNoTraits[large]
+    exec: php benchmarks/run.php getIdentityFlagsNoTraits large
+  - name: getIdentityFlagsWithTraits[small]
+    exec: php benchmarks/run.php getIdentityFlagsWithTraits small
+  - name: getIdentityFlagsWithTraits[medium]
+    exec: php benchmarks/run.php getIdentityFlagsWithTraits medium
+  - name: getIdentityFlagsWithTraits[large]
+    exec: php benchmarks/run.php getIdentityFlagsWithTraits large
+  - name: getIdentitySegmentsWithTraits[small]
+    exec: php benchmarks/run.php getIdentitySegmentsWithTraits small
+  - name: getIdentitySegmentsWithTraits[medium]
+    exec: php benchmarks/run.php getIdentitySegmentsWithTraits medium
+  - name: getIdentitySegmentsWithTraits[large]
+    exec: php benchmarks/run.php getIdentitySegmentsWithTraits large

composer.json

@@ -35,6 +35,11 @@
             "FlagsmithTest\\": "tests"
         }
     },
+    "autoload-dev": {
+        "psr-4": {
+            "FlagsmithBenchmark\\": "benchmarks"
+        }
+    },
     "config": {
         "allow-plugins": {
             "php-http/discovery": true