diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000..ee4ea46 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,42 @@ +name: CodSpeed + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read # Check out the repository and its submodules + id-token: write # Authenticate result uploads to CodSpeed via OIDC + +jobs: + benchmarks: + name: Run benchmarks + runs-on: codspeed-macro + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + submodules: recursive + persist-credentials: false + + - name: Set up JDK + uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: temurin + java-version: 21 + cache: maven + + - name: Set up Maven + uses: stCarolas/setup-maven@12eb41b233df95d49b0c11fc1b5bc8312e5d4ce0 # v5.1 + with: + maven-version: 3.10.0 + + - name: Run benchmarks + uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5.6.0 + with: + mode: walltime + run: make benchmark diff --git a/.gitmodules b/.gitmodules index dcaa9e9..e55e799 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,11 @@ [submodule "src/test/java/com/flagsmith/flagengine/enginetestdata"] path = src/test/java/com/flagsmith/flagengine/enginetestdata url = git@github.com:Flagsmith/engine-test-data.git - branch = v3.7.0 \ No newline at end of file + branch = v3.7.0 +[submodule "third-party/codspeed-jvm"] + path = third-party/codspeed-jvm + url = https://github.com/CodSpeedHQ/codspeed-jvm.git +[submodule "third-party/sdk-benchmarks"] + path = third-party/sdk-benchmarks + url = https://github.com/Flagsmith/sdk-benchmarks.git + branch = 0.0.1 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5b64a01 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +BENCHMARKS_JAR ?= target/benchmarks.jar +CODSPEED_JVM := third-party/codspeed-jvm +SDK_BENCHMARKS := third-party/sdk-benchmarks + +$(CODSPEED_JVM)/gradlew: + git submodule update --init -- $(CODSPEED_JVM) + git -C $(CODSPEED_JVM) submodule update --init -- jmh-fork/jmh-core/native-instrument-hooks/instrument-hooks + +$(SDK_BENCHMARKS)/environments: + git submodule update --init -- $(SDK_BENCHMARKS) + +$(BENCHMARKS_JAR): pom.xml $(shell find src/main src/jmh -type f) $(wildcard $(SDK_BENCHMARKS)/environments/*.json) | $(CODSPEED_JVM)/gradlew $(SDK_BENCHMARKS)/environments + cd third-party/codspeed-jvm && ./gradlew -p jmh-fork publishToMavenLocal + mvn --batch-mode --no-transfer-progress -P benchmarks -DskipTests package + +.PHONY: benchmark +benchmark: $(BENCHMARKS_JAR) + java -jar $(BENCHMARKS_JAR) diff --git a/pom.xml b/pom.xml index ce394fd..cbf492b 100644 --- a/pom.xml +++ b/pom.xml @@ -301,6 +301,10 @@ true true warning + + ${project.build.sourceDirectory} + ${project.basedir}/src/jmh/java + @@ -388,5 +392,97 @@ 4.12.0 + + + benchmarks + + 0.2.1 + + + + io.codspeed.jmh + jmh-core + ${codspeed.jmh.version} + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.2 + + + add-jmh-sources + generate-sources + + add-source + + + + ${project.basedir}/src/jmh/java + + + + + add-jmh-resources + generate-resources + + add-resource + + + + + ${project.basedir}/third-party/sdk-benchmarks + + environments/*.json + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 21 + 21 + + + io.codspeed.jmh + jmh-generator-annprocess + ${codspeed.jmh.version} + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.2 + + + package + + shade + + + ${project.build.directory}/benchmarks.jar + false + + + org.openjdk.jmh.Main + + + + + + + + + + diff --git a/src/jmh/java/com/flagsmith/SdkBenchmark.java b/src/jmh/java/com/flagsmith/SdkBenchmark.java new file mode 100644 index 0000000..68f344b --- /dev/null +++ b/src/jmh/java/com/flagsmith/SdkBenchmark.java @@ -0,0 +1,124 @@ +package com.flagsmith; + +import com.flagsmith.config.FlagsmithConfig; +import com.flagsmith.exceptions.FlagsmithClientError; +import com.flagsmith.flagengine.EvaluationContext; +import com.flagsmith.models.Segment; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import okhttp3.Interceptor; +import okhttp3.MediaType; +import okhttp3.Protocol; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; + +/** + * Benchmarks a locally evaluating SDK client against environment documents tiered by size. + */ +@State(Scope.Benchmark) +public class SdkBenchmark { + private static final MediaType JSON = MediaType.get("application/json"); + private static final String IDENTITY_OVERRIDES_IDENTIFIER = "identity-override-1"; + private static final String IDENTIFIER = "user-1"; + private static final Map TRAITS = Map.of( + "plan", "plan-7", + "age", 30, + "country", "GB", + "version", "2.5.1", + "email", "jane@example.com", + "user_id", 12, + "beta", true, + "score", 95, + "tier", "pro", + "device", "iPhone 15"); + + @Param({"small", "medium", "large"}) + public String environment; + + private FlagsmithClient client; + + /** + * Build a locally evaluating client serving the environment document from memory. + */ + @Setup + public void buildClient() throws IOException { + byte[] environmentDocument; + try (InputStream stream = getClass().getResourceAsStream( + "/environments/" + environment + ".json")) { + environmentDocument = stream.readAllBytes(); + } + + client = FlagsmithClient.newBuilder() + .setApiKey("ser.benchmark") + .withConfiguration(FlagsmithConfig.newBuilder() + .baseUri("http://flagsmith.benchmark/api/v1/") + .addHttpInterceptor(serveEnvironmentDocument(environmentDocument)) + .withLocalEvaluation(true) + .withEnvironmentRefreshIntervalSeconds(86400) // Refreshed by benchmarks only + .build()) + .build(); + } + + @TearDown + public void closeClient() { + client.close(); + } + + /** + * Get a feature value from the environment flags. + */ + @Benchmark + public Object getEnvironmentFlags() throws FlagsmithClientError { + return client.getEnvironmentFlags().getFeatureValue("feature_1"); + } + + /** + * Check a feature for an identity with identity overrides. + */ + @Benchmark + public boolean getIdentityFlagsWithIdentityOverrides() throws FlagsmithClientError { + return client.getIdentityFlags(IDENTITY_OVERRIDES_IDENTIFIER).isFeatureEnabled("feature_8"); + } + + /** + * Get a feature value for an identity without traits. + */ + @Benchmark + public Object getIdentityFlagsNoTraits() throws FlagsmithClientError { + return client.getIdentityFlags(IDENTIFIER).getFeatureValue("feature_1"); + } + + /** + * Get a feature value for an identity with traits. + */ + @Benchmark + public Object getIdentityFlagsWithTraits() throws FlagsmithClientError { + return client.getIdentityFlags(IDENTIFIER, TRAITS).getFeatureValue("feature_1"); + } + + /** + * List the segments of an identity with traits. + */ + @Benchmark + public List getIdentitySegmentsWithTraits() throws FlagsmithClientError { + return client.getIdentitySegments(IDENTIFIER, TRAITS); + } + + private static Interceptor serveEnvironmentDocument(byte[] environmentDocument) { + return chain -> new Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create(environmentDocument, JSON)) + .build(); + } +} diff --git a/third-party/codspeed-jvm b/third-party/codspeed-jvm new file mode 160000 index 0000000..af05f6f --- /dev/null +++ b/third-party/codspeed-jvm @@ -0,0 +1 @@ +Subproject commit af05f6f316a9f520ef3d7f57bee4438ce30c6191 diff --git a/third-party/sdk-benchmarks b/third-party/sdk-benchmarks new file mode 160000 index 0000000..1b05624 --- /dev/null +++ b/third-party/sdk-benchmarks @@ -0,0 +1 @@ +Subproject commit 1b056240872fd3c4ddeb34c00593887c58fd2e20