Skip to content

Commit fd343a2

Browse files
jack-bergjack-berg
authored
perf: update otel benchmark dependencies, add otel bound instrument scenarios (#2451)
Updating benchmarks after substantial efforts to improve otel performance under contention. Changes: - Add scenarios for experimental bound instruments. I understand if you want to reject this until stable. - Disable exemplars for apples to apples comparison. My understanding is that the prom client_java cases don't have exemplars enabled, can change if I'm wrong. - Update to opentelemetry-java v1.66.0. If we wait until opentelemetry-java-instrumentation version is released which bundles the v1.66.0 version, the clutter in pom.xml goes away. Benchmark results on my machine: ``` Benchmark Mode Cnt Score Error Units CounterBenchmark.codahaleIncNoLabels thrpt 25 95236.680 ± 755.718 ops/s CounterBenchmark.openTelemetryAdd thrpt 25 54598.238 ± 290.633 ops/s CounterBenchmark.openTelemetryBoundAdd thrpt 25 82770.849 ± 342.576 ops/s CounterBenchmark.openTelemetryBoundInc thrpt 25 85165.317 ± 7.693 ops/s CounterBenchmark.openTelemetryInc thrpt 25 75136.160 ± 552.123 ops/s CounterBenchmark.openTelemetryIncNoLabels thrpt 25 85517.108 ± 538.105 ops/s CounterBenchmark.prometheusAdd thrpt 25 84533.092 ± 680.646 ops/s CounterBenchmark.prometheusInc thrpt 25 85135.753 ± 23.184 ops/s CounterBenchmark.prometheusNoLabelsInc thrpt 25 85061.556 ± 194.011 ops/s CounterBenchmark.simpleclientAdd thrpt 25 9174.173 ± 1542.709 ops/s CounterBenchmark.simpleclientInc thrpt 25 8860.946 ± 1306.154 ops/s CounterBenchmark.simpleclientNoLabelsInc thrpt 25 17502.624 ± 830.879 ops/s HistogramBenchmark.openTelemetryBoundClassic thrpt 25 25261.802 ± 6559.742 ops/s HistogramBenchmark.openTelemetryBoundExponential thrpt 25 2367.637 ± 381.165 ops/s HistogramBenchmark.openTelemetryClassic thrpt 25 28697.261 ± 4082.434 ops/s HistogramBenchmark.openTelemetryExponential thrpt 25 1868.494 ± 101.732 ops/s HistogramBenchmark.prometheusClassic thrpt 25 23600.746 ± 7292.571 ops/s HistogramBenchmark.prometheusClassicPerThread thrpt 25 58674.695 ± 98.620 ops/s HistogramBenchmark.prometheusClassicSingleThread thrpt 25 14865.898 ± 12.295 ops/s HistogramBenchmark.prometheusNative thrpt 25 10726.819 ± 677.244 ops/s HistogramBenchmark.simpleclient thrpt 25 17340.988 ± 266.176 ops/s ``` The [improvements](open-telemetry/opentelemetry-java#8717) to otel explicit histogram performance came from a design inspired by prometheus client_java. The code contains attribution. I did manage to slightly improve on the design - If you agree, I'm happy to chat about it and encourage prom client_java to copy any of the bits you think are helpful. --------- Signed-off-by: Jack Berg <34418638+jack-berg@users.noreply.github.com>
1 parent 6d60a5b commit fd343a2

3 files changed

Lines changed: 204 additions & 1 deletion

File tree

‎benchmarks/pom.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,31 @@
1919
<jmh.version>1.37</jmh.version>
2020
<simpleclient.version>0.16.0</simpleclient.version>
2121
<codahale.version>3.0.2</codahale.version>
22+
<!-- TODO: remove once opentelemetry-instrumentation-bom-alpha ships with otel 1.66.0. -->
23+
<benchmark.otel.version>1.66.0</benchmark.otel.version>
2224
<coverage.skip>true</coverage.skip>
2325
<javadoc.skip>true</javadoc.skip>
2426
</properties>
2527

2628
<dependencyManagement>
2729
<dependencies>
30+
<!-- TODO: remove these two BOM imports once opentelemetry-instrumentation-bom-alpha
31+
ships with otel 1.66.0. They must be listed before the instrumentation BOM so
32+
their versions win. -->
33+
<dependency>
34+
<groupId>io.opentelemetry</groupId>
35+
<artifactId>opentelemetry-bom</artifactId>
36+
<version>${benchmark.otel.version}</version>
37+
<type>pom</type>
38+
<scope>import</scope>
39+
</dependency>
40+
<dependency>
41+
<groupId>io.opentelemetry</groupId>
42+
<artifactId>opentelemetry-bom-alpha</artifactId>
43+
<version>${benchmark.otel.version}-alpha</version>
44+
<type>pom</type>
45+
<scope>import</scope>
46+
</dependency>
2847
<dependency>
2948
<groupId>io.opentelemetry.instrumentation</groupId>
3049
<artifactId>opentelemetry-instrumentation-bom-alpha</artifactId>
@@ -73,6 +92,10 @@
7392
<groupId>io.opentelemetry</groupId>
7493
<artifactId>opentelemetry-sdk-testing</artifactId>
7594
</dependency>
95+
<dependency>
96+
<groupId>io.opentelemetry</groupId>
97+
<artifactId>opentelemetry-api-incubator</artifactId>
98+
</dependency>
7699
</dependencies>
77100
<build>
78101
<finalName>${project.artifactId}</finalName>

‎benchmarks/src/main/java/io/prometheus/metrics/benchmarks/CounterBenchmark.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,15 @@
33
import io.opentelemetry.api.OpenTelemetry;
44
import io.opentelemetry.api.common.AttributeKey;
55
import io.opentelemetry.api.common.Attributes;
6+
import io.opentelemetry.api.incubator.metrics.BoundDoubleCounter;
7+
import io.opentelemetry.api.incubator.metrics.BoundLongCounter;
8+
import io.opentelemetry.api.incubator.metrics.ExtendedDoubleCounter;
9+
import io.opentelemetry.api.incubator.metrics.ExtendedLongCounter;
610
import io.opentelemetry.api.metrics.DoubleCounter;
711
import io.opentelemetry.api.metrics.LongCounter;
812
import io.opentelemetry.api.metrics.Meter;
913
import io.opentelemetry.sdk.OpenTelemetrySdk;
14+
import io.opentelemetry.sdk.metrics.ExemplarFilter;
1015
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
1116
import io.opentelemetry.sdk.resources.Resource;
1217
import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader;
@@ -136,6 +141,7 @@ public OpenTelemetryCounter() {
136141
SdkMeterProvider.builder()
137142
.registerMetricReader(InMemoryMetricReader.create())
138143
.setResource(Resource.getDefault())
144+
.setExemplarFilter(ExemplarFilter.alwaysOff())
139145
.build();
140146
OpenTelemetry openTelemetry =
141147
OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
@@ -153,6 +159,45 @@ public OpenTelemetryCounter() {
153159
}
154160
}
155161

162+
/**
163+
* OpenTelemetry counter using the incubator "bound instrument" API, which pre-binds a set of
164+
* attributes at initialization time to avoid a per-record attribute map lookup.
165+
*/
166+
@State(Scope.Benchmark)
167+
public static class OpenTelemetryBoundCounter {
168+
169+
final BoundLongCounter longCounter;
170+
final BoundDoubleCounter doubleCounter;
171+
172+
public OpenTelemetryBoundCounter() {
173+
174+
SdkMeterProvider sdkMeterProvider =
175+
SdkMeterProvider.builder()
176+
.registerMetricReader(InMemoryMetricReader.create())
177+
.setResource(Resource.getDefault())
178+
.setExemplarFilter(ExemplarFilter.alwaysOff())
179+
.build();
180+
OpenTelemetry openTelemetry =
181+
OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
182+
Meter meter =
183+
openTelemetry
184+
.meterBuilder("instrumentation-library-name")
185+
.setInstrumentationVersion("1.0.0")
186+
.build();
187+
Attributes attributes =
188+
Attributes.of(
189+
AttributeKey.stringKey("path"), "/",
190+
AttributeKey.stringKey("status"), "200");
191+
this.longCounter =
192+
((ExtendedLongCounter) meter.counterBuilder("test1").setDescription("test").build())
193+
.bind(attributes);
194+
this.doubleCounter =
195+
((ExtendedDoubleCounter)
196+
meter.counterBuilder("test2").ofDoubles().setDescription("test").build())
197+
.bind(attributes);
198+
}
199+
}
200+
156201
@Benchmark
157202
@Threads(4)
158203
public CounterDataPoint prometheusAdd(RandomNumbers randomNumbers, PrometheusCounter counter) {
@@ -198,6 +243,25 @@ public LongCounter openTelemetryIncNoLabels(OpenTelemetryCounter counter) {
198243
return counter.longCounter;
199244
}
200245

246+
@Benchmark
247+
@Threads(4)
248+
public BoundDoubleCounter openTelemetryBoundAdd(
249+
RandomNumbers randomNumbers, OpenTelemetryBoundCounter counter) {
250+
for (int i = 0; i < randomNumbers.randomNumbers.length; i++) {
251+
counter.doubleCounter.add(randomNumbers.randomNumbers[i]);
252+
}
253+
return counter.doubleCounter;
254+
}
255+
256+
@Benchmark
257+
@Threads(4)
258+
public BoundLongCounter openTelemetryBoundInc(OpenTelemetryBoundCounter counter) {
259+
for (int i = 0; i < 10 * 1024; i++) {
260+
counter.longCounter.add(1);
261+
}
262+
return counter.longCounter;
263+
}
264+
201265
@Benchmark
202266
@Threads(4)
203267
public io.prometheus.client.Counter.Child simpleclientAdd(

‎benchmarks/src/main/java/io/prometheus/metrics/benchmarks/HistogramBenchmark.java‎

Lines changed: 117 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
package io.prometheus.metrics.benchmarks;
22

33
import io.opentelemetry.api.OpenTelemetry;
4+
import io.opentelemetry.api.common.AttributeKey;
5+
import io.opentelemetry.api.common.Attributes;
6+
import io.opentelemetry.api.incubator.metrics.BoundDoubleHistogram;
7+
import io.opentelemetry.api.incubator.metrics.ExtendedDoubleHistogram;
48
import io.opentelemetry.api.metrics.Meter;
59
import io.opentelemetry.sdk.OpenTelemetrySdk;
610
import io.opentelemetry.sdk.metrics.Aggregation;
11+
import io.opentelemetry.sdk.metrics.ExemplarFilter;
712
import io.opentelemetry.sdk.metrics.InstrumentSelector;
813
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
914
import io.opentelemetry.sdk.metrics.View;
@@ -88,13 +93,15 @@ public SimpleclientHistogram() {
8893
public static class OpenTelemetryClassicHistogram {
8994

9095
final io.opentelemetry.api.metrics.DoubleHistogram histogram;
96+
final Attributes attributes;
9197

9298
public OpenTelemetryClassicHistogram() {
9399

94100
SdkMeterProvider sdkMeterProvider =
95101
SdkMeterProvider.builder()
96102
.registerMetricReader(InMemoryMetricReader.create())
97103
.setResource(Resource.getDefault())
104+
.setExemplarFilter(ExemplarFilter.alwaysOff())
98105
.registerView(
99106
InstrumentSelector.builder().setName("test").build(),
100107
View.builder()
@@ -112,20 +119,26 @@ public OpenTelemetryClassicHistogram() {
112119
.setInstrumentationVersion("1.0.0")
113120
.build();
114121
this.histogram = meter.histogramBuilder("test").setDescription("test").build();
122+
this.attributes =
123+
Attributes.of(
124+
AttributeKey.stringKey("path"), "/",
125+
AttributeKey.stringKey("status"), "200");
115126
}
116127
}
117128

118129
@State(Scope.Benchmark)
119130
public static class OpenTelemetryExponentialHistogram {
120131

121132
final io.opentelemetry.api.metrics.DoubleHistogram histogram;
133+
final Attributes attributes;
122134

123135
public OpenTelemetryExponentialHistogram() {
124136

125137
SdkMeterProvider sdkMeterProvider =
126138
SdkMeterProvider.builder()
127139
.registerMetricReader(InMemoryMetricReader.create())
128140
.setResource(Resource.getDefault())
141+
.setExemplarFilter(ExemplarFilter.alwaysOff())
129142
.registerView(
130143
InstrumentSelector.builder().setName("test").build(),
131144
View.builder()
@@ -140,6 +153,89 @@ public OpenTelemetryExponentialHistogram() {
140153
.setInstrumentationVersion("1.0.0")
141154
.build();
142155
this.histogram = meter.histogramBuilder("test").setDescription("test").build();
156+
this.attributes =
157+
Attributes.of(
158+
AttributeKey.stringKey("path"), "/",
159+
AttributeKey.stringKey("status"), "200");
160+
}
161+
}
162+
163+
/**
164+
* OpenTelemetry classic histogram using the incubator "bound instrument" API to pre-bind
165+
* attributes at initialization time.
166+
*/
167+
@State(Scope.Benchmark)
168+
public static class OpenTelemetryBoundClassicHistogram {
169+
170+
final BoundDoubleHistogram histogram;
171+
172+
public OpenTelemetryBoundClassicHistogram() {
173+
SdkMeterProvider sdkMeterProvider =
174+
SdkMeterProvider.builder()
175+
.registerMetricReader(InMemoryMetricReader.create())
176+
.setResource(Resource.getDefault())
177+
.setExemplarFilter(ExemplarFilter.alwaysOff())
178+
.registerView(
179+
InstrumentSelector.builder().setName("test").build(),
180+
View.builder()
181+
.setAggregation(
182+
Aggregation.explicitBucketHistogram(
183+
Arrays.asList(
184+
.005, .01, .025, .05, .1, .25, .5, 1.0, 2.5, 5.0, 10.0)))
185+
.build())
186+
.build();
187+
OpenTelemetry openTelemetry =
188+
OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
189+
Meter meter =
190+
openTelemetry
191+
.meterBuilder("instrumentation-library-name")
192+
.setInstrumentationVersion("1.0.0")
193+
.build();
194+
Attributes attributes =
195+
Attributes.of(
196+
AttributeKey.stringKey("path"), "/",
197+
AttributeKey.stringKey("status"), "200");
198+
this.histogram =
199+
((ExtendedDoubleHistogram) meter.histogramBuilder("test").setDescription("test").build())
200+
.bind(attributes);
201+
}
202+
}
203+
204+
/**
205+
* OpenTelemetry exponential histogram using the incubator "bound instrument" API to pre-bind
206+
* attributes at initialization time.
207+
*/
208+
@State(Scope.Benchmark)
209+
public static class OpenTelemetryBoundExponentialHistogram {
210+
211+
final BoundDoubleHistogram histogram;
212+
213+
public OpenTelemetryBoundExponentialHistogram() {
214+
SdkMeterProvider sdkMeterProvider =
215+
SdkMeterProvider.builder()
216+
.registerMetricReader(InMemoryMetricReader.create())
217+
.setResource(Resource.getDefault())
218+
.setExemplarFilter(ExemplarFilter.alwaysOff())
219+
.registerView(
220+
InstrumentSelector.builder().setName("test").build(),
221+
View.builder()
222+
.setAggregation(Aggregation.base2ExponentialBucketHistogram(10_000, 5))
223+
.build())
224+
.build();
225+
OpenTelemetry openTelemetry =
226+
OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
227+
Meter meter =
228+
openTelemetry
229+
.meterBuilder("instrumentation-library-name")
230+
.setInstrumentationVersion("1.0.0")
231+
.build();
232+
Attributes attributes =
233+
Attributes.of(
234+
AttributeKey.stringKey("path"), "/",
235+
AttributeKey.stringKey("status"), "200");
236+
this.histogram =
237+
((ExtendedDoubleHistogram) meter.histogramBuilder("test").setDescription("test").build())
238+
.bind(attributes);
143239
}
144240
}
145241

@@ -198,7 +294,7 @@ public io.prometheus.client.Histogram simpleclient(
198294
public io.opentelemetry.api.metrics.DoubleHistogram openTelemetryClassic(
199295
RandomNumbers randomNumbers, OpenTelemetryClassicHistogram histogram) {
200296
for (int i = 0; i < randomNumbers.randomNumbers.length; i++) {
201-
histogram.histogram.record(randomNumbers.randomNumbers[i]);
297+
histogram.histogram.record(randomNumbers.randomNumbers[i], histogram.attributes);
202298
}
203299
return histogram.histogram;
204300
}
@@ -207,6 +303,26 @@ public io.opentelemetry.api.metrics.DoubleHistogram openTelemetryClassic(
207303
@Threads(4)
208304
public io.opentelemetry.api.metrics.DoubleHistogram openTelemetryExponential(
209305
RandomNumbers randomNumbers, OpenTelemetryExponentialHistogram histogram) {
306+
for (int i = 0; i < randomNumbers.randomNumbers.length; i++) {
307+
histogram.histogram.record(randomNumbers.randomNumbers[i], histogram.attributes);
308+
}
309+
return histogram.histogram;
310+
}
311+
312+
@Benchmark
313+
@Threads(4)
314+
public BoundDoubleHistogram openTelemetryBoundClassic(
315+
RandomNumbers randomNumbers, OpenTelemetryBoundClassicHistogram histogram) {
316+
for (int i = 0; i < randomNumbers.randomNumbers.length; i++) {
317+
histogram.histogram.record(randomNumbers.randomNumbers[i]);
318+
}
319+
return histogram.histogram;
320+
}
321+
322+
@Benchmark
323+
@Threads(4)
324+
public BoundDoubleHistogram openTelemetryBoundExponential(
325+
RandomNumbers randomNumbers, OpenTelemetryBoundExponentialHistogram histogram) {
210326
for (int i = 0; i < randomNumbers.randomNumbers.length; i++) {
211327
histogram.histogram.record(randomNumbers.randomNumbers[i]);
212328
}

0 commit comments

Comments
 (0)