[performance] Update benchmark project

performance/README.md

@@ -16,6 +16,38 @@ mvn compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath
 -input sample_dataset.sql -name default -rows 10"
 ```
 
+#### Target Server
+
+By default the benchmarks start a local ClickHouse Docker container and run
+against it. To run against an existing remote server instead (ClickHouse Cloud
+or any self-hosted instance), set `CLICKHOUSE_URL`:
+
+```shell
+export CLICKHOUSE_URL="https://default:my-password@abc123.clickhouse.cloud:8443?cluster=true"
+mvn compile exec:exec
+```
+
+```shell
+export CLICKHOUSE_URL="http://default@localhost:8123"
+mvn compile exec:exec
+```
+
+The URL is parsed as: `<scheme>://<username>[:<password>]@<host>[:<port>][?cluster=true]`
+- scheme (`http`/`https`) selects whether SSL is used
+- username/password come from the URL's user-info; if the password is omitted
+  the client connects without one, and if the username is omitted it defaults
+  to `default`
+- port is optional; if omitted it defaults to `8443` for `https` and `8123`
+  for `http`
+- `cluster` is optional (default `false`) and tells the benchmarks whether the
+  remote server is part of a replicated cluster (e.g. ClickHouse Cloud). When
+  `true`, a `SYSTEM SYNC REPLICA` is issued after writes. Leave it unset for a
+  plain standalone remote server — its tables aren't replicated and it will
+  reject that statement.
+
+When `CLICKHOUSE_URL` is unset, the local Docker container is started
+automatically and no other configuration is needed.
+
 #### Running Benchmarks 
 
 With default settings :
@@ -39,9 +71,21 @@ Other options:
   - "q" - QueryClient - query operation benchmarks
   - "ci" - ConcurrentInsertClient - concurrent version of insert benchmarks
   - "cq" - ConcurrentQueryClient - concurrent version of query benchmarks
-  - "lz" - Compression - compression related benchmarks
+  - "lz" - Compression - LZ4 output stream benchmarks (no server involved)
+  - "comp" - Compression - query/insert compression matrix of clients, methods, algorithms and formats (see below)
   - "writer" - Serializer - serialization only logic benchmarks
   - "reader" - DeSerilalizer - deserialization only logic benchmarks
   - "mixed" - MixedWorkload 
   - "jq" - JDBCQuery - query operations using JDBC 
-  - "ji" - JDBCInsert - insert operation using JDBC
+  - "ji" - JDBCInsert - insert operation using JDBC
+
+Compression matrix filters (used with `-b comp` or `-b all`, each defaults to all values):
+- "-cc" - clients: `v1,v2`
+- "-cm" - compression methods: `http` (`Content-Encoding`/`Accept-Encoding`), `native` (ClickHouse block compression, `use_http_compression = false`)
+- "-ca" - algorithms: `lz4,zstd,snappy,brotli`
+- "-cf" - formats: `RowBinaryWithNamesAndTypes,JSONEachRow` (any ClickHouse format name, e.g. `RowBinary`, is accepted)
+
+Ex.: `-b comp -cc v2 -cm http -ca zstd,lz4 -cf JSONEachRow`.
+
+Combinations a client doesn't support are skipped: V1 sends LZ4 only natively and other algorithms only over HTTP,
+V2 native mode is LZ4 only, V2 can't compress requests with brotli, and snappy works with neither client.

performance/pom.xml

@@ -18,6 +18,9 @@
         <ch.jdbc.revision>0.11.0-rc1-SNAPSHOT</ch.jdbc.revision>
         <jmh.version>1.37</jmh.version>
         <testcontainers.version>2.0.2</testcontainers.version>
+        <zstd-jni.version>1.5.7-6</zstd-jni.version>
+        <brotli.version>0.1.2</brotli.version>
+        <brotli4j.version>1.12.0</brotli4j.version>
 
         <antrun-plugin.version>3.1.0</antrun-plugin.version>
         <assembly-plugin.version>3.6.0</assembly-plugin.version>
@@ -94,6 +97,23 @@
             <classifier>all</classifier>
         </dependency>
 
+        <!-- Compression codecs are optional in the client and not bundled into the "all" jar -->
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <version>${zstd-jni.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.brotli</groupId>
+            <artifactId>dec</artifactId>
+            <version>${brotli.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.aayushatharva.brotli4j</groupId>
+            <artifactId>brotli4j</artifactId>
+            <version>${brotli4j.version}</version>
+        </dependency>
+
     </dependencies>
 
     <build>

performance/src/main/java/com/clickhouse/benchmark/BenchmarkRunner.java

@@ -10,6 +10,7 @@
 import com.clickhouse.benchmark.clients.MixedWorkload;
 import com.clickhouse.benchmark.clients.QueryClient;
 import com.clickhouse.benchmark.clients.Serializers;
+import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.profile.GCProfiler;
 import org.openjdk.jmh.profile.MemPoolProfiler;
@@ -21,14 +22,22 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
-import static com.clickhouse.benchmark.TestEnvironment.isCloud;
+import static com.clickhouse.benchmark.TestEnvironment.isRemote;
 
 
 public class BenchmarkRunner {
@@ -38,11 +47,11 @@ public static void main(String[] args) throws Exception {
         LOGGER.info("Starting Benchmarks");
         Map<String, String> options = parseArgs(args);
         System.out.println("Start Benchmarks with options: " + options);
-        final String env = isCloud() ? "cloud" : "local";
+        final String env = isRemote() ? "remote" : "local";
         final long time = System.currentTimeMillis();
 
         final int measurementIterations = Integer.parseInt(options.getOrDefault("-m", "10"));
-        final int measurementTime = Integer.parseInt(options.getOrDefault("-t", "" + (isCloud() ? 30 : 10)));
+        final int measurementTime = Integer.parseInt(options.getOrDefault("-t", "" + (isRemote() ? 30 : 10)));
         final String resultFile = String.format("jmh-results-%s-%s.json", env, time);
         final String outputFile = String.format("jmh-results-%s-%s.out", env, time);
         final String datasetName = options.getOrDefault("-d", "file://default.csv");
@@ -77,7 +86,8 @@ public static void main(String[] args) throws Exception {
         String[] testMaskParts = testMask.split(",");
 
         SortedSet<String> benchmarks = new TreeSet<>();
-        if (testMaskParts[0].equalsIgnoreCase("all")) {
+        boolean runAll = testMaskParts[0].equalsIgnoreCase("all");
+        if (runAll) {
             BENCHMARK_FLAGS.values().forEach((b) -> {
                 optBuilder.include(b);
                 benchmarks.add(b);
@@ -92,6 +102,20 @@ public static void main(String[] args) throws Exception {
             }
         }
 
+        if (runAll || Arrays.asList(testMaskParts).contains(COMPRESSION_MATRIX_FLAG)) {
+            List<String> matrix = compressionMatrix(options);
+            if (matrix.isEmpty()) {
+                System.out.println("No compression benchmark matches the selected clients/methods/algorithms");
+            }
+            for (String benchmark : matrix) {
+                optBuilder.include(Pattern.quote(benchmark) + "$");
+                benchmarks.add(benchmark);
+            }
+            if (options.containsKey("-cf")) {
+                optBuilder.param("format", options.get("-cf").split(","));
+            }
+        }
+
         System.out.println("Running benchmarks: " + benchmarks);
         new Runner(optBuilder.build()).run();
     }
@@ -104,7 +128,7 @@ private static Map<String, String> buildBenchmarkFlags() {
         map.put("i", InsertClient.class.getName());
         map.put("cq", ConcurrentQueryClient.class.getName());
         map.put("ci", ConcurrentInsertClient.class.getName());
-        map.put("lz", Compression.class.getName());
+        map.put("lz", Compression.class.getName() + ".CompressingOutputStream");
         map.put("reader", Deserializers.class.getName());
         map.put("writer", Serializers.class.getName());
         map.put("mixed", MixedWorkload.class.getName());
@@ -113,6 +137,43 @@ private static Map<String, String> buildBenchmarkFlags() {
         return map;
     }
 
+    private static final String COMPRESSION_MATRIX_FLAG = "comp";
+
+    private static final Pattern COMPRESSION_BENCHMARK_NAME =
+            Pattern.compile("(query|insert)(V1|V2)(Native|Http)(Lz4|Zstd|Snappy|Brotli)");
+
+    /**
+     * Selects {@link Compression} matrix benchmarks by client ({@code -cc v1,v2}), compression method
+     * ({@code -cm http,native}) and algorithm ({@code -ca lz4,zstd,snappy,brotli}). Each filter defaults to all
+     * values. Combinations a client does not support have no benchmark method and are skipped.
+     */
+    private static List<String> compressionMatrix(Map<String, String> options) {
+        Set<String> clients = filterValues(options, "-cc", "v1,v2");
+        Set<String> methods = filterValues(options, "-cm", "http,native");
+        Set<String> algorithms = filterValues(options, "-ca", "lz4,zstd,snappy,brotli");
+
+        List<String> selected = new ArrayList<>();
+        for (Method method : Compression.class.getMethods()) {
+            Matcher m = COMPRESSION_BENCHMARK_NAME.matcher(method.getName());
+            if (method.isAnnotationPresent(Benchmark.class) && m.matches()
+                    && clients.contains(m.group(2).toLowerCase())
+                    && methods.contains(m.group(3).toLowerCase())
+                    && algorithms.contains(m.group(4).toLowerCase())) {
+                selected.add(Compression.class.getName() + "." + method.getName());
+            }
+        }
+        Collections.sort(selected);
+        return selected;
+    }
+
+    private static Set<String> filterValues(Map<String, String> options, String key, String defaultValues) {
+        Set<String> values = new HashSet<>();
+        for (String value : options.getOrDefault(key, defaultValues).split(",")) {
+            values.add(value.trim().toLowerCase());
+        }
+        return values;
+    }
+
     private static Map<String, String> parseArgs(String[] args) {
         Map<String, String> options = new HashMap<>();
         for (int i = 0; i < args.length; i+=2) {

performance/src/main/java/com/clickhouse/benchmark/TestEnvironment.java

@@ -10,6 +10,7 @@
 import org.testcontainers.containers.wait.strategy.Wait;
 
 import java.net.InetSocketAddress;
+import java.net.URI;
 import java.time.Duration;
 import java.util.Collections;
 
@@ -25,50 +26,75 @@ public class TestEnvironment {
 
 
     //Environment Variables
-    public static boolean isCloud() {
-        return System.getenv("CLICKHOUSE_HOST") != null;
+    // Set CLICKHOUSE_URL to point the benchmarks at an existing remote ClickHouse
+    // server (ClickHouse Cloud or any self-hosted instance), e.g.:
+    //   https://default:my-password@abc123.clickhouse.cloud:8443?cluster=true
+    //   http://default@localhost:8123
+    // The scheme selects HTTP vs HTTPS/SSL, and credentials come from the URL's
+    // user-info (username[:password]). When unset, a local Docker container is
+    // started automatically instead.
+    //
+    // The optional "cluster" query parameter (default false) tells the benchmarks
+    // whether the remote server is part of a replicated cluster (e.g. ClickHouse
+    // Cloud) and therefore needs a SYSTEM SYNC REPLICA after writes. Leave it
+    // unset/false for a plain standalone remote server, whose tables aren't
+    // replicated and would reject that statement.
+    private static URI getRemoteUrl() {
+        String url = System.getenv("CLICKHOUSE_URL");
+        return url == null ? null : URI.create(url);
     }
-    public static String getHost() {
-        String host = System.getenv("CLICKHOUSE_HOST");
-        if (host == null) {
-            host = container.getHost();
-        }
-
-        return host;
+    public static boolean isRemote() {
+        return getRemoteUrl() != null;
     }
-    public static int getPort() {
-        String port = System.getenv("CLICKHOUSE_PORT");
-        if (port == null) {
-            if (isCloud()) {//Default handling for ClickHouse Cloud
-                port = "8443";
-            } else {
-                port = String.valueOf(container.getMappedPort(8123));
+    public static boolean isSsl() {
+        URI url = getRemoteUrl();
+        return url != null && "https".equalsIgnoreCase(url.getScheme());
+    }
+    public static boolean isCluster() {
+        URI url = getRemoteUrl();
+        String query = url == null ? null : url.getQuery();
+        if (query == null) {
+            return false;
+        }
+        for (String param : query.split("&")) {
+            String[] kv = param.split("=", 2);
+            if (kv.length == 2 && kv[0].equalsIgnoreCase("cluster")) {
+                return Boolean.parseBoolean(kv[1]);
             }
         }
-
-        return Integer.parseInt(port);
+        return false;
     }
-    public static String getPassword() {
-        String password = System.getenv("CLICKHOUSE_PASSWORD");
-        if (password == null) {
-            if (isCloud()) {
-                password = System.getenv("CLICKHOUSE_PASSWORD");
-            } else {
-                password = container.getPassword();
-            }
+    public static String getHost() {
+        URI url = getRemoteUrl();
+        return url != null ? url.getHost() : container.getHost();
+    }
+    public static int getPort() {
+        URI url = getRemoteUrl();
+        if (url != null) {
+            return url.getPort() != -1 ? url.getPort() : (isSsl() ? 8443 : 8123);
         }
-        return password;
+        return container.getMappedPort(8123);
     }
     public static String getUsername() {
-        String username = System.getenv("CLICKHOUSE_USERNAME");
-        if (username == null) {
-            if (isCloud()) {
-                username = "default";
-            } else {
-                username = container.getUsername();
-            }
+        URI url = getRemoteUrl();
+        if (url == null) {
+            return container.getUsername();
+        }
+        String userInfo = url.getUserInfo();
+        if (userInfo == null) {
+            return "default";
+        }
+        int sep = userInfo.indexOf(':');
+        return sep == -1 ? userInfo : userInfo.substring(0, sep);
+    }
+    public static String getPassword() {
+        URI url = getRemoteUrl();
+        if (url == null) {
+            return container.getPassword();
         }
-        return username;
+        String userInfo = url.getUserInfo();
+        int sep = userInfo == null ? -1 : userInfo.indexOf(':');
+        return sep == -1 ? null : userInfo.substring(sep + 1);
     }
     public static ClickHouseNode getServer() {
         return serverNode;
@@ -79,8 +105,8 @@ public static ClickHouseNode getServer() {
     public static void setupEnvironment() {
         LOGGER.info("Initializing ClickHouse test environment...");
 
-        if (isCloud()) {
-            LOGGER.info("Using ClickHouse Cloud");
+        if (isRemote()) {
+            LOGGER.info("Using remote ClickHouse server at {}:{}", getHost(), getPort());
             container = null;
         } else {
             LOGGER.info("Using ClickHouse Docker container");
@@ -95,15 +121,15 @@ public static void setupEnvironment() {
         serverNode = ClickHouseNode.builder(ClickHouseNode.builder().build())
                 .address(ClickHouseProtocol.HTTP, new InetSocketAddress(getHost(), getPort()))
                 .credentials(ClickHouseCredentials.fromUserAndPassword(getUsername(), getPassword()))
-                .options(Collections.singletonMap(ClickHouseClientOption.SSL.getKey(), isCloud() ? "true" : "false"))
+                .options(Collections.singletonMap(ClickHouseClientOption.SSL.getKey(), isSsl() ? "true" : "false"))
                 .database(DB_NAME)
                 .build();
         createDatabase();
     }
 
     public static void cleanupEnvironment() {
         LOGGER.info("Cleaning up ClickHouse test environment...");
-        if (isCloud()) {
+        if (isRemote()) {
             dropDatabase();
         }