Kafka Connect: Update runtime distribution LICENSE and NOTICE

docs/docs/kafka-connect.md

@@ -48,8 +48,12 @@ The connector zip archive is created as part of the Iceberg build. You can run t
 ```bash
 ./gradlew -x test -x integrationTest clean build
 ```
-The zip archive will be found under `./kafka-connect/kafka-connect-runtime/build/distributions`. There is
-one distribution that bundles the Hive Metastore client and related dependencies, and one that does not.
+The zip archive will be found under `./kafka-connect/kafka-connect-runtime/build/distributions`. The default
+build produces a distribution that does not bundle the Hive Metastore client. A distribution that also bundles
+the Hive Metastore client and related dependencies is not built by default; to build it, run:
+```bash
+./gradlew :iceberg-kafka-connect:iceberg-kafka-connect-runtime:hiveDistZip
+```
 Copy the distribution archive into the Kafka Connect plugins directory on all nodes.
 
 ## Requirements
@@ -125,8 +129,8 @@ Messages should be converted to a struct or map using the appropriate Kafka Conn
 The `iceberg.catalog.*` properties are required for connecting to the Iceberg catalog. The core catalog
 types are included in the default distribution, including REST, Glue, DynamoDB, Hadoop, Nessie,
 JDBC, Hive and BigQuery Metastore. JDBC drivers are not included in the default distribution, so you will need to include
-those if needed. When using a Hive catalog, you can use the distribution that includes the Hive metastore client,
-otherwise you will need to include that yourself.
+those if needed. When using a Hive catalog, you will need to include the Hive Metastore client yourself or build
+the distribution that bundles it (see [Installation](#installation)).
 
 To set the catalog type, you can set `iceberg.catalog.type` to `rest`, `hive`, or `hadoop`. For other
 catalog types, you need to instead set `iceberg.catalog.catalog-impl` to the name of the catalog class.
@@ -142,7 +146,7 @@ catalog types, you need to instead set `iceberg.catalog.catalog-impl` to the nam
 
 #### Hive example
 
-NOTE: Use the distribution that includes the HMS client (or include the HMS client yourself). Use `S3FileIO` when
+NOTE: Include the HMS client yourself, or build the distribution that bundles it. Use `S3FileIO` when
 using S3 for storage and `GCSFileIO` when using GCS (the default is `HadoopFileIO` with `HiveCatalog`).
 ```
 "iceberg.catalog.type": "hive",

kafka-connect/build.gradle

@@ -66,13 +66,21 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect') {
 project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') {
   apply plugin: 'distribution'
 
+  def awsBundle = evaluationDependsOn(':iceberg-aws-bundle')
+  def gcpBundle = evaluationDependsOn(':iceberg-gcp-bundle')
+  def azureBundle = evaluationDependsOn(':iceberg-azure-bundle')
+
   configurations {
     hive {
       extendsFrom runtimeClasspath
     }
     all {
       exclude group: 'javax.activation', module: 'activation'
       exclude group: 'org.jspecify', module: 'jspecify'
+      exclude group: 'com.google.code.findbugs', module: 'jsr305'
+      // as in iceberg-aws-bundle: no log4j binding inside the connector plugin
+      exclude group: 'org.apache.logging.log4j'
+      exclude group: 'org.apache.logging.slf4j'
       // force upgrades for dependencies with known vulnerabilities...
       resolutionStrategy {
         force 'org.codehaus.jettison:jettison:1.5.7'
@@ -120,31 +128,17 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') {
     implementation project(':iceberg-orc')
     implementation project(':iceberg-parquet')
 
+    // cloud SDK dependencies come from the bundle projects so the loose jars and the
+    // LICENSE and NOTICE stay in sync with the bundles; the bundle jars themselves are not shipped
     implementation project(':iceberg-aws')
-    implementation platform(libs.awssdk.bom)
-    implementation 'software.amazon.awssdk:apache-client'
-    implementation 'software.amazon.awssdk:auth'
-    implementation "software.amazon.awssdk:http-auth-aws-crt"
-    implementation 'software.amazon.awssdk:iam'
-    implementation 'software.amazon.awssdk:sso'
-    implementation 'software.amazon.awssdk:s3'
-    implementation 'software.amazon.awssdk:kms'
-    implementation 'software.amazon.awssdk:glue'
-    implementation 'software.amazon.awssdk:sts'
-    implementation 'software.amazon.awssdk:dynamodb'
-    implementation 'software.amazon.awssdk:lakeformation'
+    awsBundle.configurations.runtimeClasspath.allDependencies.each { add('implementation', it.copy()) }
 
     implementation project(':iceberg-gcp')
     implementation project(':iceberg-bigquery')
-    implementation platform(libs.google.libraries.bom)
-    implementation 'com.google.cloud:google-cloud-storage'
-    implementation 'com.google.cloud:google-cloud-bigquery'
-    implementation 'com.google.cloud:google-cloud-core'
+    gcpBundle.configurations.runtimeClasspath.allDependencies.each { add('implementation', it.copy()) }
 
     implementation project(':iceberg-azure')
-    implementation platform(libs.azuresdk.bom)
-    implementation 'com.azure:azure-storage-file-datalake'
-    implementation 'com.azure:azure-identity'
+    azureBundle.configurations.runtimeClasspath.allDependencies.each { add('implementation', it.copy()) }
 
     hive project(':iceberg-hive-metastore')
     hive(libs.hive2.metastore) {
@@ -253,7 +247,11 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') {
   integrationTest.dependsOn installDist
   check.dependsOn integrationTest
 
-  assemble.dependsOn distZip, hiveDistZip
+  // assemble builds every distribution's archives through the archives configuration;
+  // keep the Hive distribution opt-in via assembleHiveDist
+  configurations.archives.artifacts.removeAll {
+    it.file in [hiveDistZip.archiveFile.get().asFile, hiveDistTar.archiveFile.get().asFile]
+  }
 
   apply from: "${rootDir}/runtime-deps.gradle"
 }