Skip to content

ATLAS-5336: Upgrade Kafka to 3.9.1; embedded notification broker uses KRaft (no ZooKeeper) #689

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sarathsubramanian merged 14 commits into from
Jul 8, 2026
Merged

ATLAS-5336: Upgrade Kafka to 3.9.1; embedded notification broker uses KRaft (no ZooKeeper) #689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
ec1f39b
ATLAS-5336: Upgrade Kafka to 3.9.1; embedded notification broker uses…
Jul 4, 2026
4c766d9
Document jackson-jaxrs 2.12.7 pin for Jersey 1.19 compatibility.
Jul 6, 2026
faa7c98
Remove unused junit-jupiter-api from atlas-notification compile deps.
Jul 6, 2026
7ba6bff
Scope Kafka test and unit test deps in notification to test.
Jul 6, 2026
c6c57ca
Add test-scoped junit-jupiter-api to webapp for Kafka embedded broker…
Jul 6, 2026
66c5a25
Fix flaky ImportTaskListenerImplTest async consumer assertions on CI.
Jul 6, 2026
aeabd5d
Merge branch 'master' into ATLAS-5336.
Jul 6, 2026
90c08fb
Harden BasicSearchIT and AdminExportImportTestIT against CI flakes.
Jul 6, 2026
97d7882
Fix IT scoping and Failsafe/TestNG classpath for Kafka 3.9.
Jul 6, 2026
09b1b2f
Bind embedded KRaft broker to configured bootstrap.servers for ITs.
Jul 7, 2026
17a75eb
Pre-serialize classification request bodies to avoid Codehaus Jackson…
Jul 7, 2026
e57884e
Pin Jackson 2.16 on sqoop-bridge IT classpath for AtlasClient v1.
Jul 7, 2026
d22b7f8
Pin Jackson 2.16 databind on storm-bridge IT classpath for AtlasClien…
Jul 7, 2026
42a3c0a
Skip trino-extractor Failsafe ITs on default mvn verify.
Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions addons/sqoop-bridge/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@

<dependencies>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client-v1</artifactId>
Expand Down Expand Up @@ -129,6 +135,13 @@
</exclusions>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
Expand Down
8 changes: 7 additions & 1 deletion addons/storm-bridge/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@
<description>Apache Atlas Storm Bridge Module</description>

<dependencies>
<!-- apache atlas core dependencies -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>

<dependency>
<groupId>commons-collections</groupId>
Expand All @@ -55,7 +61,7 @@
<artifactId>metrics-graphite</artifactId>
<version>${dropwizard-metrics}</version>
</dependency>
<!-- apache atlas core dependencies -->

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client-v1</artifactId>
Expand Down
2 changes: 2 additions & 0 deletions addons/trino-extractor/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
<properties>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
<checkstyle.skip>false</checkstyle.skip>
<!-- No live-stack ITs yet; avoid Failsafe/TestNG on default mvn verify (see ATLAS-5338). -->
<skipITs>true</skipITs>
</properties>

<dependencies>
Expand Down
8 changes: 4 additions & 4 deletions client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,23 +518,23 @@ public void addClassification(ClassificationAssociateRequest request) throws Atl
}

public void addClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException {
callAPI(formatPathParameters(API_V2.ADD_CLASSIFICATIONS, guid), (Class<?>) null, classifications, (String[]) null);
callAPI(formatPathParameters(API_V2.ADD_CLASSIFICATIONS, guid), (Class<?>) null, AtlasType.toJson(classifications), (String[]) null);
}

public void addClassifications(String typeName, Map<String, String> uniqAttributes, List<AtlasClassification> classifications) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributes);

callAPI(formatPathParameters(API_V2.ADD_CLASSIFICATION_BY_TYPE_AND_ATTRIBUTE, typeName), (Class<?>) null, classifications, queryParams);
callAPI(formatPathParameters(API_V2.ADD_CLASSIFICATION_BY_TYPE_AND_ATTRIBUTE, typeName), (Class<?>) null, AtlasType.toJson(classifications), queryParams);
}

public void updateClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException {
callAPI(formatPathParameters(API_V2.UPDATE_CLASSIFICATIONS, guid), (Class<?>) null, classifications);
callAPI(formatPathParameters(API_V2.UPDATE_CLASSIFICATIONS, guid), (Class<?>) null, AtlasType.toJson(classifications));
}

public void updateClassifications(String typeName, Map<String, String> uniqAttributes, List<AtlasClassification> classifications) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributes);

callAPI(formatPathParameters(API_V2.UPDATE_CLASSIFICATION_BY_TYPE_AND_ATTRIBUTE, typeName), (Class<?>) null, classifications, queryParams);
callAPI(formatPathParameters(API_V2.UPDATE_CLASSIFICATION_BY_TYPE_AND_ATTRIBUTE, typeName), (Class<?>) null, AtlasType.toJson(classifications), queryParams);
}

public String setClassifications(AtlasEntityHeaders entityHeaders) throws AtlasServiceException {
Expand Down
2 changes: 1 addition & 1 deletion dev-support/atlas-docker/.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ ATLAS_VERSION=3.0.0-SNAPSHOT
UBUNTU_VERSION=20.04
HADOOP_VERSION=3.4.2
HBASE_VERSION=2.6.4
KAFKA_VERSION=2.8.2
KAFKA_VERSION=3.9.1
HIVE_VERSION=4.1.0

ATLAS_BACKEND=hbase
9 changes: 5 additions & 4 deletions dev-support/atlas-docker/Dockerfile.atlas-kafka
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,18 @@ FROM atlas-base:latest

ARG ATLAS_VERSION
ARG KAFKA_VERSION
ARG KAFKA_SCALA_VERSION=2.13

COPY ./dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/
COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/
COPY ./downloads/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz /home/atlas/dist/

COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/
COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/
COPY ./scripts/atlas-kafka-application.properties /home/atlas/scripts/

RUN tar xvfz /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \
RUN tar xvfz /home/atlas/dist/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz --directory=/opt/ && \
ln -s /opt/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION} /opt/kafka && \
rm -f /home/atlas/dist/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz && \
tar xvfz /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz --directory=/opt/ && \
ln -s /opt/apache-atlas-kafka-hook-${ATLAS_VERSION} /opt/apache-atlas-kafka-hook && \
rm -f /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz && \
Expand Down
2 changes: 1 addition & 1 deletion dev-support/atlas-docker/download-archives.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,6 @@ downloadIfNotPresent() {

downloadIfNotPresent hadoop-${HADOOP_VERSION}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}
downloadIfNotPresent hbase-${HBASE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hbase/${HBASE_VERSION}
downloadIfNotPresent kafka_2.12-${KAFKA_VERSION}.tgz https://archive.apache.org/dist/kafka/${KAFKA_VERSION}
downloadIfNotPresent kafka_2.13-${KAFKA_VERSION}.tgz https://archive.apache.org/dist/kafka/${KAFKA_VERSION}
downloadIfNotPresent apache-hive-${HIVE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}

5 changes: 1 addition & 4 deletions distro/src/conf/atlas-application.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,8 @@ atlas.graph.index.search.max-result-set-size=150
######### Notification Configs #########
atlas.notification.embedded=true
atlas.kafka.data=${sys:atlas.home}/data/kafka
atlas.kafka.zookeeper.connect=localhost:9026
# Embedded broker uses KRaft (no ZooKeeper). bootstrap.servers is updated at runtime when embedded=true.
atlas.kafka.bootstrap.servers=localhost:9027
atlas.kafka.zookeeper.session.timeout.ms=400
atlas.kafka.zookeeper.connection.timeout.ms=200
atlas.kafka.zookeeper.sync.time.ms=20
atlas.kafka.auto.commit.interval.ms=1000
atlas.kafka.hook.group.id=atlas

Expand Down
57 changes: 49 additions & 8 deletions notification/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,21 +63,49 @@
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>

<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
<classifier>test</classifier>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-metadata</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-server</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-server-common</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-server-common</artifactId>
<version>${kafka.version}</version>
<classifier>test</classifier>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_${kafka.scala.binary.version}</artifactId>
</dependency>

<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_${kafka.scala.binary.version}</artifactId>
<version>${kafka.version}</version>
<classifier>test</classifier>
</dependency>

<!-- KafkaClusterTestKit (embedded KRaft broker) calls org.apache.kafka.test.TestUtils, which uses junit.jupiter at runtime in the WAR. -->
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.11.4</version>
</dependency>

<dependency>
Expand All @@ -94,6 +122,19 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>

</dependencies>

<build>
Expand Down Expand Up @@ -172,7 +213,7 @@
<artifactItem>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.6</version>
<version>${zookeeper.version}</version>
</artifactItem>
</artifactItems>
</configuration>
Expand Down
Loading