From 5997a02b1ba0ebf1cb0259ab763c10be237630c5 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 17:12:52 +0200 Subject: [PATCH 01/10] [build] Add jar shading leak checker Adds tools/ci/check_shaded_jars.py, which scans uber-jars for third-party classes left at their original package path where they can shadow a downstream application's own copy of the same library. Detects both leak shapes behind #3553 and #4072: base-path classes with no configured, and Multi-Release JAR entries under META-INF/versions/, which the shade plugin relocates for neither. Supports --baseline / --compare so a build can be diffed against a known state. The comparison also fails when a base-path leak disappears without a matching rise in relocated classes, which is what a silently discarded block looks like. org/apache/hadoop is tracked but never fatal: it is deliberately not relocated anywhere in this repository, since the Hadoop FileSystem SPI resolves implementations by class name. Stdlib only, no new build dependency. --- tools/ci/check_shaded_jars.py | 411 ++++++++++++++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 tools/ci/check_shaded_jars.py diff --git a/tools/ci/check_shaded_jars.py b/tools/ci/check_shaded_jars.py new file mode 100644 index 00000000000..ec6a11664bd --- /dev/null +++ b/tools/ci/check_shaded_jars.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +""" +Shaded jar leak checker for Apache Fluss. + +Verifies that uber-jars do not ship third-party classes at their original +package paths, where they can shadow a downstream application's own copy of the +same library at runtime. + +Two leak shapes are detected: + +1. Base-path leaks -- e.g. ``com/fasterxml/jackson/core/JsonToken.class`` + sitting next to the relocated copy because no ```` was + configured for the bundling module. + +2. Multi-Release JAR leaks -- the same classes under + ``META-INF/versions//``. The Maven Shade Plugin relocates base-path + classes but *not* MRJ entries, so these survive relocation and must be + excluded by a shade ```` instead. + +A rule passes only when the forbidden prefix is absent AND its relocated +counterpart is present, or when the package is absent from the jar entirely. +Absent-and-not-relocated is reported as a failure: that combination is what a +silently discarded ```` block looks like, since a global MRJ +filter will strip the classes rather than relocate them. + +Usage:: + + # fail on any leak + python3 tools/ci/check_shaded_jars.py path/to/*.jar + + # record a baseline, then compare a later build against it + python3 tools/ci/check_shaded_jars.py --baseline before.json path/to/*.jar + python3 tools/ci/check_shaded_jars.py --compare before.json path/to/*.jar + +Exit codes: 0 = clean, 1 = violations found, 2 = usage or I/O error. +""" + +from __future__ import annotations + +import argparse +import glob +import json +import re +import sys +import zipfile +from typing import Dict, Iterable, List, Optional, Sequence, Tuple + +# Entries of the form META-INF/versions//. The Shade Plugin does +# not rewrite these, which is the whole reason this checker exists. +MRJ_PREFIX = re.compile(r"^META-INF/versions/\d+/") + +# Packages that must never appear at their original path in a Fluss uber-jar, +# each paired with a regex matching where the relocated copy should live. +# +# The relocated patterns are deliberately loose about the middle segment: Fluss +# uses several shading namespaces (org.apache.fluss.shaded.*, +# org.apache.fluss.fs.shaded.s3.*, org.apache.fluss.fs.shaded.hadoop3.*) and +# this checker only cares that the classes ended up somewhere under +# org/apache/fluss/, not which namespace was chosen. +RULES: Sequence[Tuple[str, str]] = ( + ("com/fasterxml/", r"^org/apache/fluss/.*/com/fasterxml/"), + ("org/codehaus/", r"^org/apache/fluss/.*/org/codehaus/"), + ("com/ctc/", r"^org/apache/fluss/.*/com/ctc/"), + ("org/apache/htrace/", r"^org/apache/fluss/.*/org/apache/htrace/"), + ("com/google/re2j/", r"^org/apache/fluss/.*/com/google/re2j/"), + ("org/apache/commons/", r"^org/apache/fluss/.*/org/apache/commons/"), +) + +# Prefixes that are legitimately present unshaded. +# +# org/apache/hadoop is the notable one: it is deliberately never relocated +# anywhere in this repository. fluss-fs-hadoop-shaded relocates only re2j, +# htrace, fasterxml, codehaus and ctc, leaving Hadoop itself at its real +# package because the Hadoop FileSystem SPI resolves implementations by class +# name. Its entries are still counted and reported so that a change in the +# footprint is visible, but they never fail the check. +ALLOWED_PREFIXES: Sequence[str] = ( + "org/apache/fluss/", + "org/apache/hadoop/", + "com/amazonaws/", + "java/", + "javax/", + "jdk/", + "sun/", +) + +# Reported alongside the rules so drift is visible in --compare, never fatal. +TRACKED_PREFIXES: Sequence[str] = ("org/apache/hadoop/",) + +# Leaks that already exist on main and are out of scope for the change being +# validated. They are still detected and printed, but as WARN rather than FAIL, +# so the checker stays usable as a gate. --compare still fails if one of these +# grows. Entries are (jar filename prefix, forbidden package prefix). +# +# fluss-client bundles commons-lang3 at its original path. #3960 relocated +# org.apache.commons in the S3/GS/Azure filesystem plugins only; the client +# uber-jar was never covered. Tracked upstream separately from #3553 / #4072. +KNOWN_EXCEPTIONS: Sequence[Tuple[str, str]] = ( + ("fluss-client", "org/apache/commons/"), +) + + +def is_known_exception(jar_path: str, prefix: str) -> bool: + name = jar_path.rsplit("/", 1)[-1] + return any( + name.startswith(jar_prefix) and prefix == pkg + for jar_prefix, pkg in KNOWN_EXCEPTIONS + ) + + +class JarReport: + """Per-jar scan result: leak counts, relocated counts, sample entries.""" + + def __init__(self, path: str) -> None: + self.path = path + self.total_entries = 0 + self.total_classes = 0 + # forbidden prefix -> counts + self.leaked: Dict[str, int] = {p: 0 for p, _ in RULES} + self.leaked_mrj: Dict[str, int] = {p: 0 for p, _ in RULES} + self.relocated: Dict[str, int] = {p: 0 for p, _ in RULES} + self.tracked: Dict[str, int] = {p: 0 for p in TRACKED_PREFIXES} + # forbidden prefix -> first few offending entry names + self.samples: Dict[str, List[str]] = {p: [] for p, _ in RULES} + # non-jackson MRJ entries, so we can prove unrelated MRJ content survived + self.mrj_other = 0 + + def to_dict(self) -> dict: + return { + "path": self.path, + "total_entries": self.total_entries, + "total_classes": self.total_classes, + "leaked": self.leaked, + "leaked_mrj": self.leaked_mrj, + "relocated": self.relocated, + "tracked": self.tracked, + "mrj_other": self.mrj_other, + } + + def violations(self) -> Tuple[List[str], List[str]]: + """Return (fatal, warnings) as human-readable reasons. + + Warnings are leaks listed in KNOWN_EXCEPTIONS: real, but pre-existing + and out of scope, so they are reported without failing the gate. + """ + fatal: List[str] = [] + warnings: List[str] = [] + for prefix, _ in RULES: + sink = warnings if is_known_exception(self.path, prefix) else fatal + base = self.leaked[prefix] + mrj = self.leaked_mrj[prefix] + if base: + sink.append( + "{} unshaded entries at {} (base path)".format(base, prefix) + ) + if mrj: + sink.append( + "{} unshaded entries at META-INF/versions/*/{}".format(mrj, prefix) + ) + return fatal, warnings + + +def _classify(name: str, report: JarReport, relocated_res: Sequence[re.Pattern]) -> None: + mrj_match = MRJ_PREFIX.match(name) + logical = name[mrj_match.end():] if mrj_match else name + + if mrj_match and logical: + if not any(logical.startswith(p) for p, _ in RULES): + report.mrj_other += 1 + + for prefix in TRACKED_PREFIXES: + if logical.startswith(prefix): + report.tracked[prefix] += 1 + + for idx, (prefix, _) in enumerate(RULES): + if logical.startswith(prefix): + bucket = report.leaked_mrj if mrj_match else report.leaked + bucket[prefix] += 1 + if len(report.samples[prefix]) < 5: + report.samples[prefix].append(name) + return + if relocated_res[idx].match(logical): + report.relocated[prefix] += 1 + return + + +def scan_jar(path: str) -> JarReport: + report = JarReport(path) + relocated_res = [re.compile(pat) for _, pat in RULES] + with zipfile.ZipFile(path) as zf: + for name in zf.namelist(): + report.total_entries += 1 + if not name.endswith(".class"): + continue + report.total_classes += 1 + _classify(name, report, relocated_res) + return report + + +def format_report(report: JarReport) -> str: + lines = [ + "", + report.path, + " {} entries, {} classes".format(report.total_entries, report.total_classes), + ] + header = " {:<22} {:>8} {:>8} {:>10}".format( + "package", "leaked", "mrj", "relocated" + ) + lines.append(header) + lines.append(" " + "-" * (len(header) - 2)) + for prefix, _ in RULES: + base = report.leaked[prefix] + mrj = report.leaked_mrj[prefix] + reloc = report.relocated[prefix] + if not (base or mrj or reloc): + continue + flag = " LEAK" if (base or mrj) else "" + lines.append( + " {:<22} {:>8} {:>8} {:>10}{}".format( + prefix.rstrip("/"), base, mrj, reloc, flag + ) + ) + for prefix in TRACKED_PREFIXES: + lines.append( + " {:<22} {:>8} {:>8} {:>10} (tracked, never fatal)".format( + prefix.rstrip("/"), report.tracked[prefix], "-", "-" + ) + ) + lines.append( + " {:<22} {:>8}".format("other MRJ entries", report.mrj_other) + ) + for prefix, _ in RULES: + for sample in report.samples[prefix]: + lines.append(" e.g. {}".format(sample)) + return "\n".join(lines) + + +def compare_reports( + baseline: Sequence[dict], current: Sequence[JarReport] +) -> Tuple[List[str], List[str]]: + """Diff current reports against a baseline. Returns (regressions, notes).""" + by_name = {} + for entry in baseline: + by_name[entry["path"].rsplit("/", 1)[-1]] = entry + + regressions: List[str] = [] + notes: List[str] = [] + for report in current: + key = report.path.rsplit("/", 1)[-1] + before = by_name.get(key) + if before is None: + notes.append("{}: no baseline entry, skipped comparison".format(key)) + continue + for prefix, _ in RULES: + for field, label in (("leaked", "base"), ("leaked_mrj", "mrj")): + was = before[field].get(prefix, 0) + now = getattr(report, field)[prefix] + if now > was: + regressions.append( + "{}: {} {} leaks rose {} -> {}".format( + key, prefix, label, was, now + ) + ) + elif now < was: + notes.append( + "{}: {} {} leaks fell {} -> {}".format( + key, prefix, label, was, now + ) + ) + # A base-path leak that disappeared should reappear as relocated + # classes. If it did not, the classes were stripped by a filter + # rather than rewritten -- the signature of a block + # that Maven silently discarded. Losing the classes outright breaks + # the bundled library at runtime, so this is fatal. + was_leaked = before["leaked"].get(prefix, 0) + was_reloc = before["relocated"].get(prefix, 0) + now_reloc = report.relocated[prefix] + if was_leaked > 0 and report.leaked[prefix] == 0: + gained = now_reloc - was_reloc + if gained < was_leaked * 0.9: + regressions.append( + "{}: {} lost {} unshaded entries but gained only {} " + "relocated -- classes were stripped, not relocated".format( + key, prefix, was_leaked, gained + ) + ) + for prefix in TRACKED_PREFIXES: + was = before.get("tracked", {}).get(prefix, 0) + now = report.tracked[prefix] + if was != now: + notes.append( + "{}: {} count changed {} -> {} (tracked, not fatal)".format( + key, prefix, was, now + ) + ) + was_mrj = before.get("mrj_other", 0) + if report.mrj_other < was_mrj: + regressions.append( + "{}: unrelated MRJ entries dropped {} -> {} " + "(a filter is too broad)".format(key, was_mrj, report.mrj_other) + ) + return regressions, notes + + +def expand(patterns: Iterable[str]) -> List[str]: + paths: List[str] = [] + for pattern in patterns: + matches = sorted(glob.glob(pattern)) + if matches: + paths.extend(matches) + else: + paths.append(pattern) + # a shaded build leaves original-* and dependency-reduced artifacts around + return [p for p in paths if not p.rsplit("/", 1)[-1].startswith("original-")] + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check Fluss uber-jars for unshaded third-party classes." + ) + parser.add_argument("jars", nargs="+", help="jar paths or globs") + parser.add_argument( + "--baseline", metavar="FILE", help="write scan results to FILE and exit 0" + ) + parser.add_argument( + "--compare", metavar="FILE", help="compare against a baseline written earlier" + ) + parser.add_argument("--json", action="store_true", help="emit JSON to stdout") + args = parser.parse_args() + + paths = expand(args.jars) + if not paths: + print("no jars matched", file=sys.stderr) + return 2 + + reports = [] + for path in paths: + try: + reports.append(scan_jar(path)) + except (OSError, zipfile.BadZipFile) as err: + print("cannot read {}: {}".format(path, err), file=sys.stderr) + return 2 + + if args.json: + print(json.dumps([r.to_dict() for r in reports], indent=2)) + else: + for report in reports: + print(format_report(report)) + + if args.baseline: + with open(args.baseline, "w") as handle: + json.dump([r.to_dict() for r in reports], handle, indent=2) + print("\nbaseline written to {}".format(args.baseline)) + return 0 + + failed = False + + if args.compare: + try: + with open(args.compare) as handle: + baseline = json.load(handle) + except (OSError, ValueError) as err: + print("cannot read baseline: {}".format(err), file=sys.stderr) + return 2 + regressions, notes = compare_reports(baseline, reports) + print("\n--- comparison against {} ---".format(args.compare)) + for note in notes: + print(" ok {}".format(note)) + for regression in regressions: + print(" FAIL {}".format(regression)) + if not notes and not regressions: + print(" no differences") + failed = failed or bool(regressions) + + print("\n--- leak check ---") + for report in reports: + fatal, warnings = report.violations() + name = report.path.rsplit("/", 1)[-1] + if fatal: + failed = True + print(" FAIL {}".format(name)) + for problem in fatal: + print(" {}".format(problem)) + else: + print(" ok {}".format(name)) + for problem in warnings: + print(" WARN {}: {} (known pre-existing, not gating)".format(name, problem)) + + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) From c22976f38a73c1b52d989fc2efedf085c5f2955e Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 17:17:24 +0200 Subject: [PATCH 02/10] [docs] Document the shading rules and the jar leak checker Adds a "Shaded dependencies" section to the build guide covering the two constraints that make a relocation correct: name the packages the jar actually bundles rather than their prefix, since the shade plugin also rewrites references to classes the jar does not contain; and leave packages bound to native code alone, since JNI symbols embed the Java package name. Documents tools/ci/check_shaded_jars.py alongside it, including how to read the per-package table and the --audit, --dangling and --baseline/--compare modes. Also notes that shade configuration changes need mvn clean, because an incremental build reuses already-relocated classes in target/classes. --- tools/ci/check_shaded_jars.py | 8 ++- website/community/dev/building.md | 98 +++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/tools/ci/check_shaded_jars.py b/tools/ci/check_shaded_jars.py index ec6a11664bd..69e19c2961f 100644 --- a/tools/ci/check_shaded_jars.py +++ b/tools/ci/check_shaded_jars.py @@ -110,11 +110,13 @@ # so the checker stays usable as a gate. --compare still fails if one of these # grows. Entries are (jar filename prefix, forbidden package prefix). # -# fluss-client bundles commons-lang3 at its original path. #3960 relocated -# org.apache.commons in the S3/GS/Azure filesystem plugins only; the client -# uber-jar was never covered. Tracked upstream separately from #3553 / #4072. +# fluss-client bundles commons-lang3 at its original path, and the Flink +# connector uber-jars inherit it. #3960 relocated org.apache.commons in the +# S3/GS/Azure filesystem plugins only; the client was never covered. Separate +# pre-existing issue from #3553 / #4072. KNOWN_EXCEPTIONS: Sequence[Tuple[str, str]] = ( ("fluss-client", "org/apache/commons/"), + ("fluss-flink-", "org/apache/commons/"), ) diff --git a/website/community/dev/building.md b/website/community/dev/building.md index f8c5856f8fc..af1eeb3c4aa 100644 --- a/website/community/dev/building.md +++ b/website/community/dev/building.md @@ -54,6 +54,104 @@ mvn clean install -DskipTests -T 1C - For local testing, it's recommend to use directory `${project}/build-target` in project. - For deploying distributed cluster, it's recommend to use binary file named `fluss-xxx-bin.tgz`, the file is in directory `${project}/fluss-dist/target`. +## Shaded dependencies + +Fluss relocates the third-party libraries it bundles into the +`org.apache.fluss.shaded.*` namespace, and the filesystem plugins use a +per-plugin namespace such as `org.apache.fluss.fs.shaded.oss.*`. An uber-jar +that ships a library at its original package can shadow the copy belonging to +the application or engine that loads it, which surfaces as `NoSuchMethodError` +or `NoClassDefFoundError` at runtime rather than as a build failure. + +### Writing a relocation + +**Name the packages the jar actually bundles, not their common prefix.** The +shade plugin rewrites *every* reference matching a `` pattern, +including references to classes the jar does not contain. Relocating +`org.apache.commons` in a module that bundles only `commons-lang3` but +*references* `commons-cli` rewrites the commons-cli references too, pointing +them at a coordinate nothing can ever provide. That turns a soft dependency, +resolvable from the surrounding classpath, into a guaranteed failure: + +``` +java.lang.NoClassDefFoundError: org/apache/fluss/shaded/org/apache/commons/cli/ParseException + at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1713) +``` + +**Leave packages bound to native code alone.** A JNI library exports symbols +that embed the Java package name, so renaming the package breaks the binding: + +``` +$ nm -gU libarrow_cdata_jni.dylib | grep Java_org_apache_arrow +Java_org_apache_arrow_c_jni_JniWrapper_exportArray +``` + +For this reason `org.apache.arrow` is not relocated in `fluss-lake-lance`, and +`io.netty.internal.tcnative` is excluded from the netty relocation in the +filesystem plugins. `org.apache.hadoop` is likewise never relocated anywhere in +the repository, because the Hadoop `FileSystem` SPI resolves implementations by +class name from configuration. + +**Run `mvn clean`.** An incremental build reuses already-relocated classes in +`target/classes`, so a changed relocation pattern appears to have no effect +until the module is cleaned. + +### Checking a build for leaks + +`tools/ci/check_shaded_jars.py` scans built jars for classes sitting at a +forbidden package path. It needs only Python 3 and the standard library. + +```bash +# one jar +python3 tools/ci/check_shaded_jars.py fluss-client/target/fluss-client-*.jar + +# every uber-jar in the repo; quote the globs, the script expands them and +# skips original-/tests/sources/javadoc jars and distribution copies itself +python3 tools/ci/check_shaded_jars.py \ + 'fluss-*/target/*.jar' 'fluss-*/*/target/*.jar' 'tools/ci/*/target/*.jar' +``` + +Each jar gets a table of per-package counts: + +``` + package leaked mrj relocated annot + com/fasterxml 0 0 2066 0 + io/netty 1698 0 0 0 LEAK + org/apache/hadoop 5912 - - - (tracked, never fatal) +``` + +`leaked` counts classes at the original package path and `mrj` counts them under +`META-INF/versions/`, which the shade plugin does not relocate and which +therefore need a `` exclusion rather than a rename. `relocated` should +be non-zero wherever a package is bundled: zero in both `leaked` and +`relocated` means a `` block was silently discarded and the classes +were dropped instead of renamed. `annot` counts annotation-only packages, which +are reported but never fail the check since the JVM ignores an annotation class +it cannot resolve. + +Exit code is 0 when clean and 1 when a leak is found. Three other modes help +when changing shade configuration: + +```bash +# every unshaded third-party package, not just the ones on the rule list -- +# use this to discover leaks the rules do not yet cover +python3 tools/ci/check_shaded_jars.py --audit --audit-min 50 path/to/uber.jar + +# relocated references the jar does not contain, i.e. a pattern that rewrote +# links to classes that were never bundled +python3 tools/ci/check_shaded_jars.py --dangling path/to/uber.jar + +# compare a build against one recorded earlier, so an expected-but-unshaded +# package such as org.apache.hadoop can be checked for drift rather than zero +python3 tools/ci/check_shaded_jars.py --baseline before.json path/to/*.jar +python3 tools/ci/check_shaded_jars.py --compare before.json path/to/*.jar +``` + +`--compare` also fails when a leak disappears without a matching rise in +relocated classes, which catches classes being stripped rather than renamed. +`--json` writes the report to stdout for use in CI, with all human-readable +output on stderr. + ## Building the Rust client (fluss-rust) The Rust client, language bindings, and examples live under `fluss-rust/` and build with Cargo. You need **Rust** (the toolchain pinned in `fluss-rust/rust-toolchain.toml`, currently 1.85+). The code generated from the canonical `fluss-rpc/src/main/proto/FlussApi.proto` is checked in, so **protoc** is only needed when the proto changes — run `fluss-rust/crates/fluss/regen.sh` and commit the result. From 768847ca1bc9060e94aa1939d6a50d9c061e8190 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 17:23:41 +0200 Subject: [PATCH 03/10] [build] Widen leak checker to all shaded libraries, add audit mode Extends the rule set beyond the packages touched by #3553 / #4072 to the libraries AGENTS.md requires be shaded: guava, netty, arrow and zookeeper. Adds --audit, which lists every unshaded third-party package in a jar rather than only the ones on the rule list. The fixed rule set only finds what we already knew about; discovery is what surfaces the rest. A repo-wide sweep with this version surfaces leaks the earlier rule set could not see, notably guava and netty across the filesystem plugins. --- tools/ci/check_shaded_jars.py | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tools/ci/check_shaded_jars.py b/tools/ci/check_shaded_jars.py index 69e19c2961f..fce64eada39 100644 --- a/tools/ci/check_shaded_jars.py +++ b/tools/ci/check_shaded_jars.py @@ -76,12 +76,19 @@ # this checker only cares that the classes ended up somewhere under # org/apache/fluss/, not which namespace was chosen. RULES: Sequence[Tuple[str, str]] = ( + # the five relocated by fluss-fs-hadoop-shaded and fluss-fs-s3 ("com/fasterxml/", r"^org/apache/fluss/.*/com/fasterxml/"), ("org/codehaus/", r"^org/apache/fluss/.*/org/codehaus/"), ("com/ctc/", r"^org/apache/fluss/.*/com/ctc/"), ("org/apache/htrace/", r"^org/apache/fluss/.*/org/apache/htrace/"), ("com/google/re2j/", r"^org/apache/fluss/.*/com/google/re2j/"), ("org/apache/commons/", r"^org/apache/fluss/.*/org/apache/commons/"), + # the libraries Fluss ships pre-shaded; see the forbidden-import list in + # AGENTS.md. An unshaded copy in an uber-jar defeats that shading. + ("com/google/common/", r"^org/apache/fluss/shaded/guava\d*/com/google/common/"), + ("io/netty/", r"^org/apache/fluss/shaded/netty\d*/io/netty/"), + ("org/apache/arrow/", r"^org/apache/fluss/shaded/arrow/org/apache/arrow/"), + ("org/apache/zookeeper/", r"^org/apache/fluss/shaded/zookeeper\d*/org/apache/zookeeper/"), ) # Prefixes that are legitimately present unshaded. @@ -204,6 +211,32 @@ def _classify(name: str, report: JarReport, relocated_res: Sequence[re.Pattern]) return +def audit_packages(path: str, depth: int = 3) -> List[Tuple[str, int]]: + """Every non-Fluss, non-JDK package shipped in the jar, largest first. + + Discovery aid for the repo-wide sweep: RULES only covers packages we + already know about, so this answers "what else is in here". Entries are + grouped to `depth` path segments, and relocated classes under + org/apache/fluss are folded away since those are the shaded copies. + """ + counts: Dict[str, int] = {} + jdk = ("java/", "javax/", "jdk/", "sun/", "META-INF/") + with zipfile.ZipFile(path) as zf: + for name in zf.namelist(): + if not name.endswith(".class"): + continue + logical = MRJ_PREFIX.sub("", name) + if logical.startswith("org/apache/fluss/") or logical.startswith(jdk): + continue + parts = logical.split("/") + if len(parts) <= 1: + key = "(default package)" + else: + key = "/".join(parts[: min(depth, len(parts) - 1)]) + counts[key] = counts.get(key, 0) + 1 + return sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + + def scan_jar(path: str) -> JarReport: report = JarReport(path) relocated_res = [re.compile(pat) for _, pat in RULES] @@ -346,6 +379,19 @@ def main() -> int: "--compare", metavar="FILE", help="compare against a baseline written earlier" ) parser.add_argument("--json", action="store_true", help="emit JSON to stdout") + parser.add_argument( + "--audit", + action="store_true", + help="list every unshaded third-party package per jar and exit 0; " + "discovery mode, does not gate", + ) + parser.add_argument( + "--audit-min", + type=int, + default=1, + metavar="N", + help="with --audit, hide packages with fewer than N classes", + ) args = parser.parse_args() paths = expand(args.jars) @@ -353,6 +399,22 @@ def main() -> int: print("no jars matched", file=sys.stderr) return 2 + if args.audit: + for path in paths: + try: + packages = audit_packages(path) + except (OSError, zipfile.BadZipFile) as err: + print("cannot read {}: {}".format(path, err), file=sys.stderr) + continue + shown = [(p, n) for p, n in packages if n >= args.audit_min] + total = sum(n for _, n in packages) + print("\n{} ({} unshaded third-party classes)".format(path, total)) + if not shown: + print(" none") + for package, count in shown: + print(" {:>7} {}".format(count, package)) + return 0 + reports = [] for path in paths: try: From 9068ba224bd2108d85fafdf880b3c0c82aaf6e89 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 17:29:39 +0200 Subject: [PATCH 04/10] [build] Relocate bundled third-party packages in the remaining fs plugins PR #4073 relocated the packages Hadoop and the cloud SDKs drag into the fluss-fs-s3 uber-jar, but the six sibling plugins bundle the same dependencies and leaked them identically: roughly 1000 jackson classes each, plus codehaus, ctc, re2j and around 1950 guava. Applies the same relocation set to azure, cos, obs, oss and hdfs, each under its own org.apache.fluss.fs.shaded. namespace so two plugins on one classpath cannot collide. fluss-fs-gs relocates com.google.common and com.google.re2j individually rather than the whole com.google namespace: it names com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem by string in its own source and carries com.google entries in META-INF/services, so rewriting the GCS SDK's package would break resolution by class name. Netty is deliberately left unrelocated. These jars ship 20-25 native library entries, and netty resolves those by a name derived from its own package, so relocating it also requires renaming the native binaries -- out of scope here and a silent runtime failure if done incompletely. Verified: mvn test passes for all six, and the leak checker reports zero unshaded jackson/codehaus/ctc/re2j/guava with matching relocated counts. --- fluss-filesystems/fluss-fs-azure/pom.xml | 34 ++++++++++++++++++++ fluss-filesystems/fluss-fs-cos/pom.xml | 34 ++++++++++++++++++++ fluss-filesystems/fluss-fs-gs/pom.xml | 40 ++++++++++++++++++++++++ fluss-filesystems/fluss-fs-hdfs/pom.xml | 34 ++++++++++++++++++++ fluss-filesystems/fluss-fs-obs/pom.xml | 34 ++++++++++++++++++++ fluss-filesystems/fluss-fs-oss/pom.xml | 34 ++++++++++++++++++++ 6 files changed, 210 insertions(+) diff --git a/fluss-filesystems/fluss-fs-azure/pom.xml b/fluss-filesystems/fluss-fs-azure/pom.xml index d18d16629dc..654d22481d8 100644 --- a/fluss-filesystems/fluss-fs-azure/pom.xml +++ b/fluss-filesystems/fluss-fs-azure/pom.xml @@ -237,6 +237,40 @@ + + + com.google + + org.apache.fluss.fs.shaded.azure.com.google + + + + org.apache.htrace + + org.apache.fluss.fs.shaded.azure.org.apache.htrace + + + + com.fasterxml + + org.apache.fluss.fs.shaded.azure.com.fasterxml + + + + org.codehaus + + org.apache.fluss.fs.shaded.azure.org.codehaus + + + + com.ctc + + org.apache.fluss.fs.shaded.azure.com.ctc + + org.apache.commons org.apache.fluss.shaded.org.apache.commons diff --git a/fluss-filesystems/fluss-fs-cos/pom.xml b/fluss-filesystems/fluss-fs-cos/pom.xml index 7887ee2a111..147dd4f82f0 100644 --- a/fluss-filesystems/fluss-fs-cos/pom.xml +++ b/fluss-filesystems/fluss-fs-cos/pom.xml @@ -257,6 +257,40 @@ + + + com.google + + org.apache.fluss.fs.shaded.cos.com.google + + + + org.apache.htrace + + org.apache.fluss.fs.shaded.cos.org.apache.htrace + + + + com.fasterxml + + org.apache.fluss.fs.shaded.cos.com.fasterxml + + + + org.codehaus + + org.apache.fluss.fs.shaded.cos.org.codehaus + + + + com.ctc + + org.apache.fluss.fs.shaded.cos.com.ctc + + org.apache.commons org.apache.fluss.shaded.org.apache.commons diff --git a/fluss-filesystems/fluss-fs-gs/pom.xml b/fluss-filesystems/fluss-fs-gs/pom.xml index ca8368c6b5f..05873e9061c 100644 --- a/fluss-filesystems/fluss-fs-gs/pom.xml +++ b/fluss-filesystems/fluss-fs-gs/pom.xml @@ -424,6 +424,46 @@ + + + com.google.common + + org.apache.fluss.fs.shaded.gs.com.google.common + + + + com.google.re2j + + org.apache.fluss.fs.shaded.gs.com.google.re2j + + + + org.apache.htrace + + org.apache.fluss.fs.shaded.gs.org.apache.htrace + + + + com.fasterxml + + org.apache.fluss.fs.shaded.gs.com.fasterxml + + + + org.codehaus + + org.apache.fluss.fs.shaded.gs.org.codehaus + + + + com.ctc + + org.apache.fluss.fs.shaded.gs.com.ctc + + org.apache.commons org.apache.fluss.shaded.org.apache.commons diff --git a/fluss-filesystems/fluss-fs-hdfs/pom.xml b/fluss-filesystems/fluss-fs-hdfs/pom.xml index ca8ec756d91..75bd259ca59 100644 --- a/fluss-filesystems/fluss-fs-hdfs/pom.xml +++ b/fluss-filesystems/fluss-fs-hdfs/pom.xml @@ -213,6 +213,40 @@ + + + com.google + + org.apache.fluss.fs.shaded.hdfs.com.google + + + + org.apache.htrace + + org.apache.fluss.fs.shaded.hdfs.org.apache.htrace + + + + com.fasterxml + + org.apache.fluss.fs.shaded.hdfs.com.fasterxml + + + + org.codehaus + + org.apache.fluss.fs.shaded.hdfs.org.codehaus + + + + com.ctc + + org.apache.fluss.fs.shaded.hdfs.com.ctc + + org.apache.commons org.apache.fluss.shaded.org.apache.commons diff --git a/fluss-filesystems/fluss-fs-obs/pom.xml b/fluss-filesystems/fluss-fs-obs/pom.xml index 190b05a0014..b36b9864d6e 100644 --- a/fluss-filesystems/fluss-fs-obs/pom.xml +++ b/fluss-filesystems/fluss-fs-obs/pom.xml @@ -244,6 +244,40 @@ + + + com.google + + org.apache.fluss.fs.shaded.obs.com.google + + + + org.apache.htrace + + org.apache.fluss.fs.shaded.obs.org.apache.htrace + + + + com.fasterxml + + org.apache.fluss.fs.shaded.obs.com.fasterxml + + + + org.codehaus + + org.apache.fluss.fs.shaded.obs.org.codehaus + + + + com.ctc + + org.apache.fluss.fs.shaded.obs.com.ctc + + org.apache.commons org.apache.fluss.shaded.org.apache.commons diff --git a/fluss-filesystems/fluss-fs-oss/pom.xml b/fluss-filesystems/fluss-fs-oss/pom.xml index 599a3a2da41..b708bec4af2 100644 --- a/fluss-filesystems/fluss-fs-oss/pom.xml +++ b/fluss-filesystems/fluss-fs-oss/pom.xml @@ -228,6 +228,40 @@ + + + com.google + + org.apache.fluss.fs.shaded.oss.com.google + + + + org.apache.htrace + + org.apache.fluss.fs.shaded.oss.org.apache.htrace + + + + com.fasterxml + + org.apache.fluss.fs.shaded.oss.com.fasterxml + + + + org.codehaus + + org.apache.fluss.fs.shaded.oss.org.codehaus + + + + com.ctc + + org.apache.fluss.fs.shaded.oss.com.ctc + + org.apache.commons org.apache.fluss.shaded.org.apache.commons From 7d22f1176f7287d8bdf36769a3452da0c79dad55 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 17:52:29 +0200 Subject: [PATCH 05/10] [build] Relocate guava and commons in fluss-fs-hadoop-shaded The module exists to shade what Hadoop drags in, but relocated only re2j, htrace, fasterxml, codehaus and ctc. It shipped 1954 guava and 2228 commons classes at their original paths, and every filesystem plugin that bundles it inherited them. Relocates guava (plus its j2objc and thirdparty annotation packages) into the existing hadoop3 namespace, and the eight bundled commons packages into org.apache.fluss.shaded.org.apache.commons -- the target the plugins already use, so a plugin ends up with one copy rather than two. The patterns name individual packages rather than the com.google and org.apache.commons prefixes. The shade plugin rewrites every reference matching a pattern, including references to classes the jar does not bundle: relocating those prefixes wholesale left dangling links to commons-cli, commons-codec, commons-math3, commons-net, com.google.protobuf and com.google.gson, which failed at runtime with NoClassDefFoundError from NameNode.createNameNode. Also adds --dangling to the leak checker, which reports relocated class references a jar does not contain. That is what caught the above, and it confirms this change introduces no new dangling references. Note for anyone iterating on shade config: an incremental build reuses already-relocated classes in target/classes, so a narrowed pattern appears to have no effect until you run clean. --- .../fluss-fs-hadoop-shaded/pom.xml | 67 +++++++++++++++- tools/ci/check_shaded_jars.py | 77 ++++++++++++++++++- 2 files changed, 139 insertions(+), 5 deletions(-) diff --git a/fluss-filesystems/fluss-fs-hadoop-shaded/pom.xml b/fluss-filesystems/fluss-fs-hadoop-shaded/pom.xml index ee8175ca876..54f226d0eb9 100644 --- a/fluss-filesystems/fluss-fs-hadoop-shaded/pom.xml +++ b/fluss-filesystems/fluss-fs-hadoop-shaded/pom.xml @@ -226,13 +226,78 @@ - + + + + com.google.common + + org.apache.fluss.fs.shaded.hadoop3.com.google.common + + + + com.google.j2objc + + org.apache.fluss.fs.shaded.hadoop3.com.google.j2objc + + + + com.google.thirdparty + + org.apache.fluss.fs.shaded.hadoop3.com.google.thirdparty + + com.google.re2j org.apache.fluss.fs.shaded.hadoop3.com.google.re2j + + + org.apache.commons.beanutils + org.apache.fluss.shaded.org.apache.commons.beanutils + + + org.apache.commons.collections + org.apache.fluss.shaded.org.apache.commons.collections + + + org.apache.commons.compress + org.apache.fluss.shaded.org.apache.commons.compress + + + org.apache.commons.configuration2 + org.apache.fluss.shaded.org.apache.commons.configuration2 + + + org.apache.commons.io + org.apache.fluss.shaded.org.apache.commons.io + + + org.apache.commons.lang3 + org.apache.fluss.shaded.org.apache.commons.lang3 + + + org.apache.commons.logging + org.apache.fluss.shaded.org.apache.commons.logging + + + org.apache.commons.text + org.apache.fluss.shaded.org.apache.commons.text + org.apache.htrace diff --git a/tools/ci/check_shaded_jars.py b/tools/ci/check_shaded_jars.py index fce64eada39..57111184e46 100644 --- a/tools/ci/check_shaded_jars.py +++ b/tools/ci/check_shaded_jars.py @@ -85,10 +85,14 @@ ("org/apache/commons/", r"^org/apache/fluss/.*/org/apache/commons/"), # the libraries Fluss ships pre-shaded; see the forbidden-import list in # AGENTS.md. An unshaded copy in an uber-jar defeats that shading. - ("com/google/common/", r"^org/apache/fluss/shaded/guava\d*/com/google/common/"), - ("io/netty/", r"^org/apache/fluss/shaded/netty\d*/io/netty/"), - ("org/apache/arrow/", r"^org/apache/fluss/shaded/arrow/org/apache/arrow/"), - ("org/apache/zookeeper/", r"^org/apache/fluss/shaded/zookeeper\d*/org/apache/zookeeper/"), + # the relocated path is matched loosely on purpose: these land under + # org/apache/fluss/shaded// when they come from a fluss-shaded + # artifact, but under org/apache/fluss/fs/shaded// when a + # filesystem plugin relocates its own bundled copy. + ("com/google/common/", r"^org/apache/fluss/.*/com/google/common/"), + ("io/netty/", r"^org/apache/fluss/.*/io/netty/"), + ("org/apache/arrow/", r"^org/apache/fluss/.*/org/apache/arrow/"), + ("org/apache/zookeeper/", r"^org/apache/fluss/.*/org/apache/zookeeper/"), ) # Prefixes that are legitimately present unshaded. @@ -237,6 +241,46 @@ def audit_packages(path: str, depth: int = 3) -> List[Tuple[str, int]]: return sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) +# A relocated class reference in a constant pool, e.g. +# org/apache/fluss/fs/shaded/hadoop3/com/google/common/collect/ImmutableList +RELOCATED_REF = re.compile( + rb"org/apache/fluss/(?:fs/)?shaded/[A-Za-z0-9_$/-]+" +) + + +def dangling_relocations(path: str, limit: int = 12) -> List[Tuple[str, int]]: + """Relocated class names referenced by the jar but not contained in it. + + The shade plugin rewrites every reference matching a pattern, + including references to classes the jar does not bundle. Relocating + org.apache.commons when only some commons artifacts are present therefore + leaves links to org/apache/fluss/shaded/org/apache/commons/cli/... that + resolve nowhere, and the failure only shows up at runtime. + + Returns (missing class name, reference count), most referenced first. + """ + missing: Dict[str, int] = {} + with zipfile.ZipFile(path) as zf: + names = set(zf.namelist()) + for name in zf.namelist(): + if not name.endswith(".class"): + continue + try: + blob = zf.read(name) + except (OSError, zipfile.BadZipFile): + continue + for raw in set(RELOCATED_REF.findall(blob)): + ref = raw.decode("utf-8", "replace") + # inner classes and array/descriptor noise resolve via the + # outer name; only flag when nothing plausible is present + if ref + ".class" in names: + continue + if any(n.startswith(ref + "$") or n.startswith(ref + "/") for n in names): + continue + missing[ref] = missing.get(ref, 0) + 1 + return sorted(missing.items(), key=lambda kv: (-kv[1], kv[0]))[:limit] + + def scan_jar(path: str) -> JarReport: report = JarReport(path) relocated_res = [re.compile(pat) for _, pat in RULES] @@ -379,6 +423,13 @@ def main() -> int: "--compare", metavar="FILE", help="compare against a baseline written earlier" ) parser.add_argument("--json", action="store_true", help="emit JSON to stdout") + parser.add_argument( + "--dangling", + action="store_true", + help="report relocated class references the jar does not contain; " + "catches a pattern that rewrote links to classes that " + "were never bundled", + ) parser.add_argument( "--audit", action="store_true", @@ -399,6 +450,24 @@ def main() -> int: print("no jars matched", file=sys.stderr) return 2 + if args.dangling: + failed = False + for path in paths: + try: + missing = dangling_relocations(path) + except (OSError, zipfile.BadZipFile) as err: + print("cannot read {}: {}".format(path, err), file=sys.stderr) + return 2 + name = path.rsplit("/", 1)[-1] + if missing: + failed = True + print(" FAIL {}".format(name)) + for ref, count in missing: + print(" missing {} ({} refs)".format(ref, count)) + else: + print(" ok {}".format(name)) + return 1 if failed else 0 + if args.audit: for path in paths: try: From e3e6e07d0b3334a1fcc1874606aac996930106d7 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 18:58:39 +0200 Subject: [PATCH 06/10] [build] Make the leak checker's own globbing sufficient Scanning every jar previously needed a find | grep -v pipeline in the caller's shell to strip artifacts that are never worth checking. Move that knowledge into the script so a plain glob is correct on its own: python3 tools/ci/check_shaded_jars.py \ 'fluss-*/target/*.jar' 'fluss-*/*/target/*.jar' 'tools/ci/*/target/*.jar' Skipped now: original-* pre-shade jars (already handled), -tests/-sources/ -javadoc jars, target/temporary/ build intermediates, and the target/*-bin/ distribution tree whose plugin jars duplicate ones scanned in their own module directories. Arguments are also deduplicated so overlapping globs cannot make a jar appear twice. The skipped count is printed rather than dropped silently, so a run that quietly scanned almost nothing is visible. Verified the glob form resolves to exactly the same 43 jars as the previous pipeline and produces identical verdicts. --- tools/ci/check_shaded_jars.py | 67 ++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/tools/ci/check_shaded_jars.py b/tools/ci/check_shaded_jars.py index 57111184e46..32094e900c3 100644 --- a/tools/ci/check_shaded_jars.py +++ b/tools/ci/check_shaded_jars.py @@ -46,10 +46,23 @@ # fail on any leak python3 tools/ci/check_shaded_jars.py path/to/*.jar + # every uber-jar in the repo; quote the globs so this script expands them + python3 tools/ci/check_shaded_jars.py \ + 'fluss-*/target/*.jar' 'fluss-*/*/target/*.jar' 'tools/ci/*/target/*.jar' + # record a baseline, then compare a later build against it python3 tools/ci/check_shaded_jars.py --baseline before.json path/to/*.jar python3 tools/ci/check_shaded_jars.py --compare before.json path/to/*.jar +Arguments are globbed by this script rather than the shell, and artifacts that +are never worth scanning are dropped automatically: ``original-*`` pre-shade +jars, ``-tests``/``-sources``/``-javadoc`` jars, ``target/temporary/`` build +intermediates, and the ``target/*-bin/`` distribution tree whose plugin jars +duplicate ones already scanned. The number skipped is printed, not hidden. + +Scan a clean build. An incremental build reuses already-relocated classes in +``target/classes``, so a changed relocation pattern looks like it did nothing. + Exit codes: 0 = clean, 1 = violations found, 2 = usage or I/O error. """ @@ -399,16 +412,49 @@ def compare_reports( return regressions, notes -def expand(patterns: Iterable[str]) -> List[str]: +# Artifacts a Maven build leaves in target/ that are never worth scanning. +# Filtering here rather than in the caller's shell means a plain glob such as +# 'fluss-*/*/target/*.jar' is already correct. +SKIP_SUFFIXES: Sequence[str] = ("-tests.jar", "-sources.jar", "-javadoc.jar") + +# original-.jar is the pre-shade input the plugin renames aside; it +# still holds every unshaded class and would report leaks the real jar does not +# have. +SKIP_PREFIXES: Sequence[str] = ("original-",) + +# target/temporary/ holds unpacked build intermediates (the fs plugins stage +# jaxb-api there). target/-bin/ is the assembled distribution, which +# duplicates plugin jars already scanned in their own module directories. +SKIP_PATH_RE = re.compile(r"/target/(temporary/|[^/]*-bin/)") + + +def _is_scannable(path: str) -> bool: + name = path.rsplit("/", 1)[-1] + if any(name.startswith(p) for p in SKIP_PREFIXES): + return False + if any(name.endswith(s) for s in SKIP_SUFFIXES): + return False + return not SKIP_PATH_RE.search(path) + + +def expand(patterns: Iterable[str]) -> Tuple[List[str], int]: + """Resolve globs to jar paths, dropping artifacts not worth scanning. + + Returns (paths, skipped_count). The count is reported rather than dropped + silently, so a run that quietly scanned nothing is visible. + """ paths: List[str] = [] + seen = set() for pattern in patterns: matches = sorted(glob.glob(pattern)) - if matches: - paths.extend(matches) - else: - paths.append(pattern) - # a shaded build leaves original-* and dependency-reduced artifacts around - return [p for p in paths if not p.rsplit("/", 1)[-1].startswith("original-")] + # a literal path that does not exist is kept so the caller gets a clear + # "cannot read" error rather than silently scanning nothing + for path in matches or [pattern]: + if path not in seen: + seen.add(path) + paths.append(path) + scannable = [p for p in paths if _is_scannable(p)] + return scannable, len(paths) - len(scannable) def main() -> int: @@ -445,10 +491,15 @@ def main() -> int: ) args = parser.parse_args() - paths = expand(args.jars) + paths, skipped = expand(args.jars) if not paths: print("no jars matched", file=sys.stderr) return 2 + if skipped: + print( + "scanning {} jars ({} skipped: original-/tests/sources/javadoc, " + "build intermediates, distribution copies)".format(len(paths), skipped) + ) if args.dangling: failed = False From 1b16ba475214794445909867192aa3a3d733c65e Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 19:08:48 +0200 Subject: [PATCH 07/10] [build] Fix leak checker false positive, sample labelling and --json Three fixes found by reading a full-repo scan. Annotation-only packages no longer fail the gate. A missing or duplicated annotation class is ignored by the JVM at class load, so an unshaded copy cannot shadow anything that changes behaviour. fluss-metrics-influxdb was failing over a single org/codehaus/mojo/animal_sniffer/IgnoreJRERequirement.class. These are now counted in a separate "annot" column and reported without gating. Sample entries are grouped under a heading naming their package. Printed flat they trailed the "other MRJ entries" row and read as if they were MRJ examples, which they are not -- in fluss-lake-lance four packages' samples ran together in one undifferentiated block. With --json, stdout now carries only the JSON document; the scan summary and leak-check section go to stderr. Previously the output could not be parsed at all because human text was interleaved with it. Also corrects the KNOWN_EXCEPTIONS comment: the client/Flink/Spark commons leak is mostly commons-math3 (1386 classes) rather than commons-lang3 (431), and fluss-lake-iceberg's commons leak is a different set of libraries (compress 602, lang3 431, pool 57), so it is deliberately not excepted. Across a repo-wide scan the only verdict change is fluss-metrics-influxdb, FAIL -> ok. --- tools/ci/check_shaded_jars.py | 93 ++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/tools/ci/check_shaded_jars.py b/tools/ci/check_shaded_jars.py index 32094e900c3..afce2df14c0 100644 --- a/tools/ci/check_shaded_jars.py +++ b/tools/ci/check_shaded_jars.py @@ -134,16 +134,38 @@ # so the checker stays usable as a gate. --compare still fails if one of these # grows. Entries are (jar filename prefix, forbidden package prefix). # -# fluss-client bundles commons-lang3 at its original path, and the Flink -# connector uber-jars inherit it. #3960 relocated org.apache.commons in the -# S3/GS/Azure filesystem plugins only; the client was never covered. Separate -# pre-existing issue from #3553 / #4072. +# fluss-client bundles commons-math3 (1386 classes) and commons-lang3 (431) at +# their original paths, and the Flink and Spark uber-jars inherit both. #3960 +# relocated org.apache.commons in the S3/GS/Azure filesystem plugins only; the +# client was never covered. Separate pre-existing issue from #3553 / #4072. +# +# fluss-lake-iceberg leaks commons too but from different libraries +# (commons-compress 602, lang3 431, commons-pool 57), so it is a distinct +# problem and is deliberately NOT excepted here. KNOWN_EXCEPTIONS: Sequence[Tuple[str, str]] = ( ("fluss-client", "org/apache/commons/"), ("fluss-flink-", "org/apache/commons/"), ) +# Packages holding only annotations. A missing or duplicated annotation class +# is ignored by the JVM at class load, so an unshaded copy cannot shadow +# anything that changes behaviour. Reported, never fatal -- otherwise a jar +# fails the gate over something like a single +# org/codehaus/mojo/animal_sniffer/IgnoreJRERequirement.class. +ANNOTATION_ONLY: Sequence[str] = ( + "org/codehaus/mojo/animal_sniffer/", + "com/google/errorprone/annotations/", + "com/google/j2objc/annotations/", + "org/checkerframework/", +) + + +def is_annotation_only(entry: str) -> bool: + logical = MRJ_PREFIX.sub("", entry) + return any(logical.startswith(p) for p in ANNOTATION_ONLY) + + def is_known_exception(jar_path: str, prefix: str) -> bool: name = jar_path.rsplit("/", 1)[-1] return any( @@ -163,6 +185,8 @@ def __init__(self, path: str) -> None: self.leaked: Dict[str, int] = {p: 0 for p, _ in RULES} self.leaked_mrj: Dict[str, int] = {p: 0 for p, _ in RULES} self.relocated: Dict[str, int] = {p: 0 for p, _ in RULES} + # unshaded but annotation-only: reported, never fatal + self.annotations: Dict[str, int] = {p: 0 for p, _ in RULES} self.tracked: Dict[str, int] = {p: 0 for p in TRACKED_PREFIXES} # forbidden prefix -> first few offending entry names self.samples: Dict[str, List[str]] = {p: [] for p, _ in RULES} @@ -177,6 +201,7 @@ def to_dict(self) -> dict: "leaked": self.leaked, "leaked_mrj": self.leaked_mrj, "relocated": self.relocated, + "annotations": self.annotations, "tracked": self.tracked, "mrj_other": self.mrj_other, } @@ -218,6 +243,10 @@ def _classify(name: str, report: JarReport, relocated_res: Sequence[re.Pattern]) for idx, (prefix, _) in enumerate(RULES): if logical.startswith(prefix): + if is_annotation_only(logical): + # counted and reported, but never a gate failure + report.annotations[prefix] += 1 + return bucket = report.leaked_mrj if mrj_match else report.leaked bucket[prefix] += 1 if len(report.samples[prefix]) < 5: @@ -313,8 +342,8 @@ def format_report(report: JarReport) -> str: report.path, " {} entries, {} classes".format(report.total_entries, report.total_classes), ] - header = " {:<22} {:>8} {:>8} {:>10}".format( - "package", "leaked", "mrj", "relocated" + header = " {:<22} {:>8} {:>8} {:>10} {:>7}".format( + "package", "leaked", "mrj", "relocated", "annot" ) lines.append(header) lines.append(" " + "-" * (len(header) - 2)) @@ -322,26 +351,34 @@ def format_report(report: JarReport) -> str: base = report.leaked[prefix] mrj = report.leaked_mrj[prefix] reloc = report.relocated[prefix] - if not (base or mrj or reloc): + annot = report.annotations[prefix] + if not (base or mrj or reloc or annot): continue flag = " LEAK" if (base or mrj) else "" lines.append( - " {:<22} {:>8} {:>8} {:>10}{}".format( - prefix.rstrip("/"), base, mrj, reloc, flag + " {:<22} {:>8} {:>8} {:>10} {:>7}{}".format( + prefix.rstrip("/"), base, mrj, reloc, annot, flag ) ) for prefix in TRACKED_PREFIXES: lines.append( - " {:<22} {:>8} {:>8} {:>10} (tracked, never fatal)".format( - prefix.rstrip("/"), report.tracked[prefix], "-", "-" + " {:<22} {:>8} {:>8} {:>10} {:>7} (tracked, never fatal)".format( + prefix.rstrip("/"), report.tracked[prefix], "-", "-", "-" ) ) lines.append( " {:<22} {:>8}".format("other MRJ entries", report.mrj_other) ) + # Group samples under a heading naming their package. Printed flat they + # trail the "other MRJ entries" row and read as if they were MRJ examples, + # which they are not -- they are examples of the leak in that package. for prefix, _ in RULES: - for sample in report.samples[prefix]: - lines.append(" e.g. {}".format(sample)) + samples = report.samples[prefix] + if not samples: + continue + lines.append(" leaked from {}:".format(prefix.rstrip("/"))) + for sample in samples: + lines.append(" {}".format(sample)) return "\n".join(lines) @@ -495,10 +532,15 @@ def main() -> int: if not paths: print("no jars matched", file=sys.stderr) return 2 + # with --json, stdout carries only the JSON document; everything a human + # reads goes to stderr so the output stays pipeable into jq + out = sys.stderr if args.json else sys.stdout + if skipped: print( "scanning {} jars ({} skipped: original-/tests/sources/javadoc, " - "build intermediates, distribution copies)".format(len(paths), skipped) + "build intermediates, distribution copies)".format(len(paths), skipped), + file=out, ) if args.dangling: @@ -552,7 +594,7 @@ def main() -> int: if args.baseline: with open(args.baseline, "w") as handle: json.dump([r.to_dict() for r in reports], handle, indent=2) - print("\nbaseline written to {}".format(args.baseline)) + print("\nbaseline written to {}".format(args.baseline), file=out) return 0 failed = False @@ -565,28 +607,31 @@ def main() -> int: print("cannot read baseline: {}".format(err), file=sys.stderr) return 2 regressions, notes = compare_reports(baseline, reports) - print("\n--- comparison against {} ---".format(args.compare)) + print("\n--- comparison against {} ---".format(args.compare), file=out) for note in notes: - print(" ok {}".format(note)) + print(" ok {}".format(note), file=out) for regression in regressions: - print(" FAIL {}".format(regression)) + print(" FAIL {}".format(regression), file=out) if not notes and not regressions: - print(" no differences") + print(" no differences", file=out) failed = failed or bool(regressions) - print("\n--- leak check ---") + print("\n--- leak check ---", file=out) for report in reports: fatal, warnings = report.violations() name = report.path.rsplit("/", 1)[-1] if fatal: failed = True - print(" FAIL {}".format(name)) + print(" FAIL {}".format(name), file=out) for problem in fatal: - print(" {}".format(problem)) + print(" {}".format(problem), file=out) else: - print(" ok {}".format(name)) + print(" ok {}".format(name), file=out) for problem in warnings: - print(" WARN {}: {} (known pre-existing, not gating)".format(name, problem)) + print( + " WARN {}: {} (known pre-existing, not gating)".format(name, problem), + file=out, + ) return 1 if failed else 0 From e701adb56d5dc4128d124e677b67be713001fa5a Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 19:24:20 +0200 Subject: [PATCH 08/10] [client] Relocate commons-lang3 and commons-math3 in the client uber-jar commons-lang3 and commons-math3 are compile dependencies of fluss-common and are used directly by Fluss code, but fluss-client shades everything without a block, so 1714 of their classes shipped at their original package. The Flink and Spark connector uber-jars inherit them. On an engine classpath those classes shadow the engine's own copies. #3960 fixed exactly this for the S3/GS/Azure filesystem plugins after Spark hit NoSuchMethodError on a shadowed commons-text class, but the client was never covered -- leaving the Spark connectors, the most likely place to meet a conflicting commons, shipping unshaded copies. Relocating is safe here: no public API in fluss-common or fluss-client exposes a commons type, and the commons-lang3 reflection helpers reflect over user classes rather than over lang3 itself. The patterns name lang3 and math3 individually rather than using org.apache.commons. Only those two are bundled; commons-codec and commons-logging are referenced but supplied from elsewhere, and relocating them would leave links resolving nowhere. Also empties KNOWN_EXCEPTIONS in the leak checker. The entries covered exactly this leak, and leaving them behind would mask a regression. fluss-lake-iceberg is deliberately not added: its commons leak is a different set of libraries (compress, lang3, pool) and remains an open problem. Eight jars stop leaking commons: fluss-client, the five Flink connectors and both Spark connectors. 320 tests pass across fluss-client, fluss-flink-1.20 and fluss-spark-3.5. --- fluss-client/pom.xml | 22 ++++++++++++++++++++++ tools/ci/check_shaded_jars.py | 19 ++++++++----------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/fluss-client/pom.xml b/fluss-client/pom.xml index a5761faed8f..61de853aa91 100644 --- a/fluss-client/pom.xml +++ b/fluss-client/pom.xml @@ -131,6 +131,28 @@ *:* + + + + org.apache.commons.lang3 + org.apache.fluss.shaded.org.apache.commons.lang3 + + + org.apache.commons.math3 + org.apache.fluss.shaded.org.apache.commons.math3 + + diff --git a/tools/ci/check_shaded_jars.py b/tools/ci/check_shaded_jars.py index afce2df14c0..cdfcc5b9246 100644 --- a/tools/ci/check_shaded_jars.py +++ b/tools/ci/check_shaded_jars.py @@ -134,18 +134,15 @@ # so the checker stays usable as a gate. --compare still fails if one of these # grows. Entries are (jar filename prefix, forbidden package prefix). # -# fluss-client bundles commons-math3 (1386 classes) and commons-lang3 (431) at -# their original paths, and the Flink and Spark uber-jars inherit both. #3960 -# relocated org.apache.commons in the S3/GS/Azure filesystem plugins only; the -# client was never covered. Separate pre-existing issue from #3553 / #4072. +# Currently empty. fluss-client, the Flink connectors and the Spark connectors +# used to be listed here for commons-math3/commons-lang3; that leak is fixed, +# so the entries are gone rather than left behind where they would mask a +# regression. # -# fluss-lake-iceberg leaks commons too but from different libraries -# (commons-compress 602, lang3 431, commons-pool 57), so it is a distinct -# problem and is deliberately NOT excepted here. -KNOWN_EXCEPTIONS: Sequence[Tuple[str, str]] = ( - ("fluss-client", "org/apache/commons/"), - ("fluss-flink-", "org/apache/commons/"), -) +# fluss-lake-iceberg still leaks commons, but from different libraries +# (commons-compress 602, lang3 431, commons-pool 57). It is deliberately NOT +# excepted: it is a real unfixed problem, not an accepted one. +KNOWN_EXCEPTIONS: Sequence[Tuple[str, str]] = () # Packages holding only annotations. A missing or duplicated annotation class From 876292a4775b510b5105611b7ca8f36a9ff37379 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 19:39:43 +0200 Subject: [PATCH 09/10] [lake] Relocate bundled commons and jackson in the iceberg and lance jars fluss-lake-iceberg shipped 1030 unshaded commons classes and fluss-lake-lance 1091 jackson plus 519 commons, all able to shadow an engine's own copies. iceberg relocates commons-compress, commons-lang3 and commons-pool; lance relocates jackson wholly plus commons-codec and commons-lang3. Each pattern names the packages actually bundled, because the shade plugin also rewrites references to classes a jar does not contain: iceberg references commons-io and commons-lang 2.x, and lance references commons-logging, none of which are bundled, so a broad org.apache.commons pattern would leave links resolving nowhere. Two leaks in lance are deliberately left alone. org.apache.arrow cannot be relocated. The bundled libarrow_cdata_jni exports JNI symbols that embed the Java package name -- verified with nm, e.g. Java_org_apache_arrow_c_jni_JniWrapper_exportArray. Renaming the package makes the JVM look for a symbol the library does not export, which fails with UnsatisfiedLinkError at runtime rather than at build time. io.netty in lance is Arrow's allocator layer (io.netty.buffer, io.netty.util only) and is tied to the Arrow classes above, so it stays with them. Verified: 161 tests pass across both modules, dangling relocated references fell from 16 to 12 (the remainder being pre-existing fluss-shaded-arrow references and two commons-codec resource-directory strings whose 159 resource files shade correctly relocated alongside the classes). fluss-lake-iceberg is clean afterwards; fluss-lake-lance keeps only the arrow and netty leaks described above. --- fluss-lake/fluss-lake-iceberg/pom.xml | 19 ++++++++++++++++++ fluss-lake/fluss-lake-lance/pom.xml | 28 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/fluss-lake/fluss-lake-iceberg/pom.xml b/fluss-lake/fluss-lake-iceberg/pom.xml index 3306a1b465b..e69102b40bd 100644 --- a/fluss-lake/fluss-lake-iceberg/pom.xml +++ b/fluss-lake/fluss-lake-iceberg/pom.xml @@ -327,6 +327,25 @@ org.apache.parquet org.apache.iceberg.shaded.org.apache.parquet + + + org.apache.commons.compress + org.apache.fluss.shaded.org.apache.commons.compress + + + org.apache.commons.lang3 + org.apache.fluss.shaded.org.apache.commons.lang3 + + + org.apache.commons.pool + org.apache.fluss.shaded.org.apache.commons.pool + diff --git a/fluss-lake/fluss-lake-lance/pom.xml b/fluss-lake/fluss-lake-lance/pom.xml index df3e46ade81..9ec2b301a9a 100644 --- a/fluss-lake/fluss-lake-lance/pom.xml +++ b/fluss-lake/fluss-lake-lance/pom.xml @@ -208,6 +208,34 @@ *:* + + + + com.fasterxml.jackson + org.apache.fluss.shaded.com.fasterxml.jackson + + + + org.apache.commons.codec + org.apache.fluss.shaded.org.apache.commons.codec + + + org.apache.commons.lang3 + org.apache.fluss.shaded.org.apache.commons.lang3 + + + * From cf7f00e2cf9760a0c0703a3a4b6884483d47df13 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 28 Aug 2026 19:50:50 +0200 Subject: [PATCH 10/10] [fs] Relocate netty in the azure, cos, obs and oss plugins Each of these four shipped 1698 unshaded io.netty classes, pulled in by the cloud SDK's HTTP layer, able to shadow an engine's own netty. Relocating netty is established practice here: fluss-metrics-influxdb already does it with a plain , and fluss-common consumes netty through the pre-shaded fluss-shaded-netty artifact. Neither needed special handling, and none of these four bundle a single netty native. io.netty.internal.tcnative is excluded from the relocation. tcnative is netty's JNI wrapper around OpenSSL, so its native symbols bind to that package name and renaming it would break them -- the same failure mode that rules out relocating org.apache.arrow in fluss-lake-lance. It is referenced here but not bundled, so leaving it alone both avoids references that resolve nowhere and keeps OpenSSL usable when the host supplies it. Verified in the built jar: 0 references to a shaded tcnative, 73 to the original. An earlier note in this branch claimed these jars ship netty natives and that this made relocation unsafe. That was wrong: the natives present are snappy's and Arrow/Lance JNI, and there are no netty natives at all. Verified: 264 tests pass across the filesystem modules, and no new dangling relocated references (the protobuf, jxpath and codec ones listed by --dangling are pre-existing, from the broad com.google and org.apache.commons patterns inherited from #4073 and #3960). All four plugins stop leaking netty. --- fluss-filesystems/fluss-fs-azure/pom.xml | 15 +++++++++++++++ fluss-filesystems/fluss-fs-cos/pom.xml | 15 +++++++++++++++ fluss-filesystems/fluss-fs-obs/pom.xml | 15 +++++++++++++++ fluss-filesystems/fluss-fs-oss/pom.xml | 15 +++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/fluss-filesystems/fluss-fs-azure/pom.xml b/fluss-filesystems/fluss-fs-azure/pom.xml index 654d22481d8..9ad73f14b48 100644 --- a/fluss-filesystems/fluss-fs-azure/pom.xml +++ b/fluss-filesystems/fluss-fs-azure/pom.xml @@ -275,6 +275,21 @@ org.apache.commons org.apache.fluss.shaded.org.apache.commons + + io.netty + + org.apache.fluss.fs.shaded.azure.io.netty + + + + io.netty.internal.tcnative.** + + diff --git a/fluss-filesystems/fluss-fs-cos/pom.xml b/fluss-filesystems/fluss-fs-cos/pom.xml index 147dd4f82f0..75f15ae98fc 100644 --- a/fluss-filesystems/fluss-fs-cos/pom.xml +++ b/fluss-filesystems/fluss-fs-cos/pom.xml @@ -295,6 +295,21 @@ org.apache.commons org.apache.fluss.shaded.org.apache.commons + + io.netty + + org.apache.fluss.fs.shaded.cos.io.netty + + + + io.netty.internal.tcnative.** + + diff --git a/fluss-filesystems/fluss-fs-obs/pom.xml b/fluss-filesystems/fluss-fs-obs/pom.xml index b36b9864d6e..579ebc9c213 100644 --- a/fluss-filesystems/fluss-fs-obs/pom.xml +++ b/fluss-filesystems/fluss-fs-obs/pom.xml @@ -282,6 +282,21 @@ org.apache.commons org.apache.fluss.shaded.org.apache.commons + + io.netty + + org.apache.fluss.fs.shaded.obs.io.netty + + + + io.netty.internal.tcnative.** + + diff --git a/fluss-filesystems/fluss-fs-oss/pom.xml b/fluss-filesystems/fluss-fs-oss/pom.xml index b708bec4af2..3a4ffb77338 100644 --- a/fluss-filesystems/fluss-fs-oss/pom.xml +++ b/fluss-filesystems/fluss-fs-oss/pom.xml @@ -266,6 +266,21 @@ org.apache.commons org.apache.fluss.shaded.org.apache.commons + + io.netty + + org.apache.fluss.fs.shaded.oss.io.netty + + + + io.netty.internal.tcnative.** + +