From 423bb8ac5cb894a29d219ca8bfe3cf1f3f07ca84 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Fri, 23 Oct 2026 20:19:35 +0000 Subject: [PATCH 1/9] Share configurable release asset checks and snapshot publication --- .github/buildcheck/releaseassets.py | 192 +++++++++++ .github/buildcheck/snapshot.py | 124 +++++++ .../buildcheck/tests/test_releaseassets.py | 305 ++++++++++++++++++ .github/buildcheck/tests/test_snapshot.py | 205 ++++++++++++ .github/collect-release-assets.py | 49 +++ .github/release-assets.json | 27 ++ .github/release-assets.json.license | 3 + .github/report-asset-signatures.py | 33 ++ .github/shared-files.sha256 | 12 +- .github/update-snapshot.py | 32 ++ .github/workflows/publish.yml | 200 +++--------- CLAUDE.md | 24 +- 12 files changed, 1031 insertions(+), 175 deletions(-) create mode 100644 .github/buildcheck/releaseassets.py create mode 100644 .github/buildcheck/snapshot.py create mode 100644 .github/buildcheck/tests/test_releaseassets.py create mode 100644 .github/buildcheck/tests/test_snapshot.py create mode 100644 .github/collect-release-assets.py create mode 100644 .github/release-assets.json create mode 100644 .github/release-assets.json.license create mode 100644 .github/report-asset-signatures.py create mode 100644 .github/update-snapshot.py diff --git a/.github/buildcheck/releaseassets.py b/.github/buildcheck/releaseassets.py new file mode 100644 index 0000000..36d36fe --- /dev/null +++ b/.github/buildcheck/releaseassets.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Collect signed Maven release assets and explicitly configured extra files. + +No repository names or native/Android layouts live here. See the workspace policy +policies/release-asset-scripts.md for the JSON contract and staging options. +""" + +import csv +import json +import re +import shutil +import xml.etree.ElementTree as ET +from pathlib import Path + + +def _keys(value, allowed, context): + if not isinstance(value, dict) or set(value) - set(allowed): + raise ValueError(f"invalid {context}; allowed keys: {', '.join(allowed)}") + + +def _inside(root, relative): + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise ValueError(f"expected a repository-relative path: {relative!r}") + path = (root / relative).resolve() + if not path.is_relative_to(root): + raise ValueError(f"path escapes repository: {relative}") + return path + + +def configuration(root, path=None): + """Read the explicit contract; unknown keys fail instead of silently dropping assets.""" + path = Path(path) if path is not None else Path(root) / ".github/release-assets.json" + config = json.loads(path.read_text()) + _keys(config, ("schema", "maven", "files"), "release asset configuration") + if type(config.get("schema")) is not int or config["schema"] != 1: + raise ValueError("release asset configuration requires schema: 1") + maven = config.get("maven", {}) + _keys(maven, ("standalone", "cyclonedx"), "maven configuration") + if not isinstance(maven.get("standalone", []), list): + raise ValueError("maven.standalone must be a list of module paths") + if type(maven.get("cyclonedx", False)) is not bool: + raise ValueError("maven.cyclonedx must be a boolean") + if not isinstance(config.get("files", []), list): + raise ValueError("files must be a list") + return config + + +def projects(root, standalone=()): + """Discover the reactor recursively, then optional separately published Maven projects.""" + root = Path(root).resolve() + seen, result = set(), [] + + def visit(folder, reactor=True): + folder = folder.resolve() + if not folder.is_relative_to(root) or folder in seen: + raise ValueError(f"invalid or repeated Maven module: {folder}") + seen.add(folder) + pom = ET.parse(folder / "pom.xml").getroot() + artifact = pom.findtext("{*}artifactId") + version = pom.findtext("{*}version") or pom.findtext("{*}parent/{*}version") + packaging = pom.findtext("{*}packaging", "jar") + if (not artifact or not version or not re.fullmatch(r"[\w.-]+", artifact) + or not re.fullmatch(r"[\w.+-]+", version) + or packaging not in ("pom", "jar", "maven-plugin")): + raise ValueError(f"unsupported release coordinates in {folder / 'pom.xml'}") + if result and version != result[0][2]: + raise ValueError(f"release version mismatch in {folder / 'pom.xml'}: {version}") + result.append((folder, artifact, version, packaging, reactor)) + for module in pom.findall("{*}modules/{*}module"): + visit(_inside(root, str(folder.relative_to(root) / (module.text or "").strip())), reactor) + + visit(root) + for module in standalone: + visit(_inside(root, module), reactor=False) + return result + + +def _classifiers(root, spec): + if isinstance(spec, dict): + _keys(spec, ("csv", "column"), "classifier CSV") + if not isinstance(spec.get("column"), str) or not spec["column"]: + raise ValueError("classifier CSV requires a column") + with _inside(root, spec.get("csv")).open() as stream: + rows = csv.DictReader(line for line in stream if line.strip() and not line.lstrip().startswith("#")) + if spec["column"] not in (rows.fieldnames or []): + raise ValueError(f"missing classifier CSV column: {spec['column']}") + spec = [row[spec["column"]] for row in rows] + if (not isinstance(spec, list) or not spec or any(not isinstance(c, str) + or not re.fullmatch(r"[\w.-]*", c) for c in spec) or len(set(spec)) != len(spec)): + raise ValueError("classifiers must be a nonempty list of unique classifier names") + return spec + + +def _format(template, version, classifier): + if not isinstance(template, str) or not template: + raise ValueError("file source/name must be a nonempty string") + try: + return template.format(version=version, classifier=classifier) + except (KeyError, ValueError, IndexError, AttributeError) as error: + raise ValueError(f"invalid release filename template: {template}") from error + + +def file_classifiers(root, index, config_path=None): + """Let packaging loops consume the same classifier list the collector checks.""" + if index < 0: + raise ValueError("file rule index must not be negative") + config = configuration(root, config_path) + try: + item = config.get("files", [])[index] + except IndexError as error: + raise ValueError(f"no file rule at index {index}") from error + _keys(item, ("source", "name", "classifiers"), "file rule") + return _classifiers(Path(root).resolve(), item.get("classifiers", [""])) + + +def collect(root, destination, config_path=None, *, maven_only=False, maven_assets=None): + """Copy available outputs, returning all missing files/signatures. Never hide partial output. + + maven_only stages deploy-signed files before later packaging overwrites target/. + maven_assets reads those staged files by their final names during the final collection. + """ + root, destination = Path(root).resolve(), Path(destination).resolve() + config = configuration(root, config_path) + modules = projects(root, config.get("maven", {}).get("standalone", [])) + version, expected, errors = modules[0][2], [], [] + for folder, artifact, _, packaging, reactor in modules: + base = f"{artifact}-{version}" + outputs = [(folder / "target" / f"{base}.pom", f"{base}.pom")] + if packaging != "pom": + for classifier in ("", "-sources", "-javadoc"): + name = f"{base}{classifier}.jar" + outputs.append((folder / "target" / name, name)) + if reactor and config.get("maven", {}).get("cyclonedx", False): + outputs.append((folder / "target/bom.json", f"{base}-cyclonedx.json")) + for source, name in outputs: + expected.append((Path(maven_assets) / name if maven_assets is not None else source, name)) + + for item in [] if maven_only else config.get("files", []): + _keys(item, ("source", "name", "classifiers"), "file rule") + for classifier in _classifiers(root, item.get("classifiers", [""])): + pattern = _format(item.get("source"), version, classifier) + _inside(root, pattern) + matches = sorted(root.glob(pattern)) + if len(matches) != 1: + errors.append(f"expected one release output matching {pattern}, found {len(matches)}") + continue + source = matches[0] + _inside(root, str(source.relative_to(root))) + name = _format(item["name"], version, classifier) if "name" in item else source.name + expected.append((source, name)) + + names = set() + for source, name in expected: + if (Path(name).name != name or name in (".", "..") or name.endswith(".asc") + or name in names or name + ".asc" in names): + raise ValueError(f"invalid or duplicate release asset filename: {name}") + names.update((name, name + ".asc")) + if destination == source.parent.resolve(): + raise ValueError("asset destination overlaps release inputs") + destination.mkdir(parents=True, exist_ok=True) + if any(destination.iterdir()): + raise ValueError(f"asset destination must be empty: {destination}") + for source, name in expected: + for suffix in ("", ".asc"): + path = Path(str(source) + suffix) + if not path.is_file() or path.stat().st_size == 0: + errors.append(f"missing or empty release output: {path}") + continue + shutil.copyfile(path, destination / (name + suffix)) + return errors + + +def signature_gaps(directory): + """Presence/size check for every file type; signing itself verifies cryptographic validity.""" + directory = Path(directory) + files = sorted(directory.iterdir()) if directory.is_dir() else [] + artifacts = [path for path in files if not path.name.endswith(".asc")] + if not artifacts: + return -1, [f"no artifacts in {directory} -- the collection step produced nothing"] + errors = [] + for path in artifacts: + signature = Path(str(path) + ".asc") + if not path.is_file() or path.stat().st_size == 0: + errors.append(f"missing or empty artifact: {path.name}") + elif not signature.is_file() or signature.stat().st_size == 0: + errors.append(f"unsigned: {path.name} has no nonempty detached .asc") + for path in files: + if path.name.endswith(".asc") and not path.with_suffix("").is_file(): + errors.append(f"orphan signature: {path.name}") + return len(errors), errors diff --git a/.github/buildcheck/snapshot.py b/.github/buildcheck/snapshot.py new file mode 100644 index 0000000..0c13dbb --- /dev/null +++ b/.github/buildcheck/snapshot.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Update the rolling snapshot without replacing a newer publish or deleting the release. + +The workflow serializes this entire operation with a job concurrency group. The ancestry/run +checks also reject an older build which finishes after a newer build has already published. +Available artifacts are uploaded even after an incomplete collection; only complete uploads +advance the source tag/notes and prune obsolete assets. +""" + +import json +import re +import subprocess +from pathlib import Path + +from .releaseassets import signature_gaps + + +class GitHub: + def command(self, *args): + subprocess.run(["gh", *map(str, args)], check=True) + + def api(self, method, endpoint, payload=None, optional=False): + args = ["gh", "api", "--method", method, endpoint] + if payload is not None: + args += ["--input", "-"] + result = subprocess.run(args, input=json.dumps(payload) if payload is not None else None, + text=True, capture_output=True) + if result.returncode: + if optional and "HTTP 404" in result.stderr: + return None + raise RuntimeError(f"GitHub {method} {endpoint}: {result.stderr.strip()}") + return json.loads(result.stdout) if result.stdout.strip() else None + + +def publish(repository, sha, run_id, attempt, assets_dir, complete, github=None, + *, tag="snapshot", title="Snapshot (latest)"): + """Return False for a superseded run; otherwise upload and update the rolling release.""" + if not re.fullmatch(r"[\w.-]+/[\w.-]+", repository) or not re.fullmatch(r"[0-9a-f]{40}", sha): + raise ValueError("invalid repository or published commit") + if not re.fullmatch(r"[\w][\w.-]*", tag) or ".." in tag or tag.endswith((".", ".lock")): + raise ValueError("invalid rolling release tag") + run_id, attempt = int(run_id), int(attempt) + if run_id < 1 or attempt < 1: + raise ValueError("run ID and attempt must be positive") + files = sorted(Path(assets_dir).iterdir()) + if not files or any(not path.is_file() for path in files): + raise ValueError("refusing to update snapshot without a nonempty, flat asset directory") + artifacts = [path for path in files if not path.name.endswith(".asc")] + if not artifacts: + raise ValueError("refusing to update snapshot with signatures but no artifacts") + names = {path.name for path in files} + complete = complete and signature_gaps(assets_dir)[0] == 0 + gh = github or GitHub() + repo = f"repos/{repository}" + ref = gh.api("GET", f"{repo}/git/ref/tags/{tag}", optional=True) + release = gh.api("GET", f"{repo}/releases/tags/{tag}", optional=True) + published = ref["object"] if ref else None + # Annotated tags point to tag objects, not directly to commits. + for _ in range(8): + if not published or published["type"] != "tag": + break + published = gh.api("GET", f"{repo}/git/tags/{published['sha']}")["object"] + if published and published["type"] != "commit": + raise ValueError("snapshot tag does not resolve to a commit") + if published and published["sha"] != sha: + comparison = gh.api("GET", f"{repo}/compare/{published['sha']}...{sha}") + if comparison["status"] == "behind": + print("Snapshot already names a newer commit; skipping this older build.") + return False + if comparison["status"] not in ("ahead", "identical"): + raise ValueError("refusing to replace snapshot with a divergent commit") + if release: + marker = re.search(r"", + release.get("body") or "") + if marker and marker[3] == sha and (int(marker[1]), int(marker[2])) > (run_id, attempt): + print("Snapshot already names a newer run/attempt of this commit; skipping.") + return False + + body = (f"Snapshot from [{sha[:8]}](https://github.com/{repository}/commit/{sha}).\n\n" + f"[Publish run {run_id}, attempt {attempt}]" + f"(https://github.com/{repository}/actions/runs/{run_id}/attempts/{attempt}).\n\n" + f"") + if not release: + if not ref: + ref = gh.api("POST", f"{repo}/git/refs", {"ref": f"refs/tags/{tag}", "sha": sha}) + release = gh.api("POST", f"{repo}/releases", { + "tag_name": tag, "target_commitish": sha, "name": title, + "prerelease": True, "body": body if complete else body + "\n\nIncomplete artifact collection." + }) + gh.command("release", "upload", tag, *files, "--repo", repository, "--clobber") + if not complete: + raise ValueError("available files uploaded, but collection is incomplete; " + "existing snapshot metadata and assets were retained") + + if ref: + gh.api("PATCH", f"{repo}/git/refs/tags/{tag}", {"sha": sha, "force": True}) + else: + gh.api("POST", f"{repo}/git/refs", {"ref": f"refs/tags/{tag}", "sha": sha}) + gh.api("PATCH", f"{repo}/releases/{release['id']}", { + "target_commitish": sha, "name": title, "prerelease": True, "body": body + }) + + # Read every page before deleting: deleting while paginating could skip shifted entries. + assets = [] + page = 1 + while True: + batch = gh.api("GET", f"{repo}/releases/{release['id']}/assets?per_page=100&page={page}") + assets.extend(batch) + if len(batch) < 100: + break + page += 1 + failures = [] + for asset in assets: + if asset["name"] not in names: + try: + print(f"Removing obsolete snapshot asset: {asset['name']}") + gh.api("DELETE", f"{repo}/releases/assets/{asset['id']}", optional=True) + except RuntimeError as error: + failures.append(str(error)) + if failures: + raise RuntimeError("; ".join(failures)) + return True diff --git a/.github/buildcheck/tests/test_releaseassets.py b/.github/buildcheck/tests/test_releaseassets.py new file mode 100644 index 0000000..a4ed72a --- /dev/null +++ b/.github/buildcheck/tests/test_releaseassets.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from buildcheck.releaseassets import collect, configuration, file_classifiers, projects, signature_gaps +from buildcheck import workflow +from buildcheck.tests.helpers import REPO + + +VERSION = "2.0.0-SNAPSHOT" + + +class ReleaseAssetsTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.output = self.root / "assets" + self.config = {"schema": 1, "maven": {"standalone": ["tool"], "cyclonedx": True}, "files": [ + {"source": "app/target/app-{version}-{classifier}.jar", + "classifiers": {"csv": ".github/classifiers.csv", "column": "classifier"}}, + {"source": "mobile/publication/pom.xml", "name": "mobile-{version}.pom"}, + {"source": "mobile/build/mobile-{version}.aar"}, + {"source": "archive/upstream-*-{version}.tar.gz"}]} + self.save_config() + self.write("pom.xml", f""" +parent{VERSION}pom +appplugin""") + self.outputs(".", "parent", "pom") + self.module("app", "jar") + self.module("plugin", "maven-plugin") + self.module("tool", "jar", reactor=False) + self.write(".github/classifiers.csv", "# comment\nclassifier\ncpu-linux\ngpu-linux\n") + for classifier in ("cpu-linux", "gpu-linux"): + self.asset(f"app/target/app-{VERSION}-{classifier}.jar") + self.asset("mobile/publication/pom.xml") + self.asset(f"mobile/build/mobile-{VERSION}.aar") + self.asset(f"archive/upstream-build1-{VERSION}.tar.gz") + + def write(self, name, content="artifact"): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + def save_config(self): + self.write(".github/release-assets.json", json.dumps(self.config)) + + def asset(self, name): + self.write(name) + self.write(name + ".asc", "signature of " + name) + + def outputs(self, folder, artifact, packaging, reactor=True): + self.asset(f"{folder}/target/{artifact}-{VERSION}.pom") + if packaging != "pom": + for classifier in ("", "-sources", "-javadoc"): + self.asset(f"{folder}/target/{artifact}-{VERSION}{classifier}.jar") + if reactor: + self.asset(f"{folder}/target/bom.json") + + def module(self, name, packaging, reactor=True): + self.write(f"{name}/pom.xml", f""" +{VERSION}{name} +{packaging}""") + self.outputs(name, name, packaging, reactor) + + def test_reactor_plugin_standalone_and_bom_names_match_central(self): + self.assertEqual(collect(self.root, self.output), []) + for artifact in ("app", "plugin", "tool"): + for classifier in ("", "-sources", "-javadoc"): + self.assertTrue((self.output / f"{artifact}-{VERSION}{classifier}.jar.asc").is_file()) + for artifact in ("parent", "app", "plugin"): + self.assertTrue((self.output / f"{artifact}-{VERSION}-cyclonedx.json.asc").is_file()) + self.assertFalse((self.output / "bom.json").exists()) + self.assertFalse((self.output / f"tool-{VERSION}-cyclonedx.json").exists()) + self.assertTrue((self.output / f"mobile-{VERSION}.pom.asc").is_file()) + + def test_new_reactor_module_is_automatic(self): + pom = self.root / "pom.xml" + pom.write_text(pom.read_text().replace("", "new")) + self.module("new", "jar") + self.assertEqual(collect(self.root, self.output), []) + self.assertTrue((self.output / f"new-{VERSION}.jar").is_file()) + + def test_single_library_needs_no_native_android_or_bom_configuration(self): + self.config = {"schema": 1} + self.save_config() + self.write("pom.xml", f"library{VERSION}") + self.outputs(".", "library", "jar") + self.assertEqual(collect(self.root, self.output), []) + self.assertEqual(len(list(self.output.iterdir())), 8) + + def test_missing_artifacts_signatures_boms_and_extras_preserve_available_files(self): + missing = ["app/target/bom.json", f"plugin/target/plugin-{VERSION}.pom.asc", + f"app/target/app-{VERSION}-gpu-linux.jar", f"mobile/build/mobile-{VERSION}.aar"] + for name in missing: + (self.root / name).unlink() + errors = collect(self.root, self.output) + for name in missing: + self.assertTrue(any(name in error for error in errors), errors) + self.assertTrue((self.output / f"app-{VERSION}.jar").is_file()) + + def test_empty_signature_fails(self): + self.write(f"app/target/app-{VERSION}.jar.asc", "") + self.assertTrue(any("missing or empty" in e for e in collect(self.root, self.output))) + + def test_stale_versions_unconfigured_fat_jars_and_plugin_dependencies_are_ignored(self): + for name in ("app/target/app-1.0.0.jar", f"app/target/app-{VERSION}-jar-with-dependencies.jar", + "app/target/spotbugs/third-party.jar"): + self.asset(name) + self.assertEqual(collect(self.root, self.output), []) + self.assertFalse(any("1.0.0" in p.name or "dependencies" in p.name or "third-party" in p.name + for p in self.output.iterdir())) + + def test_nonempty_destination_and_version_mismatch_fail(self): + self.write("assets/stale.jar") + with self.assertRaisesRegex(ValueError, "must be empty"): + collect(self.root, self.output) + (self.output / "stale.jar").unlink() + pom = self.root / "plugin/pom.xml" + pom.write_text(pom.read_text().replace(VERSION, "1.0.0")) + with self.assertRaisesRegex(ValueError, "version mismatch"): + collect(self.root, self.output) + + def test_ambiguous_archive_is_reported(self): + self.asset(f"archive/upstream-build2-{VERSION}.tar.gz") + self.assertTrue(any("found 2" in error for error in collect(self.root, self.output))) + + def test_staged_maven_files_survive_rebuild_and_missing_stage_file_fails_final_collection(self): + stage = self.root / "central-assets" + self.assertEqual(collect(self.root, stage, maven_only=True), []) + original = (stage / f"app-{VERSION}.jar").read_bytes() + self.write(f"app/target/app-{VERSION}.jar", "rebuilt unsigned jar") + self.assertEqual(collect(self.root, self.output, maven_assets=stage), []) + self.assertEqual((self.output / f"app-{VERSION}.jar").read_bytes(), original) + (stage / f"app-{VERSION}-javadoc.jar").unlink() + errors = collect(self.root, self.root / "partial", maven_assets=stage) + self.assertTrue(any("javadoc.jar" in e for e in errors)) + + def test_typo_or_wrong_schema_fails(self): + for config in ({"schema": 2}, {"schema": 1, "filez": []}, + {"schema": 1, "maven": {"cyclonedx": "yes"}}): + with self.subTest(config=config): + self.config = config + self.save_config() + with self.assertRaises(ValueError): + collect(self.root, self.output) + + def test_duplicate_names_traversal_bad_templates_and_empty_classifier_list_fail(self): + for rule in ({"source": "mobile/publication/pom.xml", "name": f"app-{VERSION}.jar"}, + {"source": "../outside.jar"}, {"source": "mobile/{unknown}.jar"}, + {"source": "mobile/build/*.aar", "classifiers": []}, + {"source": "mobile/build/*.aar", "name": "../outside.jar"}): + with self.subTest(rule=rule): + self.config["files"] = [rule] + self.save_config() + with self.assertRaises(ValueError): + collect(self.root, self.output) + self.assertFalse(self.output.exists()) + + def test_repeated_or_outside_module_fails(self): + for module in ("app", "../../outside"): + self.write("pom.xml", f"""parent{VERSION} +app{module}""") + with self.assertRaises(ValueError): + projects(self.root) + + def test_classifier_list_is_shared_with_packaging_loop(self): + self.assertEqual(file_classifiers(self.root, 0), ["cpu-linux", "gpu-linux"]) + self.config["files"][0]["classifiers"] = ["gpu-one", "gpu-two"] + self.save_config() + self.assertEqual(file_classifiers(self.root, 0), ["gpu-one", "gpu-two"]) + with self.assertRaisesRegex(ValueError, "no file rule"): + file_classifiers(self.root, 99) + with self.assertRaisesRegex(ValueError, "must not be negative"): + file_classifiers(self.root, -1) + + def test_current_repository_workflow_passes_collection_result_and_checks_both_uploads(self): + jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) + for kind in ("snapshot", "release"): + job = jobs[f"publish-{kind}"].text + self.assertIn(f"python3 .github/collect-release-assets.py signed-{kind}-assets", job) + self.assertIn("assets_complete: $" + "{{ steps.collect.outputs.complete }}", job) + self.assertIn(" id: collect", job) + attach = jobs["github-snapshot" if kind == "snapshot" else + "github-release" if "github-release" in jobs else "github-release-signed"].text + self.assertIn(f"python3 .github/report-asset-signatures.py {'snapshot' if kind == 'snapshot' else 'release'}-assets", attach) + self.assertLess(attach.index("id: signatures"), attach.index(" - name: Fail if anything")) + if configuration(REPO).get("maven", {}).get("cyclonedx", False): + for kind in ("snapshot", "release"): + self.assertIn(f"signed-{kind}-assets/*.json", jobs[f"publish-{kind}"].text) + + def test_current_repository_contract_collects_exact_complete_and_partial_fixture(self): + # Exercise this checkout's real POMs and JSON, not just a hand-written generic fixture. + repo = Path(REPO) + config = configuration(repo) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + expected = set() + + def copy(relative): + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(repo / relative, target) + + def artifact(relative, name): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + for suffix in ("", ".asc"): + Path(str(path) + suffix).write_text("fixture " + str(relative) + suffix) + expected.add(name + suffix) + + copy(".github/release-assets.json") + modules = projects(repo, config.get("maven", {}).get("standalone", [])) + version = modules[0][2] + for folder, name, _, packaging, reactor in modules: + relative = folder.relative_to(repo) + copy(relative / "pom.xml") + base = f"{name}-{version}" + artifact(relative / "target" / f"{base}.pom", f"{base}.pom") + if packaging != "pom": + for classifier in ("", "-sources", "-javadoc"): + artifact(relative / "target" / f"{base}{classifier}.jar", f"{base}{classifier}.jar") + if reactor and config.get("maven", {}).get("cyclonedx", False): + artifact(relative / "target/bom.json", f"{base}-cyclonedx.json") + for index, rule in enumerate(config.get("files", [])): + if isinstance(rule.get("classifiers"), dict): + copy(rule["classifiers"]["csv"]) + for classifier in file_classifiers(repo, index): + source = rule["source"].format(version=version, classifier=classifier).replace("*", "fixture") + name = rule.get("name", Path(source).name).format(version=version, classifier=classifier) + artifact(source, name) + destination = root / "collected" + self.assertEqual(collect(root, destination), []) + self.assertEqual({p.name for p in destination.iterdir()}, expected) + (destination / next(n for n in expected if n.endswith(".asc"))).unlink() + self.assertNotEqual(signature_gaps(destination)[0], 0) + # An incomplete reactor must still produce an upload directory with available files. + first = modules[0][1] + "-" + version + ".pom" + (root / "target" / first).unlink() + self.assertTrue(collect(root, root / "partial")) + self.assertTrue(any((root / "partial").iterdir())) + + def test_cli_reports_completeness_even_on_failure_and_accepts_parameters(self): + scripts = self.root / ".github" + (scripts / "buildcheck").mkdir() + for name in ("collect-release-assets.py", "buildcheck/releaseassets.py", "buildcheck/__init__.py"): + shutil.copyfile(Path(REPO) / ".github" / name, scripts / name) + cli = [sys.executable, str(scripts / "collect-release-assets.py")] + listing = subprocess.run(cli + ["--list-classifiers", "0"], capture_output=True, text=True) + self.assertEqual(listing.returncode, 0, listing.stderr) + self.assertEqual(listing.stdout.splitlines(), ["cpu-linux", "gpu-linux"]) + for complete in (True, False): + if not complete: + (self.root / f"plugin/target/plugin-{VERSION}.jar").unlink() + output = self.root / f"github-output-{complete}" + destination = self.root / f"collected-{complete}" + result = subprocess.run(cli + [str(destination), "--config", str(scripts / "release-assets.json")], + env={**os.environ, "GITHUB_OUTPUT": str(output)}, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0 if complete else 1, result.stdout + result.stderr) + self.assertEqual(output.read_text(), f"complete={str(complete).lower()}\n") + self.assertTrue((destination / f"app-{VERSION}.jar").is_file()) + + +class SignatureReportTest(unittest.TestCase): + def test_all_file_types_spaces_empty_orphan_and_missing_signatures(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + self.assertEqual(signature_gaps(root)[0], -1) + for name in ("file.jar", "mobile.aar", "parent.pom", "bom.json", "source.tar.gz", "with spaces.zip"): + (root / name).write_text("artifact") + (root / (name + ".asc")).write_text("signature") + self.assertEqual(signature_gaps(root), (0, [])) + (root / "bom.json.asc").write_text("") + (root / "source.tar.gz.asc").unlink() + (root / "orphan.jar.asc").write_text("signature") + (root / "file.jar").write_text("") + self.assertEqual(signature_gaps(root)[0], 4) + + def test_cli_always_returns_zero_and_writes_missing_for_post_upload_assertion(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + assets = root / "assets" + assets.mkdir() + (assets / "unsigned.json").write_text("bom") + output = root / "github-output" + result = subprocess.run([sys.executable, str(Path(REPO) / ".github/report-asset-signatures.py"), + str(assets)], env={**os.environ, "GITHUB_OUTPUT": str(output)}, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("::error::unsigned", result.stdout) + self.assertEqual(output.read_text(), "missing=1\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/buildcheck/tests/test_snapshot.py b/.github/buildcheck/tests/test_snapshot.py new file mode 100644 index 0000000..e14f357 --- /dev/null +++ b/.github/buildcheck/tests/test_snapshot.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from buildcheck.snapshot import GitHub, publish +from buildcheck import workflow +from buildcheck.tests.helpers import REPO + + +OLD, NEW = "a" * 40, "b" * 40 + + +class FakeGitHub: + def __init__(self): + self.ref = {"object": {"type": "commit", "sha": OLD}} + self.release = {"id": 7, "body": "Snapshot from old commit"} + self.comparison = "ahead" + self.pages = {1: [{"id": 1, "name": "old.jar"}, {"id": 2, "name": "new.json"}]} + self.calls = [] + self.fail_upload = False + self.fail_deletes = set() + self.tag = "snapshot" + + def api(self, method, endpoint, payload=None, optional=False): + self.calls.append((method, endpoint, payload)) + if method == "DELETE" and int(endpoint.rsplit("/", 1)[1]) in self.fail_deletes: + raise RuntimeError("delete failed") + if method == "GET": + if endpoint.endswith(f"git/ref/tags/{self.tag}"): + return self.ref + if endpoint.endswith(f"releases/tags/{self.tag}"): + return self.release + if "/compare/" in endpoint: + return {"status": self.comparison} + if "/git/tags/" in endpoint: + return {"object": {"type": "commit", "sha": OLD}} + if "/assets?" in endpoint: + return self.pages[int(endpoint.split("&page=")[1])] + if method == "POST" and endpoint.endswith("/releases"): + return {"id": 7} + if method == "POST" and endpoint.endswith("/git/refs"): + return {"object": {"type": "commit", "sha": payload["sha"]}} + return {} + + def command(self, *args): + self.calls.append(("upload", args, None)) + if self.fail_upload: + raise subprocess.CalledProcessError(1, "gh release upload") + + +class SnapshotTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.assets = Path(self.temp.name) + for name in ("new.json", "new.json.asc"): + (self.assets / name).write_text("artifact") + self.gh = FakeGitHub() + quiet = patch("builtins.print") + quiet.start() + self.addCleanup(quiet.stop) + + def publish(self, **kwargs): + return publish("owner/repo", NEW, kwargs.pop("run_id", 10), kwargs.pop("attempt", 1), + self.assets, kwargs.pop("complete", True), self.gh, **kwargs) + + def test_upload_precedes_tag_notes_and_prune_and_release_itself_survives(self): + self.assertTrue(self.publish()) + calls = self.gh.calls + upload = next(i for i, c in enumerate(calls) if c[0] == "upload") + tag = next(i for i, c in enumerate(calls) if c[0] == "PATCH" and "/git/refs/" in c[1]) + notes = next(c[2] for c in calls if c[0] == "PATCH" and c[1].endswith("/releases/7")) + self.assertLess(upload, tag) + self.assertIn(NEW, notes["body"]) + self.assertIn("/actions/runs/10/attempts/1", notes["body"]) + self.assertEqual(notes["target_commitish"], NEW) + deletes = [c[1] for c in calls if c[0] == "DELETE"] + self.assertEqual(deletes, ["repos/owner/repo/releases/assets/1"]) + + def test_incomplete_collection_uploads_available_files_without_promoting_or_pruning(self): + with self.assertRaisesRegex(ValueError, "collection is incomplete"): + self.publish(complete=False) + self.assertTrue(any(c[0] == "upload" for c in self.gh.calls)) + self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) + + def test_missing_signature_also_prevents_promotion(self): + (self.assets / "new.json.asc").unlink() + with self.assertRaisesRegex(ValueError, "collection is incomplete"): + self.publish() + self.assertFalse(any(c[0] == "PATCH" for c in self.gh.calls)) + + def test_late_finishing_older_commit_cannot_overwrite_newer_snapshot(self): + self.gh.comparison = "behind" + self.assertFalse(self.publish()) + self.assertTrue(all(c[0] == "GET" for c in self.gh.calls)) + + def test_empty_signature_or_orphan_also_prevents_promotion(self): + for name in ("new.json.asc", "orphan.jar.asc"): + with self.subTest(name=name): + (self.assets / name).write_text("") + with self.assertRaisesRegex(ValueError, "collection is incomplete"): + self.publish() + self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) + + def test_rolling_tag_and_title_are_parameters(self): + self.gh.tag = "nightly" + self.assertTrue(self.publish(tag="nightly", title="Nightly build")) + upload = next(c for c in self.gh.calls if c[0] == "upload") + self.assertEqual(upload[1][2], "nightly") + notes = next(c[2] for c in self.gh.calls if c[0] == "PATCH" and c[1].endswith("/releases/7")) + self.assertEqual(notes["name"], "Nightly build") + + def test_same_commit_older_run_or_attempt_is_skipped(self): + self.gh.ref["object"]["sha"] = NEW + for run, attempt in ((11, 1), (10, 2)): + self.gh.release["body"] = f"" + self.gh.calls.clear() + self.assertFalse(self.publish()) + self.assertTrue(all(c[0] == "GET" for c in self.gh.calls)) + + def test_new_attempt_of_same_commit_can_replace_old_artifacts(self): + self.gh.ref["object"]["sha"] = NEW + self.gh.release["body"] = f"" + self.assertTrue(self.publish(attempt=2)) + + def test_annotated_tags_are_resolved_before_comparing_commits(self): + self.gh.ref["object"]["type"] = "tag" + self.assertTrue(self.publish()) + self.assertTrue(any("/git/tags/" in c[1] for c in self.gh.calls if c[0] == "GET")) + + def test_creates_missing_tag_and_release_once(self): + self.gh.ref, self.gh.release = None, None + self.assertTrue(self.publish()) + self.assertEqual(sum(c[0] == "POST" and c[1].endswith("/git/refs") for c in self.gh.calls), 1) + self.assertTrue(any(c[0] == "POST" and c[1].endswith("/releases") for c in self.gh.calls)) + + def test_reads_all_asset_pages_before_deleting(self): + self.gh.pages = {1: [{"id": i, "name": f"old-{i}.jar"} for i in range(100)], + 2: [{"id": 101, "name": "last-old.jar"}, {"id": 102, "name": "new.json"}]} + self.assertTrue(self.publish()) + calls = self.gh.calls + page2 = next(i for i, c in enumerate(calls) if c[0] == "GET" and "&page=2" in c[1]) + deletion = next(i for i, c in enumerate(calls) if c[0] == "DELETE") + self.assertLess(page2, deletion) + self.assertEqual(sum(c[0] == "DELETE" for c in calls), 101) + + def test_divergent_commit_and_empty_upload_are_refused(self): + self.gh.comparison = "diverged" + with self.assertRaisesRegex(ValueError, "divergent"): + self.publish() + self.assertTrue(all(c[0] == "GET" for c in self.gh.calls)) + for path in self.assets.iterdir(): + path.unlink() + self.gh.calls.clear() + with self.assertRaisesRegex(ValueError, "nonempty"): + self.publish() + self.assertEqual(self.gh.calls, []) + + def test_upload_failure_does_not_move_tag_or_prune(self): + self.gh.fail_upload = True + with self.assertRaises(subprocess.CalledProcessError): + self.publish() + self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) + + def test_prune_tries_remaining_assets_after_a_deletion_failure(self): + self.gh.pages[1].append({"id": 3, "name": "another-old.jar"}) + self.gh.fail_deletes.add(1) + with self.assertRaisesRegex(RuntimeError, "delete failed"): + self.publish() + self.assertTrue(any(c[0] == "DELETE" and c[1].endswith("/assets/3") for c in self.gh.calls)) + + def test_signature_only_directory_does_not_mutate_snapshot(self): + (self.assets / "new.json").unlink() + with self.assertRaisesRegex(ValueError, "no artifacts"): + self.publish() + self.assertEqual(self.gh.calls, []) + + def test_workflow_serializes_entire_update_and_passes_collection_result(self): + jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) + job = jobs["github-snapshot"].text + self.assertIn(" concurrency:\n group: snapshot-release-${{ github.repository }}", job) + self.assertIn(" cancel-in-progress: false", job) + self.assertIn("ASSETS_COMPLETE: ${{ needs.publish-snapshot.outputs.assets_complete }}", job) + self.assertIn("python3 .github/update-snapshot.py snapshot-assets", job) + + def test_optional_api_failure_accepts_only_404_not_authentication_failure(self): + gh = GitHub() + for status in (403, 404): + response = subprocess.CompletedProcess([], 1, "", f"gh: failure (HTTP {status})") + with patch("buildcheck.snapshot.subprocess.run", return_value=response): + if status == 404: + self.assertIsNone(gh.api("GET", "repos/owner/repo/releases/tags/snapshot", optional=True)) + else: + with self.assertRaisesRegex(RuntimeError, "403"): + gh.api("GET", "repos/owner/repo/releases/tags/snapshot", optional=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/collect-release-assets.py b/.github/collect-release-assets.py new file mode 100644 index 0000000..88de5b8 --- /dev/null +++ b/.github/collect-release-assets.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Collect assets using the repository's explicit release-assets.json contract.""" + +import argparse +import os +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +from buildcheck.releaseassets import collect, file_classifiers + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("destination", nargs="?") + parser.add_argument("--config", help="JSON contract (default: .github/release-assets.json)") + parser.add_argument("--maven-only", action="store_true", help="stage signed Maven files before a rebuild") + parser.add_argument("--maven-assets", help="read Maven files from a previous staging directory") + parser.add_argument("--list-classifiers", type=int, metavar="RULE", help="print classifiers of a file rule (zero based)") + args = parser.parse_args() + if args.list_classifiers is None and args.destination is None: + parser.error("destination is required unless --list-classifiers is used") + errors = [] + try: + root = Path(__file__).resolve().parent.parent + if args.list_classifiers is not None: + if args.list_classifiers < 0: + raise ValueError("file rule index must not be negative") + print("\n".join(file_classifiers(root, args.list_classifiers, args.config))) + return 0 + errors = collect(root, args.destination, args.config, + maven_only=args.maven_only, maven_assets=args.maven_assets) + except (ValueError, OSError, ET.ParseError) as error: + errors.append(str(error)) + for error in errors: + print(f"::error::{error}") + if not errors: + print(f"Collected {len(list(Path(args.destination).iterdir()))} release files including signatures.") + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"complete={'false' if errors else 'true'}\n") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/release-assets.json b/.github/release-assets.json new file mode 100644 index 0000000..6af75ab --- /dev/null +++ b/.github/release-assets.json @@ -0,0 +1,27 @@ +{ + "schema": 1, + "files": [ + { + "source": "fatjar-assets/srcmorph-cli-{version}-jar-with-dependencies.jar" + }, + { + "source": "fatjar-assets/srcmorph-cli-{version}-jar-with-dependencies-{classifier}.jar", + "classifiers": [ + "cuda13-linux-x86-64", + "cuda13-windows-x86-64", + "vulkan-linux-x86-64", + "vulkan-linux-aarch64", + "vulkan-windows-x86-64", + "vulkan-windows-aarch64", + "opencl-windows-x86-64", + "opencl-windows-aarch64", + "rocm-linux-x86-64", + "rocm-windows-x86-64", + "sycl-linux-x86-64", + "sycl-windows-x86-64", + "openvino-linux-x86-64", + "openvino-windows-x86-64" + ] + } + ] +} diff --git a/.github/release-assets.json.license b/.github/release-assets.json.license new file mode 100644 index 0000000..eca5b04 --- /dev/null +++ b/.github/release-assets.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 Bernard Ladenthin + +SPDX-License-Identifier: MIT OR Apache-2.0 diff --git a/.github/report-asset-signatures.py b/.github/report-asset-signatures.py new file mode 100644 index 0000000..a8fbcef --- /dev/null +++ b/.github/report-asset-signatures.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Report signature gaps without blocking uploads; assert the missing output afterwards.""" + +import argparse +import os + +from buildcheck.releaseassets import signature_gaps + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory") + args = parser.parse_args() + try: + missing, errors = signature_gaps(args.directory) + except OSError as error: + missing, errors = -1, [str(error)] + for error in errors: + print(f"::error::{error}") + if not missing: + print("All collected artifacts have nonempty detached signatures.") + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"missing={missing}\n") + # The workflow uploads first and fails on the output afterwards. + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index b047fda..56ed26d 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -69,5 +69,13 @@ e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workfl 09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot 51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workflows/publish.yml#verify-signing-key-gradle -c2684a3f93aaf552c13f845307536c2a14dda315c7d7b1e416bc66951b811dc1 .github/workflows/publish.yml#github-snapshot -c09a030706dbea4221851ca0fcbfa6fc686165b408724724ade61724da6e123b .github/workflows/publish.yml#github-release +7f96fc1b0cffa9844379162c2f82e5f26454ab0fdc7349019ddc05423b4320af .github/workflows/publish.yml#github-snapshot +68d41a4a97d26438f5649f9c6f24a884d1448192486b386271d49051c58f6ee1 .github/workflows/publish.yml#github-release +b9bbf8762e5898709d69a2509c64750e3acf41dc0a69426578773714d263d365 .github/buildcheck/releaseassets.py +bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py +8e9f5870bae1d9e903ef35db9366cd680fcbb459076d1fdb0390248ab6b1cc30 .github/buildcheck/tests/test_releaseassets.py +c0401d973100c4ab0ec7728ec5d3a841831863cd307112c88f8cfe172316316a .github/buildcheck/tests/test_snapshot.py +bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py +6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py +930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py +71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license diff --git a/.github/update-snapshot.py b/.github/update-snapshot.py new file mode 100644 index 0000000..03dcd45 --- /dev/null +++ b/.github/update-snapshot.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Usage: update-snapshot.py ; requires the serialized github-snapshot job.""" + +import argparse +import os +import subprocess +import sys + +from buildcheck.snapshot import publish + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory") + parser.add_argument("--tag", default="snapshot") + parser.add_argument("--title", default="Snapshot (latest)") + args = parser.parse_args() + try: + publish(os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_SHA"], + os.environ["GITHUB_RUN_ID"], os.environ["GITHUB_RUN_ATTEMPT"], args.directory, + os.environ.get("ASSETS_COMPLETE") == "true", tag=args.tag, title=args.title) + return 0 + except (ValueError, OSError, RuntimeError, KeyError, subprocess.CalledProcessError) as error: + print(f"::error::{error}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c4f7c6e..a0e9ec3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -535,6 +535,8 @@ jobs: if: needs.check-snapshot.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central + outputs: + assets_complete: ${{ steps.collect.outputs.complete }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 @@ -583,18 +585,7 @@ jobs: # before the classifier loop rebuilds (and clobbers) srcmorph-cli/target. - name: Collect signed Central artifacts (plain/sources/javadoc; fat jars handled next) if: ${{ !cancelled() }} - run: | - mkdir -p signed-snapshot-assets - # Every module's jars and poms with their .asc, the reactor parent's pom included (`.` -- - # srcmorph-parent is a Central artifact of its own); maven-gpg writes - # target/-.pom next to its .pom.asc. - for m in . srcmorph srcmorph-cli srcmorph-maven-plugin; do - for f in "$m"/target/*.jar "$m"/target/*.jar.asc "$m"/target/*.pom "$m"/target/*.pom.asc; do - [ -e "$f" ] || continue - case "$f" in *-jar-with-dependencies.jar|*-jar-with-dependencies.jar.asc) continue ;; esac - cp "$f" signed-snapshot-assets/ - done - done + run: python3 .github/collect-release-assets.py central-assets --maven-only # Build one srcmorph-cli fat jar per net.ladenthin:llama GPU module jar + the default # all-platform CPU fat jar, GPG-sign each, and add them to the GitHub-Release assets. # These are GitHub-only (never Maven Central): every jar carries the multi-platform CPU @@ -615,14 +606,8 @@ jobs: # library jars come with llama-platform. Each jar here is ADDED to the default CPU # natives (-Dllama.classifier activates the gpu-natives profile), so every classifier # fat jar keeps the CPU fallback: ggml loads the module when its runtime is installed. - CLASSIFIERS=( - cuda13-linux-x86-64 cuda13-windows-x86-64 - vulkan-linux-x86-64 vulkan-linux-aarch64 vulkan-windows-x86-64 vulkan-windows-aarch64 - opencl-windows-x86-64 opencl-windows-aarch64 - rocm-linux-x86-64 rocm-windows-x86-64 - sycl-linux-x86-64 sycl-windows-x86-64 - openvino-linux-x86-64 openvino-windows-x86-64 - ) + python3 .github/collect-release-assets.py --list-classifiers 1 > fatjar-classifiers.txt + mapfile -t CLASSIFIERS < fatjar-classifiers.txt build() { mvn -B --no-transfer-progress -pl srcmorph-cli -am -DskipTests -Dmaven.javadoc.skip=true -Dllama.classifier="$1" package; } for c in "${CLASSIFIERS[@]}"; do build "$c" @@ -633,8 +618,9 @@ jobs: build "" # Collect every fat jar variant, then GPG-sign each via the cross-repo shared script # (.github/sign-fatjars.sh; java-llama.cpp dropped its copy with its fat jars) — a .asc per fat jar. - cp srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies*.jar signed-snapshot-assets/ - bash .github/sign-fatjars.sh signed-snapshot-assets + mkdir -p fatjar-assets + cp srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies*.jar fatjar-assets/ + bash .github/sign-fatjars.sh fatjar-assets # Structural check of what the loop just produced. The smoke-fatjar job LAUNCHES the # default jar; the classifier jars cannot be covered that way (a GitHub runner has no # such device, and the only command that would load the native library is a real @@ -643,7 +629,11 @@ jobs: # CPU native kept next to it. The backend directory is the point: if -Dllama.classifier= # ever stops being wired through, the loop silently ships copies of the CPU build under # GPU names and nothing else in this pipeline notices. - bash .github/verify-classifier-fatjars.sh signed-snapshot-assets "${CLASSIFIERS[@]}" + bash .github/verify-classifier-fatjars.sh fatjar-assets "${CLASSIFIERS[@]}" + - name: Collect signed artifacts + id: collect + if: ${{ !cancelled() }} + run: python3 .github/collect-release-assets.py signed-snapshot-assets --maven-assets central-assets - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 if: ${{ !cancelled() }} with: @@ -658,9 +648,14 @@ jobs: # the GitHub pre-release assets must not be lost in that case. if: ${{ !cancelled() && (needs.publish-snapshot.result == 'success' || needs.publish-snapshot.result == 'failure') }} runs-on: ubuntu-latest + # Serialize the complete upload, tag/notes update and pruning operation. + concurrency: + group: snapshot-release-${{ github.repository }} + cancel-in-progress: false permissions: contents: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-snapshot-assets @@ -674,104 +669,12 @@ jobs: # still loudly red rather than quietly wrong. # See workspace/policies/fat-jar-release-assets.md, "Attach first, then go red". id: signatures - run: | - set -uo pipefail - dir="snapshot-assets" - # Every attachable artifact type, not only jars: with '*.jar' alone the AARs were attached - # unsigned for months and nothing counted them, so the job reported success. - artifacts=$(find "$dir" -maxdepth 1 \( -name '*.jar' -o -name '*.aar' -o -name '*.pom' \) | sort) - if [ -z "$artifacts" ]; then - echo "::error::no artifacts in $dir -- the collection step produced nothing" - echo "missing=-1" >> "$GITHUB_OUTPUT" - exit 0 - fi - missing=0 - for artifact in $artifacts; do - if [ ! -e "$artifact.asc" ]; then - echo "::error::unsigned: $(basename "$artifact") has no detached .asc" - missing=$((missing + 1)) - fi - done - echo "missing=$missing" >> "$GITHUB_OUTPUT" - [ "$missing" -eq 0 ] && echo "all $(echo "$artifacts" | wc -l) artifact(s) signed" - exit 0 - - name: Update snapshot pre-release + run: python3 .github/report-asset-signatures.py snapshot-assets + - name: Update snapshot assets, source tag and notes env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release view snapshot --repo ${{ github.repository }} 2>/dev/null \ - || gh release create snapshot \ - --repo ${{ github.repository }} \ - --prerelease \ - --title "Snapshot (latest)" \ - --notes "Latest snapshot build from the main branch." - gh release upload snapshot snapshot-assets/* \ - --repo ${{ github.repository }} \ - --clobber - - name: Remove the assets this run did not upload - # `snapshot` is a rolling "latest" pre-release, and `gh release upload --clobber` replaces - # same-named assets but never removes any other, so every asset of every past version stayed - # forever: 208 on java-llama.cpp (109 of them from 5.0.x, 43 in shapes that no longer exist), - # and SpotBugs plugin jars an old collect step attached in 2026-05 on three repositories. A - # non-current asset on this release is noise by definition. The keep-set is what this run - # uploaded -- the files in the asset directory -- never a version pattern, which would have to - # be re-taught at every classifier added or dropped (how the stale shapes arose). AFTER the - # upload on purpose: a failed upload must not leave the release emptier than it was. And only - # here: a versioned v* release is history, github-release never deletes anything. Every - # deletion is printed by name, so the job log is the audit trail. - # - # Asset by asset, never the release. The first form of this job (2026-05-07) deleted the whole - # `snapshot` release with its tag and recreated it; on 2026-05-10 the runs of several main - # pushes minutes apart deleted it under each other, and every job of each burst but the last - # failed in that step with its artifacts present. The job then went with the Central - # migration and came back as `view || create` + `upload --clobber` without any cleanup, which - # is how the backlog grew (workspace/policies/fat-jar-release-assets.md has the runs). Hence - # three guards: an empty upload is refused (every asset would count as stale; the download - # and upload steps above fail first, the guard keeps it so), an asset that is already gone - # counts as removed (two overlapping dispatches may both pick it), and a failed deletion is - # reported and the rest still tried -- the assets of this upload are on the release either - # way, so this is "attach first, then red" like the signature check. - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - keep="$(ls -1 snapshot-assets | sort)" - if [ -z "$keep" ]; then - echo "::error::refusing to prune: this run uploaded nothing, so every asset of the release would count as stale" - exit 1 - fi - list_assets() { - gh release view snapshot --repo ${{ github.repository }} --json assets --jq '.assets[].name' | sort - } - current="$(list_assets)" - stale="$(comm -13 <(printf '%s\n' "$keep") <(printf '%s\n' "$current"))" - if [ -z "$stale" ]; then - echo "nothing to remove: the release holds exactly the $(printf '%s\n' "$keep" | wc -l) asset(s) of this upload" - exit 0 - fi - echo "removing $(printf '%s\n' "$stale" | wc -l) asset(s) that are not part of this upload:" - failed=0 - while IFS= read -r name; do - [ -n "$name" ] || continue - echo " deleting $name" - if gh release delete-asset snapshot "$name" --repo ${{ github.repository }} --yes; then - continue - fi - if ! remaining="$(list_assets)"; then - echo "::error::could not delete $name, and could not re-read the release to tell whether it is gone" - failed=$((failed + 1)) - elif printf '%s\n' "$remaining" | grep -qxF -- "$name"; then - echo "::error::could not delete $name" - failed=$((failed + 1)) - else - echo " $name was already gone" - fi - done <<< "$stale" - if [ "$failed" -ne 0 ]; then - echo "::error::$failed asset(s) could not be removed; the $(printf '%s\n' "$keep" | wc -l) asset(s) of this upload are on the release" - exit 1 - fi - echo "kept the $(printf '%s\n' "$keep" | wc -l) asset(s) of this upload" + ASSETS_COMPLETE: ${{ needs.publish-snapshot.outputs.assets_complete }} + run: python3 .github/update-snapshot.py snapshot-assets - name: Fail if anything was attached unsigned # After the upload on purpose: the assets must exist even when the signature does not. if: ${{ always() && steps.signatures.outputs.missing != '0' }} @@ -785,6 +688,8 @@ jobs: if: needs.check-tag.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central + outputs: + assets_complete: ${{ steps.collect.outputs.complete }} permissions: contents: write steps: @@ -822,18 +727,7 @@ jobs: # before the classifier loop rebuilds (and clobbers) srcmorph-cli/target. - name: Collect signed Central artifacts (plain/sources/javadoc; fat jars handled next) if: ${{ !cancelled() }} - run: | - mkdir -p signed-release-assets - # Every module's jars and poms with their .asc, the reactor parent's pom included (`.` -- - # srcmorph-parent is a Central artifact of its own); maven-gpg writes - # target/-.pom next to its .pom.asc. - for m in . srcmorph srcmorph-cli srcmorph-maven-plugin; do - for f in "$m"/target/*.jar "$m"/target/*.jar.asc "$m"/target/*.pom "$m"/target/*.pom.asc; do - [ -e "$f" ] || continue - case "$f" in *-jar-with-dependencies.jar|*-jar-with-dependencies.jar.asc) continue ;; esac - cp "$f" signed-release-assets/ - done - done + run: python3 .github/collect-release-assets.py central-assets --maven-only # Build one srcmorph-cli fat jar per net.ladenthin:llama GPU module jar + the default # all-platform CPU fat jar, GPG-sign each, and add them to the GitHub-Release assets. # These are GitHub-only (never Maven Central): every jar carries the multi-platform CPU @@ -854,14 +748,8 @@ jobs: # library jars come with llama-platform. Each jar here is ADDED to the default CPU # natives (-Dllama.classifier activates the gpu-natives profile), so every classifier # fat jar keeps the CPU fallback: ggml loads the module when its runtime is installed. - CLASSIFIERS=( - cuda13-linux-x86-64 cuda13-windows-x86-64 - vulkan-linux-x86-64 vulkan-linux-aarch64 vulkan-windows-x86-64 vulkan-windows-aarch64 - opencl-windows-x86-64 opencl-windows-aarch64 - rocm-linux-x86-64 rocm-windows-x86-64 - sycl-linux-x86-64 sycl-windows-x86-64 - openvino-linux-x86-64 openvino-windows-x86-64 - ) + python3 .github/collect-release-assets.py --list-classifiers 1 > fatjar-classifiers.txt + mapfile -t CLASSIFIERS < fatjar-classifiers.txt build() { mvn -B --no-transfer-progress -pl srcmorph-cli -am -DskipTests -Dmaven.javadoc.skip=true -Dllama.classifier="$1" package; } for c in "${CLASSIFIERS[@]}"; do build "$c" @@ -872,8 +760,9 @@ jobs: build "" # Collect every fat jar variant, then GPG-sign each via the cross-repo shared script # (.github/sign-fatjars.sh; java-llama.cpp dropped its copy with its fat jars) — a .asc per fat jar. - cp srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies*.jar signed-release-assets/ - bash .github/sign-fatjars.sh signed-release-assets + mkdir -p fatjar-assets + cp srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies*.jar fatjar-assets/ + bash .github/sign-fatjars.sh fatjar-assets # Structural check of what the loop just produced. The smoke-fatjar job LAUNCHES the # default jar; the classifier jars cannot be covered that way (a GitHub runner has no # such device, and the only command that would load the native library is a real @@ -882,7 +771,11 @@ jobs: # CPU native kept next to it. The backend directory is the point: if -Dllama.classifier= # ever stops being wired through, the loop silently ships copies of the CPU build under # GPU names and nothing else in this pipeline notices. - bash .github/verify-classifier-fatjars.sh signed-release-assets "${CLASSIFIERS[@]}" + bash .github/verify-classifier-fatjars.sh fatjar-assets "${CLASSIFIERS[@]}" + - name: Collect signed artifacts + id: collect + if: ${{ !cancelled() }} + run: python3 .github/collect-release-assets.py signed-release-assets --maven-assets central-assets - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 if: ${{ !cancelled() }} with: @@ -900,6 +793,7 @@ jobs: permissions: contents: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-release-assets @@ -913,27 +807,7 @@ jobs: # still loudly red rather than quietly wrong. # See workspace/policies/fat-jar-release-assets.md, "Attach first, then go red". id: signatures - run: | - set -uo pipefail - dir="release-assets" - # Every attachable artifact type, not only jars: with '*.jar' alone the AARs were attached - # unsigned for months and nothing counted them, so the job reported success. - artifacts=$(find "$dir" -maxdepth 1 \( -name '*.jar' -o -name '*.aar' -o -name '*.pom' \) | sort) - if [ -z "$artifacts" ]; then - echo "::error::no artifacts in $dir -- the collection step produced nothing" - echo "missing=-1" >> "$GITHUB_OUTPUT" - exit 0 - fi - missing=0 - for artifact in $artifacts; do - if [ ! -e "$artifact.asc" ]; then - echo "::error::unsigned: $(basename "$artifact") has no detached .asc" - missing=$((missing + 1)) - fi - done - echo "missing=$missing" >> "$GITHUB_OUTPUT" - [ "$missing" -eq 0 ] && echo "all $(echo "$artifacts" | wc -l) artifact(s) signed" - exit 0 + run: python3 .github/report-asset-signatures.py release-assets - name: Upload release assets uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3 with: diff --git a/CLAUDE.md b/CLAUDE.md index 04a127b..8de9feb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -599,16 +599,20 @@ list in `publish.yml`, every `module` row of java-llama.cpp's `natives.csv` but (srcmorph's own since java-llama.cpp dropped its fat jars with its modular natives). The convention + per-repo shapes + the classifier keep-in-sync rule are documented in [`../workspace/policies/fat-jar-release-assets.md`](../workspace/policies/fat-jar-release-assets.md). -The attach jobs are shared text with the two sibling Maven-only repositories -(`publish.yml#github-snapshot` / `#github-release` in `shared-files.sha256`): the collect steps copy -`-.pom` + `.pom.asc` with the jars (maven-gpg writes both into `target/`), the -signature check counts every attachable type (`*.jar`, `*.aar`, `*.pom`), and `github-snapshot` ends -by deleting every asset of the `snapshot` pre-release that the run did not upload, each printed by -name -- `gh release upload --clobber` never removed anything, so the release accumulated every past -version. The prune refuses an empty upload, treats an asset that is already gone as removed and -fails only after trying every deletion; versioned releases are never pruned. See the policy's -"Attach first, then go red" and "The `snapshot` pre-release holds exactly the current upload", which -also records the delete-and-recreate form of this job (May 2026) and why it is gone. +Release collection, signature reporting and rolling-snapshot updates use the byte-identical +`collect-release-assets.py`, `report-asset-signatures.py`, `update-snapshot.py` and their buildcheck +library/tests, all listed in `.github/shared-files.sha256`. Repository-specific extras live in +`.github/release-assets.json`; Maven modules are discovered from POMs. Missing artifacts/signatures +fail collection while preserving available files for upload. The whole `github-snapshot` job is +shared across all four Java repos and serializes upload, source-tag/notes updates and pruning; +only complete signed uploads advance/prune the snapshot. Versioned releases are never pruned. +The `github-release` job remains shared across the three Maven-only repos. Interface, staging +options and synchronization rules: [`../workspace/policies/release-asset-scripts.md`](../workspace/policies/release-asset-scripts.md). + +The publish jobs stage deploy-signed Maven outputs with `--maven-only` before the classifier +loop rebuilds `target/`. The final collector uses `--maven-assets central-assets` and requires +all configured fat jars from `fatjar-assets/`. The loop's classifier list comes from file rule 1 +in `.github/release-assets.json` via `--list-classifiers 1` (default CPU is built last). **srcmorph-specific smoke.** The cross-repo rule "no release asset is attached that CI has not run" is implemented here by the `smoke-fatjar` job (`needs: [build]`, gates both publish jobs): it From 93866ef2aaf08f4414298b7bf507a3beac5acfd5 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Fri, 23 Oct 2026 20:51:18 +0000 Subject: [PATCH 2/9] Disable saved checkout credentials in release upload jobs --- .github/buildcheck/tests/test_snapshot.py | 10 ++++++++++ .github/shared-files.sha256 | 6 +++--- .github/workflows/publish.yml | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/buildcheck/tests/test_snapshot.py b/.github/buildcheck/tests/test_snapshot.py index e14f357..bf148a6 100644 --- a/.github/buildcheck/tests/test_snapshot.py +++ b/.github/buildcheck/tests/test_snapshot.py @@ -200,6 +200,16 @@ def test_optional_api_failure_accepts_only_404_not_authentication_failure(self): with self.assertRaisesRegex(RuntimeError, "403"): gh.api("GET", "repos/owner/repo/releases/tags/snapshot", optional=True) + def test_release_upload_jobs_do_not_persist_checkout_credentials(self): + jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) + versioned = "github-release" if "github-release" in jobs else "github-release-signed" + for name in ("github-snapshot", versioned): + checkouts = [step for step in jobs[name].steps() + if any("uses: actions/checkout@" in line for line in step)] + self.assertTrue(checkouts, name) + for step in checkouts: + self.assertIn(" persist-credentials: false", step, name) + if __name__ == "__main__": unittest.main() diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 56ed26d..832d4d5 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -69,12 +69,12 @@ e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workfl 09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot 51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workflows/publish.yml#verify-signing-key-gradle -7f96fc1b0cffa9844379162c2f82e5f26454ab0fdc7349019ddc05423b4320af .github/workflows/publish.yml#github-snapshot -68d41a4a97d26438f5649f9c6f24a884d1448192486b386271d49051c58f6ee1 .github/workflows/publish.yml#github-release +f28843b1fc88b8876dfc6cc0cd451e5003087e904f4f20cefcf7c645e7ebea44 .github/workflows/publish.yml#github-snapshot +eaa2911c2c754ae0b1c9acff9aaba347e5d7fdfb841b4b5f2d12b271cb71d7fe .github/workflows/publish.yml#github-release b9bbf8762e5898709d69a2509c64750e3acf41dc0a69426578773714d263d365 .github/buildcheck/releaseassets.py bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py 8e9f5870bae1d9e903ef35db9366cd680fcbb459076d1fdb0390248ab6b1cc30 .github/buildcheck/tests/test_releaseassets.py -c0401d973100c4ab0ec7728ec5d3a841831863cd307112c88f8cfe172316316a .github/buildcheck/tests/test_snapshot.py +4bab88300dc2bc06d33d659a2118d81070118e65300b9cfbacdc0313d11bee43 .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a0e9ec3..57f2bd8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -656,6 +656,8 @@ jobs: contents: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-snapshot-assets @@ -794,6 +796,8 @@ jobs: contents: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-release-assets From 4ee9bd8eab2d376ad20357d6d75baff6cbf1495b Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 07:22:59 +0000 Subject: [PATCH 3/9] docs: add CodeRabbit badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 79fbe55..e57844d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ **AI:** [![Claude](https://img.shields.io/badge/Claude-D97757?logo=claude&logoColor=fff)](#) +[![CodeRabbit](https://img.shields.io/badge/CodeRabbit-AI%20Reviews-FF570A)](https://coderabbit.ai/) **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) From 3633cae53ae886f19afd6002cb931440275bc410 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 07:56:36 +0000 Subject: [PATCH 4/9] docs: use repository-specific CodeRabbit reviews badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e57844d..219a73f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ **AI:** [![Claude](https://img.shields.io/badge/Claude-D97757?logo=claude&logoColor=fff)](#) -[![CodeRabbit](https://img.shields.io/badge/CodeRabbit-AI%20Reviews-FF570A)](https://coderabbit.ai/) +![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/bernardladenthin/srcmorph?utm_source=oss&utm_medium=github&utm_campaign=bernardladenthin%2Fsrcmorph&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews) **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) From 2f915f08169280d1e4cc05d2c83ef65a27281fe8 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 11:14:55 +0000 Subject: [PATCH 5/9] Preserve runtime license notices and document scoped FOSSA review --- .fossa.yml | 11 +++ .github/collect-third-party-licenses.py | 89 +++++++++++++++++++++++++ .github/shared-files.sha256 | 3 + THIRD_PARTY.md | 31 +++++++++ docs/dependency-license-review.md | 69 +++++++++++++++++++ srcmorph-cli/pom.xml | 53 +++++++++++++++ srcmorph-maven-plugin/pom.xml | 13 ++++ srcmorph/pom.xml | 13 ++++ 8 files changed, 282 insertions(+) create mode 100644 .fossa.yml create mode 100644 .github/collect-third-party-licenses.py create mode 100644 THIRD_PARTY.md create mode 100644 docs/dependency-license-review.md diff --git a/.fossa.yml b/.fossa.yml new file mode 100644 index 0000000..b51dd8a --- /dev/null +++ b/.fossa.yml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 + +version: 3 +project: + locator: git+github.com/bernardladenthin/srcmorph + +# No fixed revision.branch: preserve the actual PR/main branch and commit. +# No license overrides or scope exclusions: test and runtime findings stay visible. +maven: + scope-exclude: [] diff --git a/.github/collect-third-party-licenses.py b/.github/collect-third-party-licenses.py new file mode 100644 index 0000000..df33f5d --- /dev/null +++ b/.github/collect-third-party-licenses.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Preserve legal files from resolved runtime JARs without filename collisions. + +Input is maven-dependency-plugin:build-classpath output on this operating system. +This performs no license approvals, downloads, source rewriting or file filtering +by license type. Archives without legal files remain visible in inventory.json. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import zipfile + + +def legal_file(name): + path = PurePosixPath(name) + if path.is_absolute() or ".." in path.parts or "\\" in name or any(":" in part for part in path.parts): + raise ValueError("Unsafe archive entry: " + name) + if name.endswith("/"): + return False + lower = path.name.lower() + # A provider's LICENSE.class is executable code, not a license document. + if path.suffix.lower() in {".class", ".java", ".kt", ".png", ".jpg", ".gif", ".jar"}: + return False + return (any(part.lower() in {"licenses", "licences"} for part in path.parts) + or re.fullmatch(r"(?:licen[sc]e|notice|copying)(?:[-_.].*)?", lower) is not None) + + +def collect(classpath, output): + marker = output / ".generated-by-collect-third-party-licenses" + if output.is_symlink(): + raise ValueError("Output must not be a symlink") + if output.exists(): + if not marker.is_file(): + raise ValueError("Refusing to replace an output directory without our marker") + shutil.rmtree(output) + output.mkdir(parents=True) + marker.write_text("Generated runtime dependency license inventory.\n", encoding="utf-8") + entries = [] + for item in sorted(set(classpath.read_text(encoding="utf-8").strip().split(os.pathsep))): + if not item: + continue + source = Path(item) + if not source.is_file() or source.suffix.lower() != ".jar": + raise ValueError("Classpath entry is not a readable JAR: " + item) + with source.open("rb") as stream: + hasher = hashlib.sha256() + for block in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(block) + digest = hasher.hexdigest() + # The filename preserves classifiers; the digest also distinguishes equal + # filenames from different groups or different bytes of a snapshot. + directory = re.sub(r"[^a-zA-Z0-9._-]", "_", source.name) + "-" + digest[:16] + files = [] + with zipfile.ZipFile(source) as archive: + for info in sorted(archive.infolist(), key=lambda entry: entry.filename): + if not legal_file(info.filename): + continue + destination = output / directory / info.filename + destination.parent.mkdir(parents=True, exist_ok=True) + data = archive.read(info) + if destination.exists() and destination.read_bytes() != data: + raise ValueError("Conflicting duplicate entry: " + info.filename) + destination.write_bytes(data) + files.append(info.filename) + entries.append({"artifact": source.name, "sha256": digest, + "directory": directory, "legal_files": files}) + (output / "inventory.json").write_text( + json.dumps({"format": 1, "scope": "runtime", "artifacts": entries}, indent=2) + "\n", + encoding="utf-8") + missing = sum(not entry["legal_files"] for entry in entries) + print(f"Runtime legal inventory: {len(entries)} JARs, {missing} without embedded legal files") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--classpath", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + collect(args.classpath, args.output) + + +if __name__ == "__main__": + main() diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 832d4d5..278d009 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -79,3 +79,6 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license +7d82df16062877522da6a3d7a41e4b28f43ce4776ae89b6f7c7933cbb9f41aa3 docs/dependency-license-review.md +8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo +a6a4d6efe20634a62f860294b3fbd30c71d017a42fc55421ae83b8a7e8aba572 .github/collect-third-party-licenses.py diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md new file mode 100644 index 0000000..4feabb4 --- /dev/null +++ b/THIRD_PARTY.md @@ -0,0 +1,31 @@ + + +# Third-party license and attribution notes + +The project is Apache-2.0. Logback in the concrete core dependency declaration is +test-scoped; its parent dependencyManagement pin is not a runtime dependency. +The runnable CLI uses SLF4J Simple and bundles the selected llama runtime/native artifacts. +Those native artifacts have additional third-party contents requiring their own review. + +## Review evidence + +See [the dependency license review](docs/dependency-license-review.md) in the source repository +for exact-version evidence and remaining decisions. This inventory is not an approval of all +licenses found by a scanner. Test-tool redistribution requires a separate review. + +For unchanged Maven components, source is available from +`https://repo.maven.apache.org/maven2////` +as `--sources.jar`; POMs and upstream source repositories provide provenance. +Preserve copyright, license and NOTICE files when redistributing dependencies. + +Runnable BAF/srcmorph CLI packaging uses Python 3.8+ to collect original dependency +legal files before the project JAR and fat JAR are assembled. Override `-Dlicense.python=python` +if that is the Python command on the build host. Legal files are under +`META-INF/third-party/dependencies/-/`. +`inventory.json` maps exact JAR hashes/classifiers to preserved files and also lists JARs +without embedded legal material, which require review or supplemental notices. +This source-location statement does not clear unknown proprietary or historical Git findings. diff --git a/docs/dependency-license-review.md b/docs/dependency-license-review.md new file mode 100644 index 0000000..cab3418 --- /dev/null +++ b/docs/dependency-license-review.md @@ -0,0 +1,69 @@ + + +# Dependency license review and distribution + +## Scope and review rules + +The Maven repositories share FOSSA policy 85127 (Standard Bundle Distribution). A flagged +license is a review request. Repository configuration does not grant a license or approve a +FOSSA finding. Decisions below are version-specific evidence, not organization-wide exceptions. + +1. Analyze the actual commit and branch. Never label a PR scan as main. FOSSA's default/tracked + branch is a server setting; `.fossa.yml` does not repair that setting. After changing it, + analyze current main and verify the recorded commit. Streambuffer's previous May 7 master + revision is not evidence for its current dependencies. +2. Keep test dependencies visible. Do not hide GPL/LGPL findings by excluding tests globally. + Internal execution and redistribution of benchmark/test JARs are separate use cases. +3. Inspect original source headers as well as inherited POM licenses. A permissive POM does + not erase another license inside the same component. Record the exact version and file. +4. Preserve dependency license/notice files under artifact-specific paths in fat JARs, since + unpacking all dependencies into one directory can overwrite identically named legal files. +5. Include attribution and license texts missing from upstream binaries before distribution. + Full dependency/source provenance must remain available; bundled notices are not a statement + that every file of an upstream package has been legally cleared. +6. Enter a documented component/version decision in FOSSA only after its usage and notices + have been checked. Broad approval of GPL, AGPL, LGPL or proprietary findings is not appropriate. + +## Verified evidence (2026-10-24) + +| Component | Evidence and decision boundary | +|---|---| +| JMH 1.37, JCStress 0.16 | Parent POMs and binary LICENSE explicitly include GPL v2 with Classpath exception. Review these exact versions for internal test/benchmark use; do not treat two exception aliases as independent obligations or approve arbitrary GPL-only code. | +| JUnit Jupiter 6.1.3, jqwik 1.9.3 | EPL-2.0; test scope. Preserve license notices if distributing these tools. | +| Lincheck 3.7 | POM declares MPL-2.0, but source files `org/jetbrains/kotlinx/lincheck/strategy/managed/MemoryLocation.kt` and `MemoryTracker.kt` explicitly say LGPL-3.0-or-later. The LGPL finding is real; never override the entire component to MPL-only. Internal test use is the reviewed boundary; redistribution requires further review. | +| Logback Classic 1.6.5 | Source headers of `AsyncAppender.java` and `BasicConfigurator.java` explicitly allow EPL-2.0 OR LGPL-2.1. Use the EPL-2.0 alternative for these unchanged components where that grant applies. The extra EPL-1.0 finding still requires its exact matched file; do not silently overwrite it. | +| Jetty EE10 12.1.14 | Parent declares EPL-2.0 and Apache-2.0. Verify the applicable grant and notices of the actual files before recording an alternative-license selection. Used by the separate jllama agent, not the core. | +| JeroMQ 0.6.0 | POM declares MPL-2.0. Runtime in BAF; preserve notices and provide the exact upstream source location for unchanged covered code. | +| jsr305 3.0.2 | POM declares Apache-2.0 but `javax/annotation/concurrent/{GuardedBy,Immutable,NotThreadSafe,ThreadSafe}.java` contain CC-BY-2.5. The attribution finding is valid. | +| jcip-annotations 1.0 | Original sources explicitly contain CC-BY-2.5 and Brian Goetz/Tim Peierls attribution. Source republication must retain the notice. | +| Bouncy Castle provider 1.86 | Source `org/bouncycastle/LICENSE.java` contains the permissive Bouncy Castle copyright/permission/warranty text. Preserve it; a custom FOSSA license name alone does not imply a usage prohibition. | + +Source archives are obtainable at `https://repo.maven.apache.org/maven2/` using the component's +Maven coordinates and exact version (`-sources.jar`). Keep the POM and source archive together +as evidence. A new version requires checking again. + +## Outstanding service decisions + +FOSSA issue approvals/license corrections are SaaS settings, not Git settings. These branches +record evidence and distribution changes; they do not claim that server-side approvals have +been applied. Inspect exact file matches for the historical Git/proprietary findings and the +remaining Logback EPL-1.0 finding. Historical revisions are not changed by deleting current files. + +The unlicensed SevenForums material in BroomCabinet needs a documented redistribution grant +or a separate owner decision to remove it. A LicenseRef is documentation, not permission. + +## Releasing + +Review `THIRD_PARTY.md` per repository, the resolved runtime graph, and the produced archive. +For fat JARs inspect `META-INF/third-party/dependencies/inventory.json` and compare +copied legal entries with the original JAR bytes. The collector reads the resolved runtime +classpath, preserves classifiers and distinguishes equal filenames by content hash. +It never treats a `LICENSE.class` bytecode file as a text license. BAF's assembly profile and +the srcmorph CLI run it before packaging; Python 3.8+ is needed on that build host. +JARs with no legal entries remain explicitly listed. No network requests or approvals are made. +Missing legal material must be supplied explicitly. This check does not replace +source-availability obligations or the review of native third-party components. diff --git a/srcmorph-cli/pom.xml b/srcmorph-cli/pom.xml index f8ca9bc..68ba45a 100644 --- a/srcmorph-cli/pom.xml +++ b/srcmorph-cli/pom.xml @@ -36,6 +36,7 @@ SPDX-License-Identifier: Apache-2.0 + python3 bernardladenthin 8 21 @@ -167,6 +168,19 @@ SPDX-License-Identifier: Apache-2.0 + + + ${project.basedir}/src/main/resources + + + ${project.basedir}/.. + META-INF/third-party/net.ladenthin/${project.artifactId} + + LICENSE + THIRD_PARTY.md + + + @@ -275,6 +289,45 @@ SPDX-License-Identifier: Apache-2.0 + + org.apache.maven.plugins + maven-dependency-plugin + + + runtime-license-classpath + process-classes + build-classpath + + runtime + jar + true + ${project.build.directory}/runtime-license-classpath.txt + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.6.4 + + + collect-runtime-licenses + prepare-package + exec + + ${license.python} + + ${project.basedir}/../.github/collect-third-party-licenses.py + --classpath + ${project.build.directory}/runtime-license-classpath.txt + --output + ${project.build.outputDirectory}/META-INF/third-party/dependencies + + + + + io.github.git-commit-id git-commit-id-maven-plugin diff --git a/srcmorph-maven-plugin/pom.xml b/srcmorph-maven-plugin/pom.xml index a8e1107..9db8417 100644 --- a/srcmorph-maven-plugin/pom.xml +++ b/srcmorph-maven-plugin/pom.xml @@ -252,6 +252,19 @@ SPDX-License-Identifier: Apache-2.0 + + + ${project.basedir}/src/main/resources + + + ${project.basedir}/.. + META-INF/third-party/net.ladenthin/${project.artifactId} + + LICENSE + THIRD_PARTY.md + + + diff --git a/srcmorph/pom.xml b/srcmorph/pom.xml index 1b43d0b..c14d66e 100644 --- a/srcmorph/pom.xml +++ b/srcmorph/pom.xml @@ -229,6 +229,19 @@ SPDX-License-Identifier: Apache-2.0 + + + ${project.basedir}/src/main/resources + + + ${project.basedir}/.. + META-INF/third-party/net.ladenthin/${project.artifactId} + + LICENSE + THIRD_PARTY.md + + + From fa1510873ec5d165f029a7900d81cddb26d783f3 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 14:30:52 +0000 Subject: [PATCH 6/9] Record SevenForums assessment as unresolved without asserting a release blocker --- .github/shared-files.sha256 | 2 +- docs/dependency-license-review.md | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 278d009..714fa53 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -79,6 +79,6 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license -7d82df16062877522da6a3d7a41e4b28f43ce4776ae89b6f7c7933cbb9f41aa3 docs/dependency-license-review.md +0dabc638fcf4a505029e8718fca0d6434e66874d8029cfd9da2626003de6a5d7 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo a6a4d6efe20634a62f860294b3fbd30c71d017a42fc55421ae83b8a7e8aba572 .github/collect-third-party-licenses.py diff --git a/docs/dependency-license-review.md b/docs/dependency-license-review.md index cab3418..f96627e 100644 --- a/docs/dependency-license-review.md +++ b/docs/dependency-license-review.md @@ -53,8 +53,13 @@ record evidence and distribution changes; they do not claim that server-side app been applied. Inspect exact file matches for the historical Git/proprietary findings and the remaining Logback EPL-1.0 finding. Historical revisions are not changed by deleting current files. -The unlicensed SevenForums material in BroomCabinet needs a documented redistribution grant -or a separate owner decision to remove it. A LicenseRef is documentation, not permission. +BroomCabinet records the SevenForums registry files as finding `BROOM-LIC-001` in its +`THIRD_PARTY.md`, with status unresolved. The reviewed contents are predominantly Microsoft +Windows default configuration; no substantial independent program logic was identified. +Whether protectable third-party contributions exist and require a redistribution grant remains +unresolved. The files and provenance markings are retained pending assessment; no release +blocker has been established from this finding. A LicenseRef documents provenance, not a grant +or scanner approval. This finding concerns BroomCabinet only. ## Releasing From ef43f388bcb2501b007f0b333093e431934c3091 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 15:19:06 +0000 Subject: [PATCH 7/9] Preserve COPYRIGHT documents in shared runtime license collector --- .github/collect-third-party-licenses.py | 2 +- .github/shared-files.sha256 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/collect-third-party-licenses.py b/.github/collect-third-party-licenses.py index df33f5d..d337a08 100644 --- a/.github/collect-third-party-licenses.py +++ b/.github/collect-third-party-licenses.py @@ -28,7 +28,7 @@ def legal_file(name): if path.suffix.lower() in {".class", ".java", ".kt", ".png", ".jpg", ".gif", ".jar"}: return False return (any(part.lower() in {"licenses", "licences"} for part in path.parts) - or re.fullmatch(r"(?:licen[sc]e|notice|copying)(?:[-_.].*)?", lower) is not None) + or re.fullmatch(r"(?:licen[sc]e|notice|copying|copyright)(?:[-_.].*)?", lower) is not None) def collect(classpath, output): diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 714fa53..f0d45c8 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -81,4 +81,4 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license 0dabc638fcf4a505029e8718fca0d6434e66874d8029cfd9da2626003de6a5d7 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo -a6a4d6efe20634a62f860294b3fbd30c71d017a42fc55421ae83b8a7e8aba572 .github/collect-third-party-licenses.py +31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py From d7326c9e86b2cb5543dcfdfd172145313946ffb5 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 15:23:53 +0000 Subject: [PATCH 8/9] Address review feedback on docs, badge links and signature subtests --- .github/buildcheck/tests/test_snapshot.py | 6 ++++-- .github/shared-files.sha256 | 4 ++-- README.md | 2 +- docs/dependency-license-review.md | 10 +++------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/buildcheck/tests/test_snapshot.py b/.github/buildcheck/tests/test_snapshot.py index bf148a6..4b74f8c 100644 --- a/.github/buildcheck/tests/test_snapshot.py +++ b/.github/buildcheck/tests/test_snapshot.py @@ -101,9 +101,11 @@ def test_late_finishing_older_commit_cannot_overwrite_newer_snapshot(self): self.assertTrue(all(c[0] == "GET" for c in self.gh.calls)) def test_empty_signature_or_orphan_also_prevents_promotion(self): - for name in ("new.json.asc", "orphan.jar.asc"): + for name, content in (("new.json.asc", ""), ("orphan.jar.asc", "signature")): with self.subTest(name=name): - (self.assets / name).write_text("") + (self.assets / "new.json.asc").write_text("signature") + (self.assets / name).write_text(content) + self.gh.calls.clear() with self.assertRaisesRegex(ValueError, "collection is incomplete"): self.publish() self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index f0d45c8..376673e 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -74,11 +74,11 @@ eaa2911c2c754ae0b1c9acff9aaba347e5d7fdfb841b4b5f2d12b271cb71d7fe .github/workfl b9bbf8762e5898709d69a2509c64750e3acf41dc0a69426578773714d263d365 .github/buildcheck/releaseassets.py bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py 8e9f5870bae1d9e903ef35db9366cd680fcbb459076d1fdb0390248ab6b1cc30 .github/buildcheck/tests/test_releaseassets.py -4bab88300dc2bc06d33d659a2118d81070118e65300b9cfbacdc0313d11bee43 .github/buildcheck/tests/test_snapshot.py +8b6ae70f43560a0095b6f16012b80ed694b07633c1cbde94dbe8c21c529461ea .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license -0dabc638fcf4a505029e8718fca0d6434e66874d8029cfd9da2626003de6a5d7 docs/dependency-license-review.md +0ed4e1ebe554c9dabf455ef1bfb3bebc292d010138f2190abb39d3aba9ea80f5 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo 31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py diff --git a/README.md b/README.md index 219a73f..5be638f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ **AI:** [![Claude](https://img.shields.io/badge/Claude-D97757?logo=claude&logoColor=fff)](#) -![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/bernardladenthin/srcmorph?utm_source=oss&utm_medium=github&utm_campaign=bernardladenthin%2Fsrcmorph&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews) +[![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/bernardladenthin/srcmorph?utm_source=oss&utm_medium=github&utm_campaign=bernardladenthin%2Fsrcmorph&labelColor=171717&color=FF570A&label=CodeRabbit+Reviews)](https://coderabbit.ai) **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) diff --git a/docs/dependency-license-review.md b/docs/dependency-license-review.md index f96627e..76b82d0 100644 --- a/docs/dependency-license-review.md +++ b/docs/dependency-license-review.md @@ -53,13 +53,9 @@ record evidence and distribution changes; they do not claim that server-side app been applied. Inspect exact file matches for the historical Git/proprietary findings and the remaining Logback EPL-1.0 finding. Historical revisions are not changed by deleting current files. -BroomCabinet records the SevenForums registry files as finding `BROOM-LIC-001` in its -`THIRD_PARTY.md`, with status unresolved. The reviewed contents are predominantly Microsoft -Windows default configuration; no substantial independent program logic was identified. -Whether protectable third-party contributions exist and require a redistribution grant remains -unresolved. The files and provenance markings are retained pending assessment; no release -blocker has been established from this finding. A LicenseRef documents provenance, not a grant -or scanner approval. This finding concerns BroomCabinet only. +Keep repository-specific assessments in that repository's `THIRD_PARTY.md`, including +affected files, evidence, review status and the current decision. A LicenseRef identifier +alone does not establish permission or scanner approval. ## Releasing From 22879cec69dfd2643997a6e7bd0bcce3e0919050 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 16:06:53 +0000 Subject: [PATCH 9/9] Gate signed publications locally and decouple Central and GitHub uploads --- .github/buildcheck/publication.py | 292 +++++++++++++++ .github/buildcheck/releaseassets.py | 15 +- .github/buildcheck/tests/test_publication.py | 126 +++++++ .../buildcheck/tests/test_releaseassets.py | 6 +- .github/publication.py | 14 + .github/release-gate-exemptions.txt | 2 + .github/shared-files.sha256 | 14 +- .github/workflows/publish.yml | 342 ++++++++---------- docs/publication-preflight.md | 84 +++++ pom.xml | 6 +- 10 files changed, 696 insertions(+), 205 deletions(-) create mode 100644 .github/buildcheck/publication.py create mode 100644 .github/buildcheck/tests/test_publication.py create mode 100644 .github/publication.py create mode 100644 docs/publication-preflight.md diff --git a/.github/buildcheck/publication.py b/.github/buildcheck/publication.py new file mode 100644 index 0000000..a8d0db5 --- /dev/null +++ b/.github/buildcheck/publication.py @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Freeze signed publication bytes once; independent consumers never rebuild them.""" +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import subprocess +import tempfile +import xml.etree.ElementTree as ET +import zipfile + +from .releaseassets import configuration, projects, signature_gaps, _classifiers, _format, _inside + +PORTAL = "https://central.sonatype.com/api/v1/publisher/upload" +SNAPSHOTS = "https://central.sonatype.com/repository/maven-snapshots/" +DEPLOY_PLUGIN = "3.1.4" + + +def digest(path): + h = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def safe_path(root, name): + p = PurePosixPath(name) + if not name or p.is_absolute() or ".." in p.parts or "\\" in name or ":" in name: + raise ValueError("unsafe publication path: " + name) + target = root / name + if target.is_symlink() or any(parent.is_symlink() for parent in target.parents): + raise ValueError("publication symlink: " + name) + return target + + +def expected_central(root, config, modules, *, only_central=True): + names = set() + for _, artifact, version, packaging, reactor in modules: + base = f"{artifact}-{version}" + names.add(base + ".pom") + if packaging != "pom": + names.update(base + suffix + ".jar" for suffix in ("", "-sources", "-javadoc")) + if reactor and config.get("maven", {}).get("cyclonedx", False): + names.add(base + "-cyclonedx.json") + for rule in config.get("files", []): + if only_central and not rule.get("central", False): + continue + for classifier in _classifiers(root, rule.get("classifiers", [""])): + matches = list(root.glob(_format(rule["source"], modules[0][2], classifier))) + if len(matches) != 1: + raise ValueError("missing/ambiguous Central input: " + rule["source"]) + names.add(_format(rule["name"], modules[0][2], classifier) if "name" in rule else matches[0].name) + return names + + +def prepare(root, assets, destination): + """Gate both publishers on exact membership, valid signatures and frozen checksums.""" + root, assets, destination = Path(root).resolve(), Path(assets).resolve(), Path(destination).resolve() + if destination.exists(): + raise ValueError("publication destination must not exist") + missing, errors = signature_gaps(assets) + if missing: + raise ValueError("incomplete publication: " + "; ".join(errors)) + config = configuration(root) + publication = config.get("publication", {}) + modules = projects(root, config.get("maven", {}).get("standalone", [])) + version = modules[0][2] + channel = "snapshot" if version.endswith("-SNAPSHOT") else "release" + staging = "central-deferred" if channel == "snapshot" else "central-staging" + roots = publication.get("central_roots", ["target/{staging}"]) + central = {} + for item in roots: + folder = _inside(root, item.format(staging=staging)) + if not folder.is_dir(): + raise ValueError("missing Central staging: " + str(folder)) + for path in folder.rglob("*"): + if not path.is_file(): + continue + relative = path.relative_to(folder).as_posix() + safe_path(folder, relative) + if not re.search(r"\.(?:pom|jar|aar|json)(?:\.asc)?$", path.name): + continue # Maven metadata/checksums are regenerated, never mistaken for payload. + parts = PurePosixPath(relative).parts + if len(parts) < 4 or parts[-2] != version: + raise ValueError("unexpected Central coordinates: " + relative) + if relative in central and digest(central[relative]) != digest(path): + raise ValueError("conflicting Central input: " + relative) + central[relative] = path + expected = expected_central(root, config, modules) + actual = {path.name for path in central.values() if not path.name.endswith(".asc")} + if actual != expected or len(central) != 2 * len(expected): + raise ValueError("Central membership mismatch; missing=" + str(sorted(expected - actual)) + + "; unexpected=" + str(sorted(actual - expected))) + github_expected = expected_central(root, config, modules, only_central=False) + github_actual = {p.name for p in assets.iterdir() if not p.name.endswith(".asc")} + if github_actual != github_expected: + raise ValueError("GitHub membership does not match the release contract") + # Require the Central pair to match GitHub; verify signatures on the frozen copy. + for path in sorted(assets.iterdir()): + if path.is_symlink() or not path.is_file(): + raise ValueError("non-flat publication input: " + str(path)) + for name, path in central.items(): + if not (assets / path.name).is_file() or digest(assets / path.name) != digest(path): + raise ValueError("Central/GitHub bytes differ: " + name) + # Build a temporary package, then promote it locally only after all checks succeed. + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="publication-", dir=destination.parent) as tmp: + package = Path(tmp) / "package" + (package / "github").mkdir(parents=True) + for path in assets.iterdir(): + shutil.copyfile(path, package / "github" / path.name) + for relative, path in central.items(): + target = safe_path(package / "central", relative) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, target) + for path in sorted((package / "github").iterdir()): + if not path.name.endswith(".asc"): + subprocess.run(["gpg", "--batch", "--verify", str(path) + ".asc", str(path)], check=True) + for path in (package / "central").rglob("*"): + if path.is_file() and digest(path) != digest(package / "github" / path.name): + raise ValueError("frozen Central/GitHub files differ: " + path.name) + manifest = {"schema": 1, "version": version, "channel": channel, + "commit": os.environ.get("GITHUB_SHA", ""), "files": {}} + for path in sorted(package.rglob("*")): + if path.is_file(): + manifest["files"][path.relative_to(package).as_posix()] = { + "sha256": digest(path), "size": path.stat().st_size} + (package / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + shutil.move(str(package), destination) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write("complete=true\n") + print(f"Publication ready: {version}; {len(expected)} Central artifacts; all signatures verified") + + +def verify(package): + """Transport integrity check; never regenerate or re-sign publication files.""" + package = Path(package).resolve() + manifest = json.loads((package / "manifest.json").read_text()) + if manifest.get("schema") != 1 or manifest.get("channel") not in ("snapshot", "release"): + raise ValueError("unsupported publication manifest") + if os.environ.get("GITHUB_SHA") and manifest.get("commit") != os.environ["GITHUB_SHA"]: + raise ValueError("publication belongs to a different commit") + actual = {p.relative_to(package).as_posix() for p in package.rglob("*") if p.is_file()} + if actual != set(manifest["files"]) | {"manifest.json"}: + raise ValueError("publication file membership changed") + for relative, info in manifest["files"].items(): + path = safe_path(package, relative) + if not path.is_file() or path.stat().st_size != info["size"] or digest(path) != info["sha256"]: + raise ValueError("publication bytes changed: " + relative) + return manifest + + +def snapshot_pom(package, destination, url=SNAPSHOTS): + """Generate transfer-only deploy-file executions, including all signature extensions. + + Maven handles timestamped snapshot names and metadata in one request per component. + There is no project build, dependency resolution, compilation or signing phase. + """ + manifest = verify(package) + if manifest["channel"] != "snapshot": + raise ValueError("snapshot transfer requires a SNAPSHOT version") + root = ET.Element("project", xmlns="http://maven.apache.org/POM/4.0.0") + def child(parent, name, text): + node = ET.SubElement(parent, name) + node.text = str(text) + return node + for name, value in [("modelVersion", "4.0.0"), ("groupId", "local.publication"), + ("artifactId", "snapshot-transfer"), ("version", "1"), ("packaging", "pom")]: + child(root, name, value) + plugin = ET.SubElement(ET.SubElement(ET.SubElement(root, "build"), "plugins"), "plugin") + child(plugin, "groupId", "org.apache.maven.plugins") + child(plugin, "artifactId", "maven-deploy-plugin") + child(plugin, "version", DEPLOY_PLUGIN) + executions = ET.SubElement(plugin, "executions") + central = Path(package).resolve() / "central" + for i, pom in enumerate(sorted(central.rglob("*.pom"))): + parts = pom.relative_to(central).parts + group, artifact, version = ".".join(parts[:-3]), parts[-3], parts[-2] + xml = ET.parse(pom).getroot() + packaging = xml.findtext("{*}packaging", "jar") + extension = "jar" if packaging == "maven-plugin" else packaging + base = f"{artifact}-{version}" + # A POM-only component has a single unclassified signature. Deploy-file + # treats an empty classifiers parameter as absent, so use that signature + # as the primary artifact and let pomFile attach its signed POM alongside. + transfer_packaging = "pom.asc" if packaging == "pom" else packaging + main = pom.parent / (f"{base}.pom.asc" if packaging == "pom" else f"{base}.{extension}") + if not main.is_file(): + raise ValueError("missing primary Central artifact: " + str(main)) + execution = ET.SubElement(executions, "execution") + child(execution, "id", f"transfer-{i}") + child(execution, "phase", "validate") + child(ET.SubElement(execution, "goals"), "goal", "deploy-file") + config = ET.SubElement(execution, "configuration") + for name, value in [("repositoryId", "central"), ("url", url), ("file", main), + ("pomFile", pom), ("groupId", group), ("artifactId", artifact), + ("version", version), ("packaging", transfer_packaging), ("generatePom", "false")]: + child(config, name, value) + attached = [] + for path in sorted(pom.parent.iterdir()): + if path in (pom, main): + continue + suffix = path.name[len(base):] if path.name.startswith(base) else "" + match = re.fullmatch(r"(?:-(.+))?\.(pom|jar|aar|json)(\.asc)?", suffix) + if not match: + raise ValueError("invalid attached artifact: " + path.name) + classifier = match[1] or "" + kind = match[2] + (match[3] or "") + if "," in str(path): + raise ValueError("comma in Maven transfer path") + attached.append((str(path), classifier, kind)) + if attached: + for name, column in [("files", 0), ("classifiers", 1), ("types", 2)]: + child(config, name, ",".join(item[column] for item in attached)) + ET.ElementTree(root).write(destination, encoding="utf-8", xml_declaration=True) + + +def upload(package): + """Transfer frozen files, accepting Portal upload without validation polling.""" + package = Path(package).resolve() + manifest = verify(package) + if manifest["channel"] == "snapshot": + with tempfile.TemporaryDirectory(prefix="snapshot-transfer-") as tmp: + pom = Path(tmp) / "pom.xml" + snapshot_pom(package, pom) + subprocess.run(["mvn", "-B", "--no-transfer-progress", "-Dmaven.repository.credentialScope=id", + "-f", str(pom), "validate"], check=True) + return + with tempfile.TemporaryDirectory(prefix="central-transfer-") as tmp: + bundle = Path(tmp) / "central-bundle.zip" + with zipfile.ZipFile(bundle, "w", zipfile.ZIP_DEFLATED) as archive: + for relative in manifest["files"]: + if not relative.startswith("central/"): + continue + path = package / relative + name = relative[len("central/"):] + hashes = {a: hashlib.new(a) for a in ("md5", "sha1", "sha256", "sha512")} + with path.open("rb") as source, archive.open(name, "w", force_zip64=True) as target: + for block in iter(lambda: source.read(1024 * 1024), b""): + target.write(block) + for h in hashes.values(): + h.update(block) + if not name.endswith(".asc"): + for algorithm, h in hashes.items(): + archive.writestr(name + "." + algorithm, h.hexdigest()) + token = base64.b64encode((os.environ["MAVEN_USERNAME"] + ":" + + os.environ["MAVEN_PASSWORD"]).encode()).decode() + config = Path(tmp) / "curl.conf" + config.touch(mode=0o600) + config.write_text('header = "Authorization: Bearer ' + token + '"\n') + response = subprocess.run(["curl", "--fail-with-body", "--silent", "--show-error", + "--connect-timeout", "30", "--max-time", "1800", "--config", str(config), + "-F", "bundle=@" + str(bundle), PORTAL + "?publishingType=AUTOMATIC"], + text=True, capture_output=True) + if response.returncode: + raise RuntimeError("Central upload failed: " + response.stderr + response.stdout) + deployment = response.stdout.strip().strip('"') + if not re.fullmatch(r"[A-Za-z0-9_-]+", deployment): + raise RuntimeError("Central returned no valid deployment receipt") + print("Central accepted deployment: " + deployment) + print("Validation and publication continue asynchronously at Central; no polling.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + prepare_parser = commands.add_parser("prepare") + prepare_parser.add_argument("assets", type=Path) + prepare_parser.add_argument("package", type=Path) + verify_parser = commands.add_parser("verify") + verify_parser.add_argument("package", type=Path) + verify_parser.add_argument("--export-github", type=Path) + upload_parser = commands.add_parser("upload") + upload_parser.add_argument("package", type=Path) + args = parser.parse_args() + if args.command == "prepare": + prepare(Path(__file__).resolve().parents[2], args.assets, args.package) + elif args.command == "verify": + verify(args.package) + if args.export_github: + if args.export_github.exists(): + raise ValueError("GitHub export destination must not exist") + shutil.copytree(args.package / "github", args.export_github) + else: + upload(args.package) diff --git a/.github/buildcheck/releaseassets.py b/.github/buildcheck/releaseassets.py index 36d36fe..de10377 100644 --- a/.github/buildcheck/releaseassets.py +++ b/.github/buildcheck/releaseassets.py @@ -33,9 +33,15 @@ def configuration(root, path=None): """Read the explicit contract; unknown keys fail instead of silently dropping assets.""" path = Path(path) if path is not None else Path(root) / ".github/release-assets.json" config = json.loads(path.read_text()) - _keys(config, ("schema", "maven", "files"), "release asset configuration") + _keys(config, ("schema", "maven", "files", "publication"), "release asset configuration") if type(config.get("schema")) is not int or config["schema"] != 1: raise ValueError("release asset configuration requires schema: 1") + publication = config.get("publication", {}) + _keys(publication, ("central_roots",), "publication configuration") + if "central_roots" in publication and (not isinstance(publication["central_roots"], list) + or not publication["central_roots"] + or any(not isinstance(path, str) or not path for path in publication["central_roots"])): + raise ValueError("publication.central_roots must be a nonempty list of paths") maven = config.get("maven", {}) _keys(maven, ("standalone", "cyclonedx"), "maven configuration") if not isinstance(maven.get("standalone", []), list): @@ -44,6 +50,9 @@ def configuration(root, path=None): raise ValueError("maven.cyclonedx must be a boolean") if not isinstance(config.get("files", []), list): raise ValueError("files must be a list") + for rule in config.get("files", []): + if isinstance(rule, dict) and "central" in rule and type(rule["central"]) is not bool: + raise ValueError("file rule central must be a boolean") return config @@ -111,7 +120,7 @@ def file_classifiers(root, index, config_path=None): item = config.get("files", [])[index] except IndexError as error: raise ValueError(f"no file rule at index {index}") from error - _keys(item, ("source", "name", "classifiers"), "file rule") + _keys(item, ("source", "name", "classifiers", "central"), "file rule") return _classifiers(Path(root).resolve(), item.get("classifiers", [""])) @@ -138,7 +147,7 @@ def collect(root, destination, config_path=None, *, maven_only=False, maven_asse expected.append((Path(maven_assets) / name if maven_assets is not None else source, name)) for item in [] if maven_only else config.get("files", []): - _keys(item, ("source", "name", "classifiers"), "file rule") + _keys(item, ("source", "name", "classifiers", "central"), "file rule") for classifier in _classifiers(root, item.get("classifiers", [""])): pattern = _format(item.get("source"), version, classifier) _inside(root, pattern) diff --git a/.github/buildcheck/tests/test_publication.py b/.github/buildcheck/tests/test_publication.py new file mode 100644 index 0000000..9fd5f3b --- /dev/null +++ b/.github/buildcheck/tests/test_publication.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Publication gate and transport invariants; no signing key or remote writes.""" +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import xml.etree.ElementTree as ET + +from buildcheck.publication import prepare, verify, snapshot_pom +from buildcheck import workflow +from buildcheck.tests.helpers import REPO + + +class PublicationTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + (self.root / '.github').mkdir() + (self.root / '.github/release-assets.json').write_text('{"schema":1}') + self.pom = '4.0.0local.testdemo1-SNAPSHOT' + (self.root / 'pom.xml').write_text(self.pom) + self.assets = self.root / 'assets' + self.assets.mkdir() + self.stage = self.root / 'target/central-deferred/local/test/demo/1-SNAPSHOT' + self.stage.mkdir(parents=True) + for suffix in ['.pom', '.jar', '-sources.jar', '-javadoc.jar']: + name = 'demo-1-SNAPSHOT' + suffix + for extra in ['', '.asc']: + data = self.pom if suffix == '.pom' and not extra else name + extra + (self.assets / (name + extra)).write_text(data) + (self.stage / (name + extra)).write_text(data) + self.package = self.root / 'package' + self.signatures = patch('buildcheck.publication.subprocess.run') + self.run = self.signatures.start() + self.addCleanup(self.signatures.stop) + quiet = patch('builtins.print') + quiet.start() + self.addCleanup(quiet.stop) + + def freeze(self): + prepare(self.root, self.assets, self.package) + + def test_freezes_once_and_verifies_every_signature(self): + self.freeze() + self.assertEqual(self.run.call_count, 4) + self.assertEqual(verify(self.package)['channel'], 'snapshot') + (self.assets / 'demo-1-SNAPSHOT.jar').write_text('later rebuild') + self.assertNotEqual((self.package / 'github/demo-1-SNAPSHOT.jar').read_text(), 'later rebuild') + verify(self.package) + + def test_missing_signature_or_wrong_central_bytes_block_both_publishers(self): + sig = self.assets / 'demo-1-SNAPSHOT.jar.asc' + sig.unlink() + with self.assertRaisesRegex(ValueError, 'incomplete'): + self.freeze() + self.assertFalse(self.package.exists()) + sig.write_text('signature') + with self.assertRaisesRegex(ValueError, 'bytes differ'): + self.freeze() + self.assertFalse(self.package.exists()) + + def test_invalid_signature_blocks_package_creation(self): + import subprocess + self.run.side_effect = subprocess.CalledProcessError(1, 'gpg') + with self.assertRaises(subprocess.CalledProcessError): + self.freeze() + self.assertFalse(self.package.exists()) + + def test_transport_tampering_and_extra_files_are_rejected(self): + self.freeze() + unexpected = self.package / 'unexpected.txt' + unexpected.write_text('extra') + with self.assertRaisesRegex(ValueError, 'membership'): + verify(self.package) + unexpected.unlink() + (self.package / 'github/demo-1-SNAPSHOT.jar').write_text('tampered') + with self.assertRaisesRegex(ValueError, 'bytes changed'): + verify(self.package) + + def test_snapshot_transfer_retains_signature_extensions(self): + self.freeze() + target = self.root / 'transfer.xml' + snapshot_pom(self.package, target, 'file:///unused-local-diagnostic') + xml = ET.parse(target).getroot() + config = xml.find('.//{*}configuration') + self.assertIn('jar.asc', config.findtext('{*}types')) + self.assertIn('pom.asc', config.findtext('{*}types')) + self.assertEqual(xml.findtext('.//{*}phase'), 'validate') + self.assertEqual(config.findtext('{*}generatePom'), 'false') + + def test_pom_only_component_transfers_signature_and_pom_together(self): + pom = self.pom.replace('', 'pom') + (self.root / 'pom.xml').write_text(pom) + for directory in (self.assets, self.stage): + for path in directory.iterdir(): + if '.jar' in path.name: + path.unlink() + (directory / 'demo-1-SNAPSHOT.pom').write_text(pom) + self.freeze() + target = self.root / 'pom-transfer.xml' + snapshot_pom(self.package, target) + config = ET.parse(target).getroot().find('.//{*}configuration') + self.assertEqual(config.findtext('{*}packaging'), 'pom.asc') + self.assertTrue(config.findtext('{*}file').endswith('.pom.asc')) + self.assertTrue(config.findtext('{*}pomFile').endswith('.pom')) + self.assertIsNone(config.find('{*}files')) + + def test_publishers_depend_only_on_the_same_checked_handoff(self): + jobs = workflow.parse((Path(REPO) / '.github/workflows/publish.yml').read_text()) + for channel in ['snapshot', 'release']: + prep = jobs['publish-' + channel] + central = jobs['central-' + channel] + github_name = 'github-snapshot' if channel == 'snapshot' else ('github-release' if 'github-release' in jobs else 'github-release-signed') + github = jobs[github_name] + self.assertEqual(central.needs, ['publish-' + channel]) + self.assertEqual(github.needs, ['publish-' + channel]) + self.assertIn('steps.preflight.outputs.complete', prep.text) + self.assertLess(prep.text.index('publication.py prepare'), prep.text.index('uses: actions/upload-artifact@')) + self.assertIn('-DskipUpload=true deploy', prep.text) + self.assertNotIn('publishAllPublicationsToCentralSnapshotsRepository', prep.text) + self.assertIn('publication.py upload publication-package', central.text) + self.assertIn('publication.py verify publication-package', github.text) + self.assertNotIn('Fail if anything', github.text) diff --git a/.github/buildcheck/tests/test_releaseassets.py b/.github/buildcheck/tests/test_releaseassets.py index a4ed72a..8908fbc 100644 --- a/.github/buildcheck/tests/test_releaseassets.py +++ b/.github/buildcheck/tests/test_releaseassets.py @@ -188,12 +188,12 @@ def test_current_repository_workflow_passes_collection_result_and_checks_both_up for kind in ("snapshot", "release"): job = jobs[f"publish-{kind}"].text self.assertIn(f"python3 .github/collect-release-assets.py signed-{kind}-assets", job) - self.assertIn("assets_complete: $" + "{{ steps.collect.outputs.complete }}", job) + self.assertIn("assets_complete: $" + "{{ steps.preflight.outputs.complete }}", job) self.assertIn(" id: collect", job) attach = jobs["github-snapshot" if kind == "snapshot" else "github-release" if "github-release" in jobs else "github-release-signed"].text - self.assertIn(f"python3 .github/report-asset-signatures.py {'snapshot' if kind == 'snapshot' else 'release'}-assets", attach) - self.assertLess(attach.index("id: signatures"), attach.index(" - name: Fail if anything")) + self.assertIn(f"python3 .github/publication.py verify publication-package --export-github {kind}-assets", attach) + self.assertNotIn("Fail if anything", attach) if configuration(REPO).get("maven", {}).get("cyclonedx", False): for kind in ("snapshot", "release"): self.assertIn(f"signed-{kind}-assets/*.json", jobs[f"publish-{kind}"].text) diff --git a/.github/publication.py b/.github/publication.py new file mode 100644 index 0000000..a1dd8cf --- /dev/null +++ b/.github/publication.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Prepare, verify or upload a signed publication without rebuilding its files.""" +import subprocess +import sys +from buildcheck.publication import main + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError, RuntimeError, KeyError, subprocess.CalledProcessError) as error: + print(f"::error::{error}") + sys.exit(1) diff --git a/.github/release-gate-exemptions.txt b/.github/release-gate-exemptions.txt index 5663e04..bfbdfa7 100644 --- a/.github/release-gate-exemptions.txt +++ b/.github/release-gate-exemptions.txt @@ -14,3 +14,5 @@ github-snapshot: runs after publishing; attaches the assets to the rolling snaps github-release: runs after publishing; attaches the signed assets to the tag release verify-signing-key: red by design where the secret is withheld (forks, other contributors' branches); the publish jobs sign with the same key and fail on it themselves verify-signing-key-gradle: same as verify-signing-key, for the Gradle/BouncyCastle signing path +central-snapshot: Independent post-preflight Central transfer; must not gate the GitHub publication path. +central-release: Independent post-preflight Central transfer; must not gate the GitHub publication path. diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 376673e..061eb81 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -69,11 +69,11 @@ e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workfl 09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot 51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workflows/publish.yml#verify-signing-key-gradle -f28843b1fc88b8876dfc6cc0cd451e5003087e904f4f20cefcf7c645e7ebea44 .github/workflows/publish.yml#github-snapshot -eaa2911c2c754ae0b1c9acff9aaba347e5d7fdfb841b4b5f2d12b271cb71d7fe .github/workflows/publish.yml#github-release -b9bbf8762e5898709d69a2509c64750e3acf41dc0a69426578773714d263d365 .github/buildcheck/releaseassets.py +9e7609a8280585c13baa5418afc52e09d12753fea2c1f7d2d7b392e8a74e67b6 .github/workflows/publish.yml#github-snapshot +52f1f7c99a604f40fb3f3685909c060b2c270e3f3dd5d28f3119be4b3f5d4052 .github/workflows/publish.yml#github-release +ed1ba202b9d81a00893931a090112bfb28770a514030a5c08d5441c83369f8b7 .github/buildcheck/releaseassets.py bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py -8e9f5870bae1d9e903ef35db9366cd680fcbb459076d1fdb0390248ab6b1cc30 .github/buildcheck/tests/test_releaseassets.py +8bc93d2ca737d7641abdd6311444233d9603c188dfd9d89d785b70cb92fb0f93 .github/buildcheck/tests/test_releaseassets.py 8b6ae70f43560a0095b6f16012b80ed694b07633c1cbde94dbe8c21c529461ea .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py @@ -82,3 +82,9 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 0ed4e1ebe554c9dabf455ef1bfb3bebc292d010138f2190abb39d3aba9ea80f5 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo 31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py +4d16ef3f5f4fa5fa39d191b030e1cbff3c943c67b33ae9b392537e4b08f35004 .github/buildcheck/publication.py +9180fac217c75b3e11f67e244b9171f447d2c21665b1802574b57b97019f7173 .github/publication.py +c8d911dfd539016ad404460a3879d0b1d9635ba5c4e0843e9ce98acd5edb3f17 .github/buildcheck/tests/test_publication.py +b6a8429621c3aa5af71c4d4a63bb729cb220c858b74ba9f3d1ba14813602fddb docs/publication-preflight.md +b3cb68727cc2b0a4e6831b3b9301b6e54420dc6c52aa068dcd8b2b38a6c2308b .github/workflows/publish.yml#central-snapshot +5bf3977c27f1a5ae7f9894cc6e1b26b014ea3ee6bb854fe89daa9cbb26785a23 .github/workflows/publish.yml#central-release diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 57f2bd8..54caf9c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -530,16 +530,22 @@ jobs: run: echo "Confirmed on tag ${{ github.ref }}" publish-snapshot: - name: Publish Snapshot to Central - needs: [check-snapshot, code-style, smoke-fatjar, plugin-it, shared-files, vmlens] + name: Prepare and verify signed snapshot artifacts + needs: + - check-snapshot + - code-style + - smoke-fatjar + - plugin-it + - shared-files + - vmlens if: needs.check-snapshot.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central outputs: - assets_complete: ${{ steps.collect.outputs.complete }} + assets_complete: ${{ steps.preflight.outputs.complete }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad with: java-version-file: .java-version distribution: temurin @@ -555,148 +561,94 @@ jobs: VERSION=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n1) echo "Resolved project version: $VERSION" case "$VERSION" in - *-SNAPSHOT) echo "OK: -SNAPSHOT version, continuing snapshot deploy." ;; + *-SNAPSHOT) echo "OK: -SNAPSHOT version, continuing local snapshot staging." ;; *) echo "::error::Refusing to publish non-SNAPSHOT version '$VERSION' from the snapshot job. Snapshot publishing requires a -SNAPSHOT version; releases go through the v* tag path."; exit 1 ;; esac - # Informational only (nothing depends on it): logs the effective POM with the same - # profile as the deploy below, so the resolved central-publishing configuration - # (waitUntil/waitMaxTime etc.) is visible for debugging. - name: Show effective POM (debug) - run: mvn --batch-mode --no-transfer-progress -P release help:effective-pom - # -Dmaven.repository.credentialScope=id: Maven 3.10 (on the runners since 2026-10) sends a server's - # credentials only to its declared origins, and for the id `central` that is repo.maven.apache.org -- - # an upload to central.sonatype.com then goes out anonymously and fails with HTTP 401. - # Restores id-only matching; replace with setup-java's `mvn-server-repository-origins: - # central:https://central.sonatype.com` once a release of setup-java ships that input. - - name: Deploy snapshot - run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release deploy -DskipTests + run: mvn --batch-mode --no-transfer-progress -P release -DskipUpload=true help:effective-pom + - name: Stage snapshot + run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release -DskipUpload=true deploy -DskipTests env: - MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - # Runs even when the deploy step failed: a Central publish-poll timeout reds the - # job *after* the bundle was uploaded (and typically published server-side), while - # the signed jars + .asc files already exist in each module's target/ (signing happens at - # verify). Collecting on failure lets the github-snapshot job still attach them. - # NOTE: the fat jar (jar-with-dependencies) is EXCLUDED here — it is no longer attached/ - # deployed to Central (srcmorph-cli/pom.xml sets false), so it is unsigned - # in target/ at this point. It is built + signed + collected by the next step instead. This - # step runs BEFORE that one so the deploy-signed plain/sources/javadoc jars are captured - # before the classifier loop rebuilds (and clobbers) srcmorph-cli/target. - name: Collect signed Central artifacts (plain/sources/javadoc; fat jars handled next) - if: ${{ !cancelled() }} run: python3 .github/collect-release-assets.py central-assets --maven-only - # Build one srcmorph-cli fat jar per net.ladenthin:llama GPU module jar + the default - # all-platform CPU fat jar, GPG-sign each, and add them to the GitHub-Release assets. - # These are GitHub-only (never Maven Central): every jar carries the multi-platform CPU - # natives (llama-platform), and a classifier jar adds one ggml backend module on top (since - # llama 6.0.0 a GPU natives jar holds only that module), which ggml loads next to the CPU - # library when the vendor runtime is present and skips otherwise. Signing uses the key - # setup-java already imported (armored detached .asc, same shape Central produces). - name: Build, sign & verify classifier fat jars (GitHub-only assets, never Central) - if: ${{ !cancelled() }} env: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - run: | - set -euo pipefail - # Source of truth: java-llama.cpp's .github/natives.csv for the pinned ${llama.version}: - # every row of kind `module` (a GPU backend module, additive to the library jar of its - # platform) except the Android one, which no `java -jar` can use; the cpu-*/metal-* - # library jars come with llama-platform. Each jar here is ADDED to the default CPU - # natives (-Dllama.classifier activates the gpu-natives profile), so every classifier - # fat jar keeps the CPU fallback: ggml loads the module when its runtime is installed. - python3 .github/collect-release-assets.py --list-classifiers 1 > fatjar-classifiers.txt - mapfile -t CLASSIFIERS < fatjar-classifiers.txt - build() { mvn -B --no-transfer-progress -pl srcmorph-cli -am -DskipTests -Dmaven.javadoc.skip=true -Dllama.classifier="$1" package; } - for c in "${CLASSIFIERS[@]}"; do - build "$c" - f=$(ls srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies.jar) - cp "$f" "${f%.jar}-${c}.jar" - done - # default CPU build LAST so the unsuffixed fat jar is the all-platform CPU variant - build "" - # Collect every fat jar variant, then GPG-sign each via the cross-repo shared script - # (.github/sign-fatjars.sh; java-llama.cpp dropped its copy with its fat jars) — a .asc per fat jar. - mkdir -p fatjar-assets - cp srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies*.jar fatjar-assets/ - bash .github/sign-fatjars.sh fatjar-assets - # Structural check of what the loop just produced. The smoke-fatjar job LAUNCHES the - # default jar; the classifier jars cannot be covered that way (a GitHub runner has no - # such device, and the only command that would load the native library is a real - # generation), so this asserts instead that each jar is the artifact its name claims -- - # one jar per classifier, the backend directory the name promises, and every default - # CPU native kept next to it. The backend directory is the point: if -Dllama.classifier= - # ever stops being wired through, the loop silently ships copies of the CPU build under - # GPU names and nothing else in this pipeline notices. - bash .github/verify-classifier-fatjars.sh fatjar-assets "${CLASSIFIERS[@]}" + run: "set -euo pipefail\n# Source of truth: java-llama.cpp's .github/natives.csv for the pinned ${llama.version}:\n# every row of kind `module` (a GPU backend module, additive\ + \ to the library jar of its\n# platform) except the Android one, which no `java -jar` can use; the cpu-*/metal-*\n# library jars come with llama-platform. Each jar here is ADDED\ + \ to the default CPU\n# natives (-Dllama.classifier activates the gpu-natives profile), so every classifier\n# fat jar keeps the CPU fallback: ggml loads the module when its\ + \ runtime is installed.\npython3 .github/collect-release-assets.py --list-classifiers 1 > fatjar-classifiers.txt\nmapfile -t CLASSIFIERS < fatjar-classifiers.txt\nbuild() { mvn\ + \ -B --no-transfer-progress -pl srcmorph-cli -am -DskipTests -Dmaven.javadoc.skip=true -Dllama.classifier=\"$1\" package; }\nfor c in \"${CLASSIFIERS[@]}\"; do\n build \"$c\"\ + \n f=$(ls srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies.jar)\n cp \"$f\" \"${f%.jar}-${c}.jar\"\ndone\n# default CPU build LAST so the unsuffixed fat jar is the\ + \ all-platform CPU variant\nbuild \"\"\n# Collect every fat jar variant, then GPG-sign each via the cross-repo shared script\n# (.github/sign-fatjars.sh; java-llama.cpp dropped\ + \ its copy with its fat jars) \u2014 a .asc per fat jar.\nmkdir -p fatjar-assets\ncp srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies*.jar fatjar-assets/\nbash .github/sign-fatjars.sh\ + \ fatjar-assets\n# Structural check of what the loop just produced. The smoke-fatjar job LAUNCHES the\n# default jar; the classifier jars cannot be covered that way (a GitHub\ + \ runner has no\n# such device, and the only command that would load the native library is a real\n# generation), so this asserts instead that each jar is the artifact its name\ + \ claims --\n# one jar per classifier, the backend directory the name promises, and every default\n# CPU native kept next to it. The backend directory is the point: if -Dllama.classifier=\n\ + # ever stops being wired through, the loop silently ships copies of the CPU build under\n# GPU names and nothing else in this pipeline notices.\nbash .github/verify-classifier-fatjars.sh\ + \ fatjar-assets \"${CLASSIFIERS[@]}\"\n" - name: Collect signed artifacts id: collect - if: ${{ !cancelled() }} run: python3 .github/collect-release-assets.py signed-snapshot-assets --maven-assets central-assets - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - if: ${{ !cancelled() }} + - name: Verify and freeze the complete signed publication + id: preflight + run: python3 .github/publication.py prepare signed-snapshot-assets publication-package + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 with: name: signed-snapshot-assets - path: signed-snapshot-assets/ + path: publication-package/ + if-no-files-found: error + if: ${{ !cancelled() && steps.preflight.outcome == 'success' }} github-snapshot: name: Update Snapshot Pre-release on GitHub - needs: [publish-snapshot] - # Also runs when publish-snapshot FAILED (not when skipped/cancelled): a Central - # publish-poll timeout reds that job after the artifacts were already uploaded — - # the GitHub pre-release assets must not be lost in that case. - if: ${{ !cancelled() && (needs.publish-snapshot.result == 'success' || needs.publish-snapshot.result == 'failure') }} + needs: + - publish-snapshot + if: ${{ !cancelled() && needs.publish-snapshot.outputs.assets_complete == 'true' }} runs-on: ubuntu-latest - # Serialize the complete upload, tag/notes update and pruning operation. concurrency: group: snapshot-release-${{ github.repository }} cancel-in-progress: false permissions: contents: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 with: name: signed-snapshot-assets - path: snapshot-assets/ - - name: Report unsigned assets (does not block the upload) - # Deliberately NON-blocking, and deliberately BEFORE the upload. Both attach jobs run even - # when their publish job failed, because a Central publish-poll timeout must not cost the - # GitHub assets: if Central is unreachable these are the ONLY way to get the artifacts at - # all. Refusing to attach on a signing failure would defeat exactly that. So annotate here, - # upload regardless, and fail the job afterwards. Assets always land; an unsigned release is - # still loudly red rather than quietly wrong. - # See workspace/policies/fat-jar-release-assets.md, "Attach first, then go red". - id: signatures - run: python3 .github/report-asset-signatures.py snapshot-assets + path: publication-package/ + digest-mismatch: error + - name: Verify transport integrity and export checked GitHub files + run: python3 .github/publication.py verify publication-package --export-github snapshot-assets - name: Update snapshot assets, source tag and notes env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ASSETS_COMPLETE: ${{ needs.publish-snapshot.outputs.assets_complete }} run: python3 .github/update-snapshot.py snapshot-assets - - name: Fail if anything was attached unsigned - # After the upload on purpose: the assets must exist even when the signature does not. - if: ${{ always() && steps.signatures.outputs.missing != '0' }} - run: | - echo "::error::${{ steps.signatures.outputs.missing }} asset(s) attached without a signature (-1 means none were collected at all)" - exit 1 publish-release: - name: Publish Release to Central - needs: [check-tag, code-style, smoke-fatjar, plugin-it, shared-files, vmlens] + name: Prepare and verify signed release artifacts + needs: + - check-tag + - code-style + - smoke-fatjar + - plugin-it + - shared-files + - vmlens if: needs.check-tag.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central outputs: - assets_complete: ${{ steps.collect.outputs.complete }} + assets_complete: ${{ steps.preflight.outputs.complete }} permissions: contents: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad with: java-version-file: .java-version distribution: temurin @@ -706,119 +658,127 @@ jobs: server-password: MAVEN_PASSWORD gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE - # Informational only (nothing depends on it): logs the effective POM with the same - # profile as the deploy below, so the resolved central-publishing configuration - # (waitUntil/waitMaxTime etc.) is visible for debugging. - name: Show effective POM (debug) - run: mvn --batch-mode --no-transfer-progress -P release help:effective-pom - # -Dmaven.repository.credentialScope=id: see the first deploy step of this file (Maven 3.10). - - name: Deploy release - run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release deploy -DskipTests + run: mvn --batch-mode --no-transfer-progress -P release -DskipUpload=true help:effective-pom + - name: Stage release + run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release -DskipUpload=true deploy -DskipTests env: - MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - # Runs even when the deploy step failed: a Central publish-poll timeout reds the - # job *after* the bundle was uploaded (and typically published server-side), while - # the signed jars + .asc files already exist in each module's target/ (signing happens at - # verify). Collecting on failure lets the github-release job still attach them. - # NOTE: the fat jar (jar-with-dependencies) is EXCLUDED here — it is no longer attached/ - # deployed to Central (srcmorph-cli/pom.xml sets false), so it is unsigned - # in target/ at this point. It is built + signed + collected by the next step instead. This - # step runs BEFORE that one so the deploy-signed plain/sources/javadoc jars are captured - # before the classifier loop rebuilds (and clobbers) srcmorph-cli/target. - name: Collect signed Central artifacts (plain/sources/javadoc; fat jars handled next) - if: ${{ !cancelled() }} run: python3 .github/collect-release-assets.py central-assets --maven-only - # Build one srcmorph-cli fat jar per net.ladenthin:llama GPU module jar + the default - # all-platform CPU fat jar, GPG-sign each, and add them to the GitHub-Release assets. - # These are GitHub-only (never Maven Central): every jar carries the multi-platform CPU - # natives (llama-platform), and a classifier jar adds one ggml backend module on top (since - # llama 6.0.0 a GPU natives jar holds only that module), which ggml loads next to the CPU - # library when the vendor runtime is present and skips otherwise. Signing uses the key - # setup-java already imported (armored detached .asc, same shape Central produces). - name: Build, sign & verify classifier fat jars (GitHub-only assets, never Central) - if: ${{ !cancelled() }} env: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - run: | - set -euo pipefail - # Source of truth: java-llama.cpp's .github/natives.csv for the pinned ${llama.version}: - # every row of kind `module` (a GPU backend module, additive to the library jar of its - # platform) except the Android one, which no `java -jar` can use; the cpu-*/metal-* - # library jars come with llama-platform. Each jar here is ADDED to the default CPU - # natives (-Dllama.classifier activates the gpu-natives profile), so every classifier - # fat jar keeps the CPU fallback: ggml loads the module when its runtime is installed. - python3 .github/collect-release-assets.py --list-classifiers 1 > fatjar-classifiers.txt - mapfile -t CLASSIFIERS < fatjar-classifiers.txt - build() { mvn -B --no-transfer-progress -pl srcmorph-cli -am -DskipTests -Dmaven.javadoc.skip=true -Dllama.classifier="$1" package; } - for c in "${CLASSIFIERS[@]}"; do - build "$c" - f=$(ls srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies.jar) - cp "$f" "${f%.jar}-${c}.jar" - done - # default CPU build LAST so the unsuffixed fat jar is the all-platform CPU variant - build "" - # Collect every fat jar variant, then GPG-sign each via the cross-repo shared script - # (.github/sign-fatjars.sh; java-llama.cpp dropped its copy with its fat jars) — a .asc per fat jar. - mkdir -p fatjar-assets - cp srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies*.jar fatjar-assets/ - bash .github/sign-fatjars.sh fatjar-assets - # Structural check of what the loop just produced. The smoke-fatjar job LAUNCHES the - # default jar; the classifier jars cannot be covered that way (a GitHub runner has no - # such device, and the only command that would load the native library is a real - # generation), so this asserts instead that each jar is the artifact its name claims -- - # one jar per classifier, the backend directory the name promises, and every default - # CPU native kept next to it. The backend directory is the point: if -Dllama.classifier= - # ever stops being wired through, the loop silently ships copies of the CPU build under - # GPU names and nothing else in this pipeline notices. - bash .github/verify-classifier-fatjars.sh fatjar-assets "${CLASSIFIERS[@]}" + run: "set -euo pipefail\n# Source of truth: java-llama.cpp's .github/natives.csv for the pinned ${llama.version}:\n# every row of kind `module` (a GPU backend module, additive\ + \ to the library jar of its\n# platform) except the Android one, which no `java -jar` can use; the cpu-*/metal-*\n# library jars come with llama-platform. Each jar here is ADDED\ + \ to the default CPU\n# natives (-Dllama.classifier activates the gpu-natives profile), so every classifier\n# fat jar keeps the CPU fallback: ggml loads the module when its\ + \ runtime is installed.\npython3 .github/collect-release-assets.py --list-classifiers 1 > fatjar-classifiers.txt\nmapfile -t CLASSIFIERS < fatjar-classifiers.txt\nbuild() { mvn\ + \ -B --no-transfer-progress -pl srcmorph-cli -am -DskipTests -Dmaven.javadoc.skip=true -Dllama.classifier=\"$1\" package; }\nfor c in \"${CLASSIFIERS[@]}\"; do\n build \"$c\"\ + \n f=$(ls srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies.jar)\n cp \"$f\" \"${f%.jar}-${c}.jar\"\ndone\n# default CPU build LAST so the unsuffixed fat jar is the\ + \ all-platform CPU variant\nbuild \"\"\n# Collect every fat jar variant, then GPG-sign each via the cross-repo shared script\n# (.github/sign-fatjars.sh; java-llama.cpp dropped\ + \ its copy with its fat jars) \u2014 a .asc per fat jar.\nmkdir -p fatjar-assets\ncp srcmorph-cli/target/srcmorph-cli-*-jar-with-dependencies*.jar fatjar-assets/\nbash .github/sign-fatjars.sh\ + \ fatjar-assets\n# Structural check of what the loop just produced. The smoke-fatjar job LAUNCHES the\n# default jar; the classifier jars cannot be covered that way (a GitHub\ + \ runner has no\n# such device, and the only command that would load the native library is a real\n# generation), so this asserts instead that each jar is the artifact its name\ + \ claims --\n# one jar per classifier, the backend directory the name promises, and every default\n# CPU native kept next to it. The backend directory is the point: if -Dllama.classifier=\n\ + # ever stops being wired through, the loop silently ships copies of the CPU build under\n# GPU names and nothing else in this pipeline notices.\nbash .github/verify-classifier-fatjars.sh\ + \ fatjar-assets \"${CLASSIFIERS[@]}\"\n" - name: Collect signed artifacts id: collect - if: ${{ !cancelled() }} run: python3 .github/collect-release-assets.py signed-release-assets --maven-assets central-assets - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - if: ${{ !cancelled() }} + - name: Verify and freeze the complete signed publication + id: preflight + run: python3 .github/publication.py prepare signed-release-assets publication-package + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 with: name: signed-release-assets - path: signed-release-assets/ + path: publication-package/ + if-no-files-found: error + if: ${{ !cancelled() && steps.preflight.outcome == 'success' }} github-release: name: Attach Binaries to GitHub Release - needs: [publish-release] - # Also runs when publish-release FAILED (not when skipped/cancelled): a Central - # publish-poll timeout reds that job after the artifacts were already uploaded — - # the GitHub release assets must not be lost in that case. - if: ${{ !cancelled() && (needs.publish-release.result == 'success' || needs.publish-release.result == 'failure') }} + needs: + - publish-release + if: ${{ !cancelled() && needs.publish-release.outputs.assets_complete == 'true' }} runs-on: ubuntu-latest permissions: contents: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 with: name: signed-release-assets - path: release-assets/ - - name: Report unsigned assets (does not block the upload) - # Deliberately NON-blocking, and deliberately BEFORE the upload. Both attach jobs run even - # when their publish job failed, because a Central publish-poll timeout must not cost the - # GitHub assets: if Central is unreachable these are the ONLY way to get the artifacts at - # all. Refusing to attach on a signing failure would defeat exactly that. So annotate here, - # upload regardless, and fail the job afterwards. Assets always land; an unsigned release is - # still loudly red rather than quietly wrong. - # See workspace/policies/fat-jar-release-assets.md, "Attach first, then go red". - id: signatures - run: python3 .github/report-asset-signatures.py release-assets + path: publication-package/ + digest-mismatch: error + - name: Verify transport integrity and export checked GitHub files + run: python3 .github/publication.py verify publication-package --export-github release-assets - name: Upload release assets - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 with: files: release-assets/* - - name: Fail if anything was attached unsigned - # After the upload on purpose: the assets must exist even when the signature does not. - if: ${{ always() && steps.signatures.outputs.missing != '0' }} - run: | - echo "::error::${{ steps.signatures.outputs.missing }} asset(s) attached without a signature (-1 means none were collected at all)" - exit 1 + + + central-snapshot: + name: Upload checked snapshot artifacts to Maven Central + needs: + - publish-snapshot + if: ${{ !cancelled() && needs.publish-snapshot.outputs.assets_complete == 'true' }} + runs-on: ubuntu-latest + environment: maven-central + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad + with: + java-version-file: .java-version + distribution: temurin + server-id: central + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: signed-snapshot-assets + path: publication-package/ + digest-mismatch: error + - name: Transfer frozen artifacts (no rebuild, signing or validation polling) + run: python3 .github/publication.py upload publication-package + env: + MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} + + central-release: + name: Upload checked release artifacts to Maven Central + needs: + - publish-release + if: ${{ !cancelled() && needs.publish-release.outputs.assets_complete == 'true' }} + runs-on: ubuntu-latest + environment: maven-central + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad + with: + java-version-file: .java-version + distribution: temurin + server-id: central + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: signed-release-assets + path: publication-package/ + digest-mismatch: error + - name: Transfer frozen artifacts (no rebuild, signing or validation polling) + run: python3 .github/publication.py upload publication-package + env: + MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} diff --git a/docs/publication-preflight.md b/docs/publication-preflight.md new file mode 100644 index 0000000..58094c5 --- /dev/null +++ b/docs/publication-preflight.md @@ -0,0 +1,84 @@ + + +# Signed publication preflight and independent uploads + +## Invariants + +Build and sign every intended output locally. Complete the native, classifier and Android +checks applicable to those outputs. Before any publication-package upload, collect the exact +repository contract and run one complete local gate. Missing files, invalid signatures, +version mismatches or differing Central/GitHub bytes must fail before either publisher runs. + +After the gate, neither publisher compiles, packages nor signs artifacts again. Both use the +same frozen package from GitHub Actions. The Central and GitHub Release jobs depend only on +that handoff, never on each other's success. A Central outage must not hide the GitHub files. +Internal compiler caches, test reports and native-build intermediates are not publication packages. + +## Preparation + +Maven's Central publishing plugin 0.12.0 supports `-DskipUpload=true deploy`: it stages signed +release files under `target/central-staging` or snapshots under `target/central-deferred`, +without a remote upload. This is different from `skipPublishing`, which omits artifacts. +Gradle publications are similarly written to their local staging repository first. + +Preserve the signed Maven files before any later fat-jar rebuild can overwrite `target/`: + +```bash +python3 .github/collect-release-assets.py central-assets --maven-only +# Build and sign additional GitHub-only variants. +python3 .github/collect-release-assets.py signed-release-assets --maven-assets central-assets +python3 .github/publication.py prepare signed-release-assets publication-package +``` + +Use `signed-snapshot-assets` on the snapshot path. A repository without a subsequent rebuild +can collect directly without the intermediate `central-assets` directory. + +The gate checks exact file membership, nonempty artifacts/signatures, cryptographic GPG +verification, and byte equality between Central staging and the corresponding GitHub files. +It freezes `central/` in Maven layout, `github/` as flat release files, and a `manifest.json` +with commit, version, channel, file sizes and SHA-256. `complete=true` is emitted only after +success. The handoff upload is allowed only after that gate, even if an optional provenance +attestation subsequently fails. No incomplete publication package is attached to a release. + +## Repository configuration + +`.github/release-assets.json` remains the source of truth. File rules can set `central: true` +for additional Central publications/classifiers. The default is false, so runnable fat jars +and source archives do not accidentally become Central artifacts. Maven reactor outputs, +configured standalone Maven projects and enabled reactor BOMs are always Central outputs. + +Optional `publication.central_roots` lists repository-relative staging directories. The default +is `["target/{staging}"]`; `{staging}` becomes `central-staging` or `central-deferred` according +to the project version. This supports separate Maven or Gradle staging without repository names +in the shared implementation. Unexpected or missing Central members fail the gate. + +## Independent consumers + +```bash +python3 .github/publication.py verify publication-package --export-github release-assets +python3 .github/publication.py upload publication-package +``` + +Each consumer verifies the complete handoff manifest after download. This is a transport +integrity check, not a second build/signing pass. GitHub publishes only the exported `github/` +files; transport manifests and Central repository directories are not release attachments. + +For releases, the Central consumer forms a Portal bundle from the frozen Central files with +checksums and requests automatic publication. It waits only for HTTP upload acceptance and +records the deployment ID. There is no validation/publication polling. The Maven profiles also +use `autoPublish=true` and `waitUntil=uploaded` for manual deployments. + +For snapshots, a transfer-only Maven POM invokes pinned `maven-deploy-plugin:3.1.4:deploy-file` +executions during `validate`. Maven maintains snapshot timestamps and metadata; signatures are +transferred in the same component request with their proper extensions. There are no source +build phases. For a POM-only component its signature is the primary transfer artifact and the +signed POM is supplied through `pomFile`, avoiding an empty-classifier parameter edge case. + +Local gate failures stop both publication paths. Authentication or transfer failures remain +visible in the affected upload job, but never gate the other publisher. Slow Portal processing +does not fail the build. Upload acceptance is not a claim of successful Central validation; +inspect the recorded deployment in Portal if later processing fails. Re-running a transfer +uses the same frozen package. A network-interrupted GitHub release update is not atomic. diff --git a/pom.xml b/pom.xml index 3a72bcf..566b60c 100644 --- a/pom.xml +++ b/pom.xml @@ -300,10 +300,8 @@ SPDX-License-Identifier: Apache-2.0 central true - - validated - - 21600 + + uploaded