From 0a34d104dfdc9c312a85434a8bdc95101cafd89f Mon Sep 17 00:00:00 2001 From: mck Date: Mon, 31 Aug 2026 23:10:22 +0200 Subject: [PATCH 1/2] CI: Take the report merge off the test fan-out, stop the agent pod churn, report split balance Three faults measured on a Kubernetes clone of this pipeline. Jenkinsfile: - generateTestReports moves from cassandra-medium, the label of the ~460 test splits it queued behind, to cassandra-report: 109 minutes of a 163 minute build, for ~3 minutes of work - one ant junitreport per target, three at a time under an explicit -Xmx - size every `xargs -P` and `xz -T` from the container's cgroup quota. nproc answers for the node: 8 against a 2 cpu limit - record each cell's duration and longest suites, and archive them - test-burn takes 5 splits, which separates its three long classes - remove testDataPublishers as they were oom-ing ci-cassandra.a.o's controller https://the-asf.slack.com/archives/CK23JSY2K/p1787841152008959?thread_ts=1787836602.176469&cid=CK23JSY2K , and it was noise because of https://github.com/jenkinsci/test-stability-plugin/issues/11 (we use butler instead) jenkins-deployment.yaml: - slaveConnectTimeout 600 and waitForPodSec 900, the two deadlines on one provisioning attempt. 30 seconds met only a warm node, and churned 1,110 pods in one 24 minute window with 0 agents connected and 483 `KubernetesProvisioningLimits ... went below zero` warnings - idleMinutes 5, a pod holding a whole node - the four instanceCaps to the demand a pre-commit run measured; containerCap to their sum, so no pool's share is decided by which holds its agents longest, and maxRequestsPerHostStr with it - add the agent-dind-report template, on a node pool of its own cell_balance.py: - new. Per target, how evenly its splits divided and what the worst cell ran, to the console and ci_summary.html. Splits stay round-robin, so the lever is the split count - guarded in generate-ci-summary.sh, never failing a build run-ci-test.py: - the fixture's node group maximums follow the raised caps patch by Mick Semb Wever; reviewed by xxx for CASSANDRA-##### Assisted-by: Claude Code:claude-opus-5 --- .build/ci/cell_balance.py | 281 +++++++++++++++++++++++++++ .build/ci/generate-ci-summary.sh | 8 +- .build/run-ci.d/run-ci-test.py | 13 +- .jenkins/Jenkinsfile | 131 +++++++++++-- .jenkins/k8s/README.md | 12 +- .jenkins/k8s/jenkins-deployment.yaml | 155 ++++++++++++--- 6 files changed, 548 insertions(+), 52 deletions(-) create mode 100755 .build/ci/cell_balance.py diff --git a/.build/ci/cell_balance.py b/.build/ci/cell_balance.py new file mode 100755 index 000000000000..a0b15a1f96d0 --- /dev/null +++ b/.build/ci/cell_balance.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""How evenly each test target's splits divided, and which of them came near their deadline. + +What makes a split long: `_split_tests` in ../run-tests.sh deals an alphabetically sorted class list +round-robin with `split -n r/K/N`, which balances the count of classes and knows nothing of their duration. + +Round-robin is required, so that which split holds a test is predictable and easy to find. + +This report cannot tell a structurally long split from one that drew a +slow agent, so change a split count only when consecutive builds agree. + +Usage: + + cell_balance.py [--input build/test] [--output build/ci_summary.html] [--budget-margin 15] + +`--input` holds `cell-times/*.tsv`, one per cell from `recordCellTime` in the Jenkinsfile, and +`output//` of decompressed JUnit XML. With no records it says so and exits 0: a build whose cells all +timed out has nothing to measure, and this is not the check to fail it. +""" + +import argparse +import os +import statistics +import sys +import xml.etree.ElementTree as ElementTree + +# Columns of a cell-times record, in order. test_seconds covers every class and not only the twelve .suites +# lists, so the setup figure is exact rather than a residue. A seven-field record from before it still reads. +FIELDS = ("step", "arch", "jdk", "split", "splits", "timeout_hours", "duration_ms", "test_seconds") + +# Enough of the worst cell's longest suites to show whether one class dominates or the whole split is heavy, +# which is the distinction that decides what to do. More when over budget, that being the one to act on. +LONGEST_SUITES = 6 +LONGEST_SUITES_OVER_BUDGET = 12 + +# The floor keeps it quiet for cells with no setup worth reporting +UNDER_RECORDED_TEST_SHARE = 0.25 +UNDER_RECORDED_MINIMUM_MINUTES = 5.0 + +# Name the test classes only when the worst cell's longest reaches this +NAME_CLASSES_ABOVE_MINUTES = 5.0 + + +def read_records(cell_times_dir): + """Every cell record, as a list of dicts. A malformed file is skipped rather than fatal.""" + records = [] + if not os.path.isdir(cell_times_dir): + return records + # The .suites files sit in this directory too, so only .tsv is a record. + for name in sorted(n for n in os.listdir(cell_times_dir) if n.endswith(".tsv")): + path = os.path.join(cell_times_dir, name) + try: + with open(path, encoding="utf-8") as handle: + fields = handle.readline().rstrip("\n").split("\t") + if len(fields) not in (len(FIELDS) - 1, len(FIELDS)): + continue + record = dict(zip(FIELDS, fields)) + record["minutes"] = int(record["duration_ms"]) / 60000.0 + + record["test_minutes"] = (float(record["test_seconds"]) / 60.0 + if record.get("test_seconds") else None) + record["deadline_minutes"] = float(record["timeout_hours"]) * 60.0 + record["label"] = f"{record['step']} jdk{record['jdk']} {record['split']}/{record['splits']}" + record["suites"] = read_suites(path[: -len(".tsv")] + ".suites") + records.append(record) + except (OSError, ValueError): + continue + return records + + +def read_suites(path): + """(suite name, minutes) for one cell, longest first, or [] when the file is absent.""" + try: + with open(path, encoding="utf-8") as handle: + fields = [line.rstrip("\n").partition("\t") for line in handle] + return [(name, float(seconds) / 60.0) for seconds, _, name in fields if name] + except (OSError, ValueError): + return [] + + +def summarise(records, budget_margin): + """One row per target, worst first.""" + by_step = {} + for record in records: + by_step.setdefault(record["step"], []).append(record) + + rows = [] + for step, cells in by_step.items(): + durations = sorted(cell["minutes"] for cell in cells) + worst = max(cells, key=lambda cell: cell["minutes"]) + median = statistics.median(durations) + budget = max(cell["deadline_minutes"] for cell in cells) - budget_margin + rows.append({ + "step": step, + "cells": len(cells), + "min": durations[0], + "median": median, + "max": durations[-1], + # Guarded, because a target whose median is zero is a target that ran nothing. + "ratio": durations[-1] / median if median > 0 else 0.0, + "budget": budget, + "over_budget": durations[-1] > budget, + "worst_label": worst["label"], + "worst_suites": worst["suites"], + # The cell's own total where it recorded one, and the sum of what it listed otherwise. + "worst_test_minutes": (worst["test_minutes"] if worst["test_minutes"] is not None + else sum(minutes for _, minutes in worst["suites"])), + "worst_test_minutes_exact": worst["test_minutes"] is not None, + }) + rows.sort(key=lambda row: -row["max"]) + return rows + + +def suite_times(target_output_dir): + """(suite name, minutes) for every JUnit suite under one target, longest first.""" + suites = [] + for root, _, names in os.walk(target_output_dir): + for name in names: + if not name.endswith(".xml"): + continue + try: + element = ElementTree.parse(os.path.join(root, name)).getroot() + except (OSError, ElementTree.ParseError): + continue + # A file's root is from ant and from pytest, so both are searched. + for suite in [element] if element.tag == "testsuite" else element.findall(".//testsuite"): + try: + suites.append((suite.get("name") or "?", float(suite.get("time") or 0) / 60.0)) + except ValueError: + continue + suites.sort(key=lambda pair: -pair[1]) + return suites + + +def print_report(rows, output_root, budget_margin): + if not rows: + print("No cell-times records were found, so there is no split balance to report.") + return + + print() + print("Split balance, by test target. `worst` is one cell's whole duration, setup included, against a") + print(f"`budget` of its deadline less {budget_margin} min. Splitting balances the count of classes and" + " not their") + print("duration, so a high w/med is a split that drew slow ones. Confirm against the previous build.") + print() + header = (f"{'target':34s} {'cells':>5s} {'min':>7s} {'median':>7s} {'worst':>7s}" + f" {'w/med':>6s} {'budget':>7s} {'':4s} worst cell") + print(header) + print("-" * len(header)) + for row in rows: + flag = "OVER" if row["over_budget"] else "" + print(f"{row['step']:34s} {row['cells']:5d} {row['min']:7.1f} {row['median']:7.1f}" + f" {row['max']:7.1f} {row['ratio']:6.2f} {row['budget']:7.0f} {flag:4s}" + f" {row['worst_label']}") + + if not any(row["over_budget"] for row in rows): + print() + print("Every target's worst cell is inside its budget.") + + print() + print("What each worst cell ran. One class near the cell's whole duration cannot be split further;") + print("several of comparable size can be. Classes are named only where the longest reaches" + f" {NAME_CLASSES_ABOVE_MINUTES:.0f} min,") + print("below which none of them is the reason.") + for row in rows: + print() + print_worst_cell(row, output_root) + + +def print_share(minutes, total, text): + """One breakdown line: minutes, what share of the cell they are, and what spent them.""" + print(f" {minutes:7.1f} min {100 * minutes / total if total else 0:4.0f}% {text}") + + +def print_worst_cell(row, output_root): + """The longest classes of one target's worst cell, and the time that was not test time.""" + flag = " OVER BUDGET" if row["over_budget"] else "" + print(f"{row['worst_label']}: {row['max']:.1f} min against a budget of {row['budget']:.0f}{flag}") + + suites, total, tests = row["worst_suites"], row["max"], row["worst_test_minutes"] + if not suites: + fallback = suite_times(os.path.join(output_root, row["step"]))[:LONGEST_SUITES] + if not fallback: + print(" no per-cell suite record, and no JUnit XML for this target either") + return + print(" no per-cell suite record, so these are the longest across the whole target:") + for name, minutes in fallback: + print(f" {minutes:7.1f} min {name}") + return + + longest = max(minutes for _, minutes in suites) + if longest < NAME_CLASSES_ABOVE_MINUTES: + # No class is large enough to be the reason, so the whole test time is one line + print_share(tests, total, f"every test class in this cell, the longest of them {longest:.1f} min") + else: + shown = suites[:LONGEST_SUITES_OVER_BUDGET if row["over_budget"] else LONGEST_SUITES] + for name, minutes in shown: + print_share(minutes, total, name) + other = tests - sum(minutes for _, minutes in shown) + if other > 0.05: + print_share(other, total, "every other test class in this cell") + + setup = total - tests + if setup <= 0: + return + share = tests / total if total else 1.0 + if total >= UNDER_RECORDED_MINIMUM_MINUTES and share < UNDER_RECORDED_TEST_SHARE: + print_share(setup, total, f"setup at most: only {tests:.1f} min of tests was recorded," + f" {100 * share:.0f}% of the cell,") + print(" so tests are missing from the JUnit XML and this is an upper bound") + else: + exact = "" if row["worst_test_minutes_exact"] else ", or a test class this cell did not record" + print_share(setup, total, "not running tests: the node, the image pulls, the compile and the" + f" virtualenv{exact}") + + +def html_report(rows, budget_margin): + parts = ["

Split balance

"] + if not rows: + parts.append("

No cell-times records were found.

") + return "".join(parts) + parts.append( + "

worst is one cell's whole duration, setup included, which is what the cell deadline" + f" governs; budget is that deadline less {budget_margin} minutes of margin." + " Round-robin splitting balances the number of classes in a split and not their duration, so a high" + " worst/median is a split that drew slow classes. Confirm against the previous build" + " before changing a split count.

") + parts.append("" + "") + for row in rows: + style = " bgcolor='#ffdddd'" if row["over_budget"] else "" + parts.append( + f"" + f"" + f"") + parts.append("
targetcellsminmedianworstworst/medianbudgetworst cell
{row['step']}{row['cells']}{row['min']:.1f}{row['median']:.1f}{row['max']:.1f}{row['ratio']:.2f}{row['budget']:.0f}{row['worst_label']}
") + return "".join(parts) + + +def main(): + parser = argparse.ArgumentParser(description="Report how evenly each test target's splits divided.") + parser.add_argument("--input", default="build/test", + help="directory holding cell-times/ and output/, default build/test") + parser.add_argument("--output", default=None, + help="existing .html file to append the table to; omitted prints only") + parser.add_argument("--budget-margin", type=int, default=15, + help="minutes of margin to keep under a cell's deadline, default 15") + args = parser.parse_args() + + records = read_records(os.path.join(args.input, "cell-times")) + rows = summarise(records, args.budget_margin) + print_report(rows, os.path.join(args.input, "output"), args.budget_margin) + + if args.output and os.path.isfile(args.output): + try: + with open(args.output, "a", encoding="utf-8") as handle: + handle.write(html_report(rows, args.budget_margin)) + except OSError as error: + print(f"could not append to {args.output}: {error}", file=sys.stderr) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.build/ci/generate-ci-summary.sh b/.build/ci/generate-ci-summary.sh index 3828fa2c1e55..b5a34a36dfc7 100755 --- a/.build/ci/generate-ci-summary.sh +++ b/.build/ci/generate-ci-summary.sh @@ -68,7 +68,13 @@ cat >${DIST_DIR}/ci_summary.html < str: maxSize is the only in-cluster record of what a pool can hold, and a pool at zero nodes has no nodes to count, so the check reads it from here. """ + # Two groups per size, each pair summing to that size's instanceCap in jenkins-deployment.yaml. Raise + # these whenever a cap is raised, or the committed values stop passing their own check. groups = [(f"eks-amd64-{size}-ondemand-{n}-{n}cfd1c1", 0, maximum) - for size, maximum in (("large", 80), ("medium", 65), ("small", 25)) for n in (2, 3)] + for size, maximum in (("large", 153), ("medium", 75), ("small", 10), ("report", 2)) + for n in (2, 3)] groups.append(("eks-jenkins-controller-0-2acd8787", 1, 1)) return yaml.safe_dump({"nodeGroups": [ @@ -341,9 +344,9 @@ def test_check_agent_capacity_allows_the_committed_values(self): self.assertEqual(0, self.capacity_check(self.deployed_values(), nodes=self.LARGE_NODE)) def test_check_agent_capacity_blocks_a_cap_above_the_pool(self): - # 200 against the 160 nodes two large groups can hold: 40 agents could never be scheduled, which is + # 400 against the 306 nodes two large groups can hold: 94 agents could never be scheduled, which is # not idle but a churn loop, and is what preceded the 2026-08-11 controller stall - over = self.deployed_values("large", instanceCap=200, instanceCapStr="200") + over = self.deployed_values("large", instanceCap=400, instanceCapStr="400") self.assertEqual(1, self.capacity_check(over)) # a cluster whose ceilings cannot be read leaves it unchecked rather than blocking a valid deploy self.assertEqual(0, self.capacity_check(over, autoscaler=False)) @@ -352,14 +355,14 @@ def test_check_agent_capacity_reads_maxsize_wherever_it_is_published(self): # the live cluster nests a group's maximum under its health condition, and the check also takes it # from the group. Reading the wrong key costs nothing visible: the ceilings come out empty and # every cap passes unchecked, so the shapes are pinned here rather than in a deploy - over = self.deployed_values("large", instanceCap=200, instanceCapStr="200") + over = self.deployed_values("large", instanceCap=400, instanceCapStr="400") for nested in (True, False): self.assertEqual(1, self.capacity_check(over, nested=nested)) self.assertEqual(0, self.capacity_check(self.deployed_values(), nested=nested)) def test_check_agent_capacity_blocks_contradictory_config(self): # the plugin takes the cap from either key, so a disagreement resolves to whichever applies last - self.assertEqual(1, self.capacity_check(self.deployed_values("large", instanceCapStr="200"))) + self.assertEqual(1, self.capacity_check(self.deployed_values("large", instanceCapStr="400"))) # a nodeSelector the live nodes contradict strands every agent of that size typo = self.deployed_values("large", nodeSelector="cassandra.jenkins.agent.large=ture") self.assertEqual(1, self.capacity_check(typo, nodes=self.LARGE_NODE)) diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 83e58b67f81c..71d2de7a8f00 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -35,6 +35,10 @@ // - cassandra-small + cassandra-${arch}-small : 1 cpu, 1GB ram (alias for above but for any arch) // - cassandra-medium + cassandra-${arch}-medium : 3 cpu, 5GB ram // - cassandra-large + cassandra-${arch}-large : 7 cpu, 16GB ram +// - cassandra-report + cassandra-${arch}-report : 3 cpu, 5GB ram +// +// cassandra-report is the medium capability on nodes the test fan-out cannot take. generateTestReports is +// its only user and runs last, let it through quickly // // Performance targets required a `cassandra-${arch}-large-dedicated` labelled nodes. // @@ -61,6 +65,26 @@ import groovy.transform.Field @Field List archsSupported = ["amd64", "arm64"] @Field List pythonsSupported = ["3.8", "3.11", "3.12", "3.13"] @Field String pythonDefault = "3.8" +// Shell defining cpus(), to prepend to any `sh` script fanning out with `xargs -P` or `xz -T` +@Field String cpusShell = ''' +cpus() { + cpus_n= + # The v2 line of /proc/self/cgroup is "0::", this cgroup relative to the host's root. + cpus_own="$(sed -n 's/^0:://p' /proc/self/cgroup 2>/dev/null | head -1)" + for cpus_file in "/sys/fs/cgroup${cpus_own}/cpu.max" /sys/fs/cgroup/cpu.max ; do + cpus_quota= cpus_period= + read -r cpus_quota cpus_period 2>/dev/null < "${cpus_file}" || continue + # "max " is a cgroup with no cpu limit, where nproc is right. + case "${cpus_quota}${cpus_period}" in ""|*[!0-9]*) continue ;; esac + [ "${cpus_period:-0}" -gt 0 ] || continue + cpus_n=$(( cpus_quota / cpus_period )) + break + done + [ -n "${cpus_n}" ] || cpus_n="$(nproc 2>/dev/null || echo 1)" + [ "${cpus_n}" -ge 1 ] 2>/dev/null || cpus_n=1 + echo "${cpus_n}" +} +''' /** CONSTANTS end **********************************/ pipeline { @@ -199,7 +223,7 @@ def tasks() { 'test-latest': [splits: 20], 'test-compression': [splits: 20], 'stress-test': [splits: 1, size: 'small'], - 'test-burn': [splits: 4], + 'test-burn': [splits: 5], 'long-test': [splits: 4], 'test-oa': [splits: 20], 'test-system-keyspace-directory': [splits: 20], @@ -423,8 +447,8 @@ def test(command, cell) { script_vars = "${script_vars} cython=\'${cell.cython}\'" } script_vars = fetchDTestsSource(command, script_vars) + def timer = System.currentTimeMillis() timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes - def timer = System.currentTimeMillis() try { buildJVMDTestJars(cell, script_vars, logfile) script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" @@ -453,13 +477,14 @@ def test(command, cell) { } } dir("build") { + recordCellTime(cell, cell_suffix, splits, command.timeout_hours, System.currentTimeMillis() - timer) organiseTestResultFiles(cell, cell_suffix) if (!cell.step.startsWith("microbench")) { - junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] + junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml" } debugOomKiller() compressTestResultFiles() - archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true + archiveArtifacts artifacts: "test/cell-times/**,test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) } } finally { @@ -616,6 +641,55 @@ def _stash(cell) { stash name: "${cell.arch}_${cell.jdk}" } +// One line per cell, for the split balance table .build/ci/cell_balance.py puts in ci_summary.html. +// +// The duration is the cell's own, not the sum of its tests: the cell deadline covers the setup before the +// first test too. Measured on build cassandra-eks-k8s #11, a cell spent 10.5 minutes on tests and 17.9 on +// the image pulls, the compile and the virtualenv, so test time alone calls that cell comfortable. +def recordCellTime(cell, cell_suffix, splits, timeout_hours, duration_ms) { + // Both files are written before organiseTestResultFiles: here test/output/ holds only this cell's results, + // and one step later they are merged with the split gone from every path, so no report can say which cell + // ran which test again. + // + // Aggregated by a testcase's classname, the one granularity both xml shapes share. ant writes a file per + // class, so its is already the class; pytest writes one per cell named after + // --junit-prefix, so its suite name is useless. Both write . + // + // The cell's whole test time is recorded too, over every class and not only the twelve listed, so + // cell_balance.py can subtract it from the duration and state the setup exactly. + sh label: "recording cell time...", script: """ + mkdir -p test/cell-times + suites='test/cell-times/${cell.step}${cell_suffix}.suites' + + # \\t, longest first, with the total on stderr. grep -o rather than a line-oriented + # read, because pytest puts many testcase elements on one line. A parser would be tidier; this runs + # once per cell, of which a build has over a thousand. + find test/output -type f -name '*.xml' -print0 2>/dev/null \\ + | xargs -0 -r grep -ho ']*>' 2>/dev/null \\ + | awk ' + { + cls = ""; secs = "" + # time=" is anchored on a space because name=" is a substring of classname=". Anchoring both + # costs nothing and records the trap. + if (match(\$0, /classname="[^"]*"/)) { cls = substr(\$0, RSTART + 11, RLENGTH - 12) } + if (match(\$0, /[ \\t]time="[^"]*"/)) { secs = substr(\$0, RSTART + 7, RLENGTH - 8) } + if (cls != "" && secs != "") { total[cls] += secs; grand += secs } + } + END { + for (c in total) { printf "%.3f\\t%s\\n", total[c], c } + printf "%.3f\\n", grand > "/dev/stderr" + }' 2>test/cell-times/grand \\ + | sort -rn | head -12 > "\${suites}" || true + test_seconds="\$(cat test/cell-times/grand 2>/dev/null || echo 0)" + rm -f test/cell-times/grand + + printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \\ + '${cell.step}' '${cell.arch}' '${cell.jdk}' '${cell.split}' '${splits}' \\ + '${timeout_hours}' '${duration_ms}' "\${test_seconds}" \\ + > 'test/cell-times/${cell.step}${cell_suffix}.tsv' + """ +} + def organiseTestResultFiles(cell, cell_suffix) { sh label: "organise test result files...", script: """ mkdir -p test/output/${cell.step} @@ -634,9 +708,11 @@ def debugOomKiller() { } def compressTestResultFiles() { - sh label: "compress test result files...", script: """ + sh label: "compress test result files...", script: """${cpusShell} { set +x; } 2>/dev/null - find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f + # -n64, not -n1: one xz per file is a fork and an exec for a few kilobytes, and 64 per process still + # leaves more batches than parallel slots. cpus(), not nproc: see cpusShell. + find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n64 -P"\$(cpus)" xz -f echo "\$(find test/output -type f -name "*.xml.xz" | wc -l) test result files compressed" """ } @@ -646,7 +722,7 @@ def compressTestResultFiles() { ///////////////////////////////////////// def generateTestReports() { - node("cassandra-medium") { + node("cassandra-report") { cleanAgent("generateTestReports") checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]]) def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz" @@ -661,31 +737,44 @@ def generateTestReports() { unzip -x -d build/test -q output.zip ) ${teeSuffix} """ } else { - copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true + copyArtifacts filter: 'test/cell-times/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true } // merge and summarise test reports if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name "*.xml.xz" -print -quit)"', returnStatus: true) == 0) { // merge splits for each target's test report, other axes are kept separate - // TODO parallelised for loop // TODO results_details.tar.xz needs to include all logs for failed tests - sh label: "merging splits test reports...", script: """${script_vars} ( + sh label: "merging splits test reports...", script: """${script_vars}${cpusShell} ( echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l - find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f --decompress - - for target in \$(ls build/test/output/) ; do - if test -d build/test/output/\${target} ; then - mkdir -p build/test/reports/\${target} - echo "Report for \${target} (\$(find build/test/output/\${target} -name '*.xml' | wc -l) test files)" - CASSANDRA_DOCKER_ANT_OPTS="-Dbuild.test.output.dir=build/test/output/\${target} -Dbuild.test.report.dir=build/test/reports/\${target}" + find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n64 -P"\$(cpus)" xz -f --decompress + + # One ant junitreport per test target, three at a time. + report_jobs=\$(cpus) ; [ "\${report_jobs}" -le 3 ] || report_jobs=3 + report_target() { + target="\$1" + test -d "build/test/output/\${target}" || return 0 + mkdir -p "build/test/reports/\${target}" + # Held and printed as one block per target: concurrent runs otherwise interleave line by line + report_log="\$(mktemp)" + { + echo "Report for \${target} (\$(find "build/test/output/\${target}" -name '*.xml' | wc -l) test files)" + # -Xmx2g per concurrent ant jvm; unset, each takes a quarter of dind's limit. Exported inside + # the function, which xargs runs as its own process, so targets cannot see each other's value. + CASSANDRA_DOCKER_ANT_OPTS="-Xmx2g -Dbuild.test.output.dir=build/test/output/\${target} -Dbuild.test.report.dir=build/test/reports/\${target}" export CASSANDRA_DOCKER_ANT_OPTS .build/docker/_docker_run.sh debian-build.docker ci/generate-test-report.sh - fi - done + } > "\${report_log}" 2>&1 + report_status=\$? + cat "\${report_log}" ; rm -f "\${report_log}" + return \${report_status} + } + export -f report_target + ls build/test/output/ | xargs -r -n1 -P"\${report_jobs}" bash -xc 'report_target "\$0"' .build/docker/_docker_run.sh debian-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh" tar -cf build/results_details.tar -C build/test/ reports - xz -8f build/results_details.tar ) ${teeSuffix} + # -T with a count: xz's own -T0 reads nproc and would thread for the node, not this container. + xz -8f -T"\$(cpus)" build/results_details.tar ) ${teeSuffix} """ dir('build/') { @@ -762,4 +851,4 @@ def emailContent() { ------------------------------------------------------------------------------- For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/ ''' -} \ No newline at end of file +} diff --git a/.jenkins/k8s/README.md b/.jenkins/k8s/README.md index 14ac7175fdd1..b8d631d7595a 100644 --- a/.jenkins/k8s/README.md +++ b/.jenkins/k8s/README.md @@ -24,15 +24,21 @@ ZONE="us-central1-c" gcloud container clusters create ${CLUSTER_NAME} --machine-type e2-standard-8 --disk-type=pd-ssd --num-nodes 1 --node-labels=cassandra.jenkins.controller=true --autoscaling-profile optimize-utilization --zone ${ZONE} # small resource nodes -gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE} +gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=20 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE} + +# report resource nodes, for the generateTestReports stage (the agent-dind-report podTemplate) +# a standard machine, not highcpu: that template's dind container has a 9G memory limit +gcloud container node-pools create agents-report --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=4 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.report=true --zone ${ZONE} # medium resource nodes # preference (by cost): n2-highcpu-8, c3-highcpu-8, n4-highcpu-8, n1-highcpu-16 -gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE} +gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=150 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE} # large resource nodes -gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE} +gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=306 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE} +# Each --max-nodes above is that size's agent.podTemplates.*.instanceCap in jenkins-deployment.yaml, one agent per node. +# Raise the pool first: `.build/run-ci --only-setup` blocks a deploy whose instanceCap exceeds the nodes its pool can hold. # For each sized resource nodes, pick any machine type that fits, those listed above should work and be the most cost-effective, but this can change region to region # See https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/Jenkinsfile#L35-L38 # and agent.podTemplates.*.resourceLimitCpu and agent.podTemplates.*.resourceLimitMemory (adding gke/eks requirements) in https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/k8s/jenkins-deployment.yaml diff --git a/.jenkins/k8s/jenkins-deployment.yaml b/.jenkins/k8s/jenkins-deployment.yaml index 6e59340a13a8..37e257da19b5 100644 --- a/.jenkins/k8s/jenkins-deployment.yaml +++ b/.jenkins/k8s/jenkins-deployment.yaml @@ -170,11 +170,16 @@ controller: enabled: true agent: disableDefaultAgent: true - maxRequestsPerHostStr: "3200" - containerCap: 300 + # Concurrent requests the plugin's client may hold to one host, the API server being its only host. The + # plugin's default of 32 is for a cloud of a few agents. + maxRequestsPerHostStr: "5120" + # The cap across every podTemplate below, and equal to the sum of their four instanceCaps, so on a cluster + # whose pools can each reach their own cap this never binds. Deliberately: a lower shared cap does not + # divide the shortfall evenly, it hands it to whichever pool holds its agents longest, which is large. + containerCap: 480 node-selector: cassandra.jenkins.agent: true - waitForPodSec: "180" + waitForPodSec: "900" # Reap agent pods the controller no longer tracks. A restart of the controller JVM clears its in-memory # agent registry, and because agent pods are bare pods with no ownerReference nothing else deletes them. # Preferred over a pod activeDeadlineSeconds, which cannot distinguish an orphan from a long (6h) build. @@ -206,13 +211,15 @@ agent: nodeSelector: 'cassandra.jenkins.agent.small=true' # 0 = no pod lifetime cap: idleMinutes reuse means a pod's age is unrelated to any one build's timeout. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' - idleMinutes: 1 - # should match the small pool's 50 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. - instanceCap: 50 - instanceCapStr: "50" + idleMinutes: 5 + # This pool's ceiling, not the cluster's; agent.containerCap above is that, and is the one to lower + # when the account cannot hold this many nodes. Must not exceed the small pool's own maximum + # (README's --max-nodes), one agent per node. No other template selects that pool. + instanceCap: 20 + instanceCapStr: "20" nodeUsageMode: "NORMAL" showRawYaml: 'true' - slaveConnectTimeout: '30' + slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp @@ -308,19 +315,126 @@ agent: volumeMounts: - name: docker-storage mountPath: /var/lib/docker + agent-dind-report: | + - name: agent-dind-report + label: agent-dind cassandra-report cassandra-amd64-report + nodeSelector: 'cassandra.jenkins.agent.report=true' + activeDeadlineSeconds: '0' + idleMinutes: 5 + instanceCap: 4 + instanceCapStr: "4" + nodeUsageMode: "NORMAL" + showRawYaml: 'true' + slaveConnectTimeout: '600' + yamlMergeStrategy: override + containers: + - name: jnlp + # https://github.com/jenkinsci/kubernetes-plugin#pipeline-support + alwaysPullImage: true + envVars: + - envVar: + key: DOCKER_TLS_CERTDIR + value: /certs/client/ + - envVar: + key: DOCKER_CERT_PATH + value: /certs/client/ + - envVar: + key: DOCKER_TLS_VERIFY + value: 'true' + - envVar: + key: DOCKER_HOST + value: tcp://localhost:2376 + - envVar: + key: JENKINS_JAVA_OPTS + value: '-Dorg.jenkinsci.plugins.durabletask.BourneShellScript.USE_BINARY_WRAPPER=true' + image: apache.jfrog.io/cassan-docker/apache/cassandra-jenkins-k8s + livenessProbe: + failureThreshold: '0' + initialDelaySeconds: '0' + periodSeconds: '0' + successThreshold: '0' + timeoutSeconds: '0' + privileged: 'true' + resourceRequestCpu: 1 + resourceLimitCpu: 2 + resourceRequestMemory: 1G + resourceLimitMemory: 2400M + # the workspace emptyDir: the downloaded output.zip, the decompressed test xml, and the tar + resourceRequestEphemeralStorage: 10Gi + resourceLimitEphemeralStorage: 20Gi + ttyEnabled: 'true' + workingDir: /home/jenkins/agent + - name: dind + alwaysPullImage: 'false' + envVars: + - envVar: + key: DOCKER_TLS_CERTDIR + value: /certs + - envVar: + key: "DOCKER_IPTABLES_LEGACY" + value: "1" + image: docker:dind + args: "--default-address-pool base=192.168.96.0/20,size=24" # overwrite docker subnet in case of overlapping + livenessProbe: + failureThreshold: '0' + initialDelaySeconds: '0' + periodSeconds: '0' + successThreshold: '0' + timeoutSeconds: '0' + privileged: 'true' + resourceRequestCpu: 2 + resourceLimitCpu: 5 + resourceRequestMemory: 3400M + # 9G against the medium template's 5G: generateTestReports now runs several ant junitreport + # jvms at once, each in a container this daemon holds. Moves with the Jenkinsfile's + # report_jobs and -Xmx. + resourceLimitMemory: 9G + # docker's images and containers, in the docker-storage emptyDir + resourceRequestEphemeralStorage: 40Gi + resourceLimitEphemeralStorage: 60Gi + ttyEnabled: 'true' + workingDir: /home/jenkins/agent + volumes: + # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit + - emptyDirVolume: + memory: 'false' + mountPath: /certs + # limit one agent pod per node for simpler operations (like orphan cleanup) + yaml: | + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: jenkins/cassius-jenkins-agent + operator: In + values: + - "true" + topologyKey: kubernetes.io/hostname + # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge. + # 60Gi bounds the images and their containers alone, of the pod's 80Gi + # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow. + volumes: + - name: docker-storage + emptyDir: + sizeLimit: 60Gi + containers: + - name: dind + volumeMounts: + - name: docker-storage + mountPath: /var/lib/docker agent-dind-medium: | - name: agent-dind-medium label: agent-dind cassandra-medium cassandra-amd64-medium nodeSelector: 'cassandra.jenkins.agent.medium=true' - # 0 = no pod lifetime cap: idleMinutes reuse means a pod's age is unrelated to any one build's timeout. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' - idleMinutes: 1 - # should match the medium pools 100 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. - instanceCap: 100 - instanceCapStr: "100" + idleMinutes: 5 + instanceCap: 150 + instanceCapStr: "150" nodeUsageMode: "NORMAL" showRawYaml: 'true' - slaveConnectTimeout: '30' + slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp @@ -420,17 +534,14 @@ agent: - name: agent-dind-large label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated nodeSelector: 'cassandra.jenkins.agent.large=true' - # 0 = no pod lifetime cap. A microbench cell runs up to 6h (timeout_hours in the Jenkinsfile) - # and idleMinutes reuse extends a pod's age past any one build, so age cannot stand in for - # health here. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' - idleMinutes: 1 - # should match the large pools 160 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. - instanceCap: 160 - instanceCapStr: "160" + idleMinutes: 5 + instanceCap: 306 + instanceCapStr: "306" nodeUsageMode: "NORMAL" showRawYaml: 'true' - slaveConnectTimeout: '30' + # Raised from 30 seconds. See the small template above. + slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp From 4e108a651460647119fb1a5ece134a0a982b8147 Mon Sep 17 00:00:00 2001 From: mck Date: Fri, 4 Sep 2026 07:56:34 +0200 Subject: [PATCH 2/2] =?UTF-8?q?SQUASH=20=E2=80=93=C2=A0review=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .build/ci/cell_balance.py | 20 ++++++++++++++++---- .build/ci/generate-ci-summary.sh | 6 ++---- .jenkins/Jenkinsfile | 12 ++++++------ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.build/ci/cell_balance.py b/.build/ci/cell_balance.py index a0b15a1f96d0..5db459af4b34 100755 --- a/.build/ci/cell_balance.py +++ b/.build/ci/cell_balance.py @@ -253,12 +253,25 @@ def html_report(rows, budget_margin): return "".join(parts) +def write_into_body(path, html): + """Put the table inside the document's body, ci_parser.py having already closed it. + + Insert before the last `` instead, and append only when the file has no body to insert into. + """ + with open(path, encoding="utf-8") as handle: + document = handle.read() + head, tag, tail = document.rpartition("") + updated = head + html + tag + tail if tag else document + html + with open(path, "w", encoding="utf-8") as handle: + handle.write(updated) + + def main(): parser = argparse.ArgumentParser(description="Report how evenly each test target's splits divided.") parser.add_argument("--input", default="build/test", help="directory holding cell-times/ and output/, default build/test") parser.add_argument("--output", default=None, - help="existing .html file to append the table to; omitted prints only") + help="existing .html file to insert the table into; omitted prints only") parser.add_argument("--budget-margin", type=int, default=15, help="minutes of margin to keep under a cell's deadline, default 15") args = parser.parse_args() @@ -269,10 +282,9 @@ def main(): if args.output and os.path.isfile(args.output): try: - with open(args.output, "a", encoding="utf-8") as handle: - handle.write(html_report(rows, args.budget_margin)) + write_into_body(args.output, html_report(rows, args.budget_margin)) except OSError as error: - print(f"could not append to {args.output}: {error}", file=sys.stderr) + print(f"could not write the table into {args.output}: {error}", file=sys.stderr) return 0 diff --git a/.build/ci/generate-ci-summary.sh b/.build/ci/generate-ci-summary.sh index b5a34a36dfc7..3767bc43f020 100755 --- a/.build/ci/generate-ci-summary.sh +++ b/.build/ci/generate-ci-summary.sh @@ -71,10 +71,8 @@ EOL # Unguarded, and this script is `sh -e`: a summary the parser could not write is a failure. ${CASSANDRA_DIR}/.build/ci/ci_parser.py --mute --input ${DIST_DIR}/test/output/ --output ${DIST_DIR}/ci_summary.html -# How evenly each target's splits divided, and which came near their cell deadline. After the parser, which -# writes the body this appends to. Guarded: an unbalanced split is a thing to plan, not to fail a build over. -${CASSANDRA_DIR}/.build/ci/cell_balance.py --input ${DIST_DIR}/test --output ${DIST_DIR}/ci_summary.html \ - || echo "failed cell_balance.py" +# How evenly each target's splits divided, and which came near their cell deadline. +${CASSANDRA_DIR}/.build/ci/cell_balance.py --input ${DIST_DIR}/test --output ${DIST_DIR}/ci_summary.html || echo "failed cell_balance.py" exit 0 diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 71d2de7a8f00..2e264f837db0 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -35,10 +35,8 @@ // - cassandra-small + cassandra-${arch}-small : 1 cpu, 1GB ram (alias for above but for any arch) // - cassandra-medium + cassandra-${arch}-medium : 3 cpu, 5GB ram // - cassandra-large + cassandra-${arch}-large : 7 cpu, 16GB ram -// - cassandra-report + cassandra-${arch}-report : 3 cpu, 5GB ram +// - cassandra-report + cassandra-${arch}-report : 2 cpu, 9GB ram // -// cassandra-report is the medium capability on nodes the test fan-out cannot take. generateTestReports is -// its only user and runs last, let it through quickly // // Performance targets required a `cassandra-${arch}-large-dedicated` labelled nodes. // @@ -733,8 +731,10 @@ def generateTestReports() { // copyArtifacts takes >4hrs, hack with manual download sh label: "manual download (instead of copyArtifacts)...", script: """${script_vars} ( mkdir -p build/test - wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip - unzip -x -d build/test -q output.zip ) ${teeSuffix} + wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip + unzip -x -d build/test -q output.zip + ( wget -q ${BUILD_URL}/artifact/test/cell-times/*zip*/cell-times.zip && unzip -x -d build/test -q cell-times.zip ) || echo "no cell-times artefact to download" + ) ${teeSuffix} """ } else { copyArtifacts filter: 'test/cell-times/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true @@ -747,7 +747,7 @@ def generateTestReports() { echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n64 -P"\$(cpus)" xz -f --decompress - # One ant junitreport per test target, three at a time. + # One ant junitreport per test target, up to three at a time: cpus() is this jnlp container's limit, so the report template's 2 cpu runs two. report_jobs=\$(cpus) ; [ "\${report_jobs}" -le 3 ] || report_jobs=3 report_target() { target="\$1"