diff --git a/.build/ci/cell_balance.py b/.build/ci/cell_balance.py new file mode 100755 index 000000000000..1545e5183d17 --- /dev/null +++ b/.build/ci/cell_balance.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""How evenly each test target's splits divided, and which of them came near their deadline. + +What makes a split long: `_split_tests` in ../run-tests.sh deals an alphabetically sorted class list +round-robin with `split -n r/K/N`, which balances the count of classes and knows nothing of their duration. + +Round-robin is required, so that which split holds a test is predictable and easy to find. + +This report cannot tell a structurally long split from one that drew a +slow agent, so change a split count only when consecutive builds agree. + +Usage: + + cell_balance.py [--input build/test] [--output build/ci_summary.html] [--budget-margin 15] + +`--input` holds `cell-times/*.tsv`, one per cell from `recordCellTime` in the Jenkinsfile, and +`output//` of decompressed JUnit XML. A cell records whether it ended ok, hit its deadline, or failed, +so a target that never finishes still gets a row. With no records at all it says so and exits 0: this is not +the check to fail a build on. +""" + +import argparse +import os +import statistics +import sys +import xml.etree.ElementTree as ElementTree + +# Columns of a cell-times record, in order. test_seconds covers every class and not only the twelve .suites +# lists, so the setup figure is exact rather than a residue. outcome is ok, timeout, or failed. Records from +# before either column still read: a shorter record leaves the trailing keys unset. +FIELDS = ("step", "arch", "jdk", "split", "splits", "timeout_hours", "duration_ms", "test_seconds", "outcome") + +# Enough of the worst cell's longest suites to show whether one class dominates or the whole split is heavy, +# which is the distinction that decides what to do. More when over budget, that being the one to act on. +LONGEST_SUITES = 6 +LONGEST_SUITES_OVER_BUDGET = 12 + +# The floor keeps it quiet for cells with no setup worth reporting +UNDER_RECORDED_TEST_SHARE = 0.25 +UNDER_RECORDED_MINIMUM_MINUTES = 5.0 + +# Name the test classes only when the worst cell's longest reaches this +NAME_CLASSES_ABOVE_MINUTES = 5.0 + + +def read_records(cell_times_dir): + """Every cell record, as a list of dicts. A malformed file is skipped rather than fatal.""" + records = [] + if not os.path.isdir(cell_times_dir): + return records + # The .suites files sit in this directory too, so only .tsv is a record. + for name in sorted(n for n in os.listdir(cell_times_dir) if n.endswith(".tsv")): + path = os.path.join(cell_times_dir, name) + try: + with open(path, encoding="utf-8") as handle: + fields = handle.readline().rstrip("\n").split("\t") + if not (len(FIELDS) - 2 <= len(fields) <= len(FIELDS)): + continue + record = dict(zip(FIELDS, fields)) + record["minutes"] = int(record["duration_ms"]) / 60000.0 + record["timed_out"] = record.get("outcome") == "timeout" + record["failed"] = record.get("outcome") == "failed" + + record["test_minutes"] = (float(record["test_seconds"]) / 60.0 + if record.get("test_seconds") else None) + record["deadline_minutes"] = float(record["timeout_hours"]) * 60.0 + record["label"] = f"{record['step']} jdk{record['jdk']} {record['split']}/{record['splits']}" + record["suites"] = read_suites(path[: -len(".tsv")] + ".suites") + records.append(record) + except (OSError, ValueError): + continue + return records + + +def read_suites(path): + """(suite name, minutes) for one cell, longest first, or [] when the file is absent.""" + try: + with open(path, encoding="utf-8") as handle: + fields = [line.rstrip("\n").partition("\t") for line in handle] + return [(name, float(seconds) / 60.0) for seconds, _, name in fields if name] + except (OSError, ValueError): + return [] + + +def summarise(records, budget_margin): + """One row per target, worst first.""" + by_step = {} + for record in records: + by_step.setdefault(record["step"], []).append(record) + + rows = [] + for step, cells in by_step.items(): + durations = sorted(cell["minutes"] for cell in cells) + worst = max(cells, key=lambda cell: cell["minutes"]) + median = statistics.median(durations) + budget = max(cell["deadline_minutes"] for cell in cells) - budget_margin + rows.append({ + "step": step, + "cells": len(cells), + "min": durations[0], + "median": median, + "max": durations[-1], + # Guarded, because a target whose median is zero is a target that ran nothing. + "ratio": durations[-1] / median if median > 0 else 0.0, + "budget": budget, + "over_budget": durations[-1] > budget, + # Counted because neither duration is a plain measurement: a killed cell's is its deadline, and a + # failed cell's stops wherever the failure came. + "timed_out_cells": sum(1 for cell in cells if cell["timed_out"]), + "failed_cells": sum(1 for cell in cells if cell["failed"]), + "worst_timed_out": worst["timed_out"], + "worst_label": worst["label"], + "worst_suites": worst["suites"], + # The cell's own total where it recorded one, and the sum of what it listed otherwise. + "worst_test_minutes": (worst["test_minutes"] if worst["test_minutes"] is not None + else sum(minutes for _, minutes in worst["suites"])), + "worst_test_minutes_exact": worst["test_minutes"] is not None, + }) + rows.sort(key=lambda row: -row["max"]) + return rows + + +def suite_times(target_output_dir): + """(suite name, minutes) for every JUnit suite under one target, longest first.""" + suites = [] + for root, _, names in os.walk(target_output_dir): + for name in names: + if not name.endswith(".xml"): + continue + try: + element = ElementTree.parse(os.path.join(root, name)).getroot() + except (OSError, ElementTree.ParseError): + continue + # A file's root is from ant and from pytest, so both are searched. + for suite in [element] if element.tag == "testsuite" else element.findall(".//testsuite"): + try: + suites.append((suite.get("name") or "?", float(suite.get("time") or 0) / 60.0)) + except ValueError: + continue + suites.sort(key=lambda pair: -pair[1]) + return suites + + +def print_report(rows, output_root, budget_margin): + if not rows: + print("No cell-times records were found, so there is no split balance to report.") + return + + print() + print("Split balance, by test target. `worst` is one cell's whole duration, setup included, against a") + print(f"`budget` of its deadline less {budget_margin} min. Splitting balances the count of classes and" + " not their") + print("duration, so a high w/med is a split that drew slow ones. Confirm against the previous build.") + print() + header = (f"{'target':34s} {'cells':>5s} {'min':>7s} {'median':>7s} {'worst':>7s}" + f" {'w/med':>6s} {'budget':>7s} {'':4s} worst cell") + print(header) + print("-" * len(header)) + for row in rows: + flag = "KILL" if row["worst_timed_out"] else "OVER" if row["over_budget"] else "" + print(f"{row['step']:34s} {row['cells']:5d} {row['min']:7.1f} {row['median']:7.1f}" + f" {row['max']:7.1f} {row['ratio']:6.2f} {row['budget']:7.0f} {flag:4s}" + f" {row['worst_label']}{cell_outcome_note(row)}") + + if not any(row["over_budget"] for row in rows): + print() + print("Every target's worst cell is inside its budget.") + + killed = sum(row["timed_out_cells"] for row in rows) + failed = sum(row["failed_cells"] for row in rows) + if killed or failed: + print() + print(f"Cells killed at their deadline (KILL): {killed}. Cells ended by a failure: {failed}. A killed" + " cell's duration is that deadline, so read it as a lower bound; a failed cell's stops where the" + " failure came, so it can pull min and median down.") + + print() + print("What each worst cell ran. One class near the cell's whole duration cannot be split further;") + print("several of comparable size can be. Classes are named only where the longest reaches" + f" {NAME_CLASSES_ABOVE_MINUTES:.0f} min,") + print("below which none of them is the reason.") + for row in rows: + print() + print_worst_cell(row, output_root) + + +def cell_outcome_note(row): + """What to add after a worst cell's label, when it did not simply finish.""" + if row["worst_timed_out"]: + return " (killed at its deadline)" + other = row["timed_out_cells"] + row["failed_cells"] + return f" (+{other} killed or failed)" if other else "" + + +def print_share(minutes, total, text): + """One breakdown line: minutes, what share of the cell they are, and what spent them.""" + print(f" {minutes:7.1f} min {100 * minutes / total if total else 0:4.0f}% {text}") + + +def print_worst_cell(row, output_root): + """The longest classes of one target's worst cell, and the time that was not test time.""" + flag = " OVER BUDGET" if row["over_budget"] else "" + print(f"{row['worst_label']}: {row['max']:.1f} min against a budget of {row['budget']:.0f}{flag}") + if row["worst_timed_out"]: + print(" killed at its deadline, so this duration is a lower bound and the classes below are only" + " the ones that finished") + + suites, total, tests = row["worst_suites"], row["max"], row["worst_test_minutes"] + if not suites: + fallback = suite_times(os.path.join(output_root, row["step"]))[:LONGEST_SUITES] + if not fallback: + print(" no per-cell suite record, and no JUnit XML for this target either") + return + print(" no per-cell suite record, so these are the longest across the whole target:") + for name, minutes in fallback: + print(f" {minutes:7.1f} min {name}") + return + + longest = max(minutes for _, minutes in suites) + if longest < NAME_CLASSES_ABOVE_MINUTES: + # No class is large enough to be the reason, so the whole test time is one line + print_share(tests, total, f"every test class in this cell, the longest of them {longest:.1f} min") + else: + shown = suites[:LONGEST_SUITES_OVER_BUDGET if row["over_budget"] else LONGEST_SUITES] + for name, minutes in shown: + print_share(minutes, total, name) + other = tests - sum(minutes for _, minutes in shown) + if other > 0.05: + print_share(other, total, "every other test class in this cell") + + setup = total - tests + if setup <= 0: + return + share = tests / total if total else 1.0 + if total >= UNDER_RECORDED_MINIMUM_MINUTES and share < UNDER_RECORDED_TEST_SHARE: + print_share(setup, total, f"setup at most: only {tests:.1f} min of tests was recorded," + f" {100 * share:.0f}% of the cell,") + print(" so tests are missing from the JUnit XML and this is an upper bound") + else: + exact = "" if row["worst_test_minutes_exact"] else ", or a test class this cell did not record" + print_share(setup, total, "not running tests: the node, the image pulls, the compile and the" + f" virtualenv{exact}") + + +def html_report(rows, budget_margin): + parts = ["

Split balance

"] + if not rows: + parts.append("

No cell-times records were found.

") + return "".join(parts) + parts.append( + "

worst is one cell's whole duration, setup included, which is what the cell deadline" + f" governs; budget is that deadline less {budget_margin} minutes of margin." + " Round-robin splitting balances the number of classes in a split and not their duration, so a high" + " worst/median is a split that drew slow classes. Confirm against the previous build" + " before changing a split count.

") + killed = sum(row["timed_out_cells"] for row in rows) + failed = sum(row["failed_cells"] for row in rows) + if killed or failed: + parts.append( + f"

Cells killed at their deadline: {killed}. Cells ended by a failure: {failed}. A killed cell's" + " duration is that deadline, so read it as a lower bound; a failed cell's stops where the failure" + " came, so it can pull min and median down.

") + parts.append("" + "") + for row in rows: + style = " bgcolor='#ffdddd'" if row["over_budget"] else "" + parts.append( + f"" + f"" + f"") + parts.append("
targetcellsminmedianworstworst/medianbudgetworst cell
{row['step']}{row['cells']}{row['min']:.1f}{row['median']:.1f}{row['max']:.1f}{row['ratio']:.2f}{row['budget']:.0f}{row['worst_label']}{cell_outcome_note(row)}
") + return "".join(parts) + + +def write_into_body(path, html): + """Put the table inside the document's body, ci_parser.py having already closed it. + + Insert before the last `` instead, and append only when the file has no body to insert into. + """ + with open(path, encoding="utf-8") as handle: + document = handle.read() + head, tag, tail = document.rpartition("") + updated = head + html + tag + tail if tag else document + html + with open(path, "w", encoding="utf-8") as handle: + handle.write(updated) + + +def main(): + parser = argparse.ArgumentParser(description="Report how evenly each test target's splits divided.") + parser.add_argument("--input", default="build/test", + help="directory holding cell-times/ and output/, default build/test") + parser.add_argument("--output", default=None, + help="existing .html file to insert the table into; omitted prints only") + parser.add_argument("--budget-margin", type=int, default=15, + help="minutes of margin to keep under a cell's deadline, default 15") + args = parser.parse_args() + + records = read_records(os.path.join(args.input, "cell-times")) + rows = summarise(records, args.budget_margin) + print_report(rows, os.path.join(args.input, "output"), args.budget_margin) + + if args.output and os.path.isfile(args.output): + try: + write_into_body(args.output, html_report(rows, args.budget_margin)) + except OSError as error: + print(f"could not write the table into {args.output}: {error}", file=sys.stderr) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.build/ci/generate-ci-summary.sh b/.build/ci/generate-ci-summary.sh index 3828fa2c1e55..3767bc43f020 100755 --- a/.build/ci/generate-ci-summary.sh +++ b/.build/ci/generate-ci-summary.sh @@ -68,7 +68,11 @@ cat >${DIST_DIR}/ci_summary.html < str: maxSize is the only in-cluster record of what a pool can hold, and a pool at zero nodes has no nodes to count, so the check reads it from here. """ + # Two groups per size, each pair summing to that size's instanceCap in jenkins-deployment.yaml. Raise + # these whenever a cap is raised, or the committed values stop passing their own check. groups = [(f"eks-amd64-{size}-ondemand-{n}-{n}cfd1c1", 0, maximum) - for size, maximum in (("large", 80), ("medium", 65), ("small", 25)) for n in (2, 3)] + for size, maximum in (("large", 153), ("medium", 75), ("small", 10), ("report", 2)) + for n in (2, 3)] groups.append(("eks-jenkins-controller-0-2acd8787", 1, 1)) return yaml.safe_dump({"nodeGroups": [ @@ -341,9 +344,9 @@ def test_check_agent_capacity_allows_the_committed_values(self): self.assertEqual(0, self.capacity_check(self.deployed_values(), nodes=self.LARGE_NODE)) def test_check_agent_capacity_blocks_a_cap_above_the_pool(self): - # 200 against the 160 nodes two large groups can hold: 40 agents could never be scheduled, which is + # 400 against the 306 nodes two large groups can hold: 94 agents could never be scheduled, which is # not idle but a churn loop, and is what preceded the 2026-08-11 controller stall - over = self.deployed_values("large", instanceCap=200, instanceCapStr="200") + over = self.deployed_values("large", instanceCap=400, instanceCapStr="400") self.assertEqual(1, self.capacity_check(over)) # a cluster whose ceilings cannot be read leaves it unchecked rather than blocking a valid deploy self.assertEqual(0, self.capacity_check(over, autoscaler=False)) @@ -352,14 +355,14 @@ def test_check_agent_capacity_reads_maxsize_wherever_it_is_published(self): # the live cluster nests a group's maximum under its health condition, and the check also takes it # from the group. Reading the wrong key costs nothing visible: the ceilings come out empty and # every cap passes unchecked, so the shapes are pinned here rather than in a deploy - over = self.deployed_values("large", instanceCap=200, instanceCapStr="200") + over = self.deployed_values("large", instanceCap=400, instanceCapStr="400") for nested in (True, False): self.assertEqual(1, self.capacity_check(over, nested=nested)) self.assertEqual(0, self.capacity_check(self.deployed_values(), nested=nested)) def test_check_agent_capacity_blocks_contradictory_config(self): # the plugin takes the cap from either key, so a disagreement resolves to whichever applies last - self.assertEqual(1, self.capacity_check(self.deployed_values("large", instanceCapStr="200"))) + self.assertEqual(1, self.capacity_check(self.deployed_values("large", instanceCapStr="400"))) # a nodeSelector the live nodes contradict strands every agent of that size typo = self.deployed_values("large", nodeSelector="cassandra.jenkins.agent.large=ture") self.assertEqual(1, self.capacity_check(typo, nodes=self.LARGE_NODE)) diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 83e58b67f81c..cf17c8dc195b 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -35,6 +35,8 @@ // - cassandra-small + cassandra-${arch}-small : 1 cpu, 1GB ram (alias for above but for any arch) // - cassandra-medium + cassandra-${arch}-medium : 3 cpu, 5GB ram // - cassandra-large + cassandra-${arch}-large : 7 cpu, 16GB ram +// - cassandra-report + cassandra-${arch}-report : 2 cpu, 9GB ram +// // // Performance targets required a `cassandra-${arch}-large-dedicated` labelled nodes. // @@ -61,6 +63,28 @@ import groovy.transform.Field @Field List archsSupported = ["amd64", "arm64"] @Field List pythonsSupported = ["3.8", "3.11", "3.12", "3.13"] @Field String pythonDefault = "3.8" + +// Shell defining cpus(), to prepend to any `sh` script fanning out with `xargs -P` or `xz -T` +@Field String cpusShell = ''' +cpus() { + cpus_n= + # The v2 line of /proc/self/cgroup is "0::", this cgroup relative to the host's root. + cpus_own="$(sed -n 's/^0:://p' /proc/self/cgroup 2>/dev/null | head -1)" + for cpus_file in "/sys/fs/cgroup${cpus_own}/cpu.max" /sys/fs/cgroup/cpu.max ; do + cpus_quota= cpus_period= + read -r cpus_quota cpus_period 2>/dev/null < "${cpus_file}" || continue + # "max " is a cgroup with no cpu limit, where nproc is right. + case "${cpus_quota}${cpus_period}" in ""|*[!0-9]*) continue ;; esac + [ "${cpus_period:-0}" -gt 0 ] || continue + cpus_n=$(( cpus_quota / cpus_period )) + break + done + [ -n "${cpus_n}" ] || cpus_n="$(nproc 2>/dev/null || echo 1)" + [ "${cpus_n}" -ge 1 ] 2>/dev/null || cpus_n=1 + echo "${cpus_n}" +} +''' + /** CONSTANTS end **********************************/ pipeline { @@ -199,7 +223,7 @@ def tasks() { 'test-latest': [splits: 20], 'test-compression': [splits: 20], 'stress-test': [splits: 1, size: 'small'], - 'test-burn': [splits: 4], + 'test-burn': [splits: 5], 'long-test': [splits: 4], 'test-oa': [splits: 20], 'test-system-keyspace-directory': [splits: 20], @@ -416,52 +440,44 @@ def test(command, cell) { fetchDockerImages(['ubuntu-test']) def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}" def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz" - def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail - script_vars = "${script_vars} python_version=\'${cell.python}\'" - script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" - if ("cqlsh-test" == cell.step) { - script_vars = "${script_vars} cython=\'${cell.cython}\'" - } - script_vars = fetchDTestsSource(command, script_vars) - timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes - def timer = System.currentTimeMillis() - try { - buildJVMDTestJars(cell, script_vars, logfile) - script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" - def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true - dir("build") { - archiveArtifacts artifacts: "${logfile}", fingerprint: true - } - if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } - } catch (exc) { - if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { - def descriptions = [] - for (def cause in exc.getCauses()) { - echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" - if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption') || cause.getClass().getName().contains('ParallelStep$FailFastCause')) { - throw exc // user abort or fail-fast — do not retry + def script_vars = testScriptVars(command, cell) + def timer = System.currentTimeMillis() + def outcome = "ok" + try { + timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes + try { + buildJVMDTestJars(cell, script_vars, logfile) + script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" + def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true + dir("build") { + archiveArtifacts artifacts: "${logfile}", fingerprint: true + } + if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } + } catch (exc) { + outcome = "failed" + if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { + def descriptions = [] + for (def cause in exc.getCauses()) { + echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" + if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption') || cause.getClass().getName().contains('ParallelStep$FailFastCause')) { + throw exc // user abort or fail-fast — do not retry + } + if (cause.getClass().getName().contains('TimeoutStepExecution')) { outcome = "timeout" } + descriptions.add(cause.getShortDescription()) } - descriptions.add(cause.getShortDescription()) + error("Retryable interruption: ${descriptions.join(', ')}") } - error("Retryable interruption: ${descriptions.join(', ')}") + throw exc + } finally { + def duration = System.currentTimeMillis() - timer + def formattedTime = String.format("%tT.%tL", duration, duration) + echo "Time ${cell.step}${cell_suffix}: ${formattedTime}" } - throw exc - } finally { - def duration = System.currentTimeMillis() - timer - def formattedTime = String.format("%tT.%tL", duration, duration) - echo "Time ${cell.step}${cell_suffix}: ${formattedTime}" - } - } - dir("build") { - organiseTestResultFiles(cell, cell_suffix) - if (!cell.step.startsWith("microbench")) { - junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] } - debugOomKiller() - compressTestResultFiles() - archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true - copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) + } finally { + recordCellTime(cell, cell_suffix, splits, command.timeout_hours, System.currentTimeMillis() - timer, outcome) } + processResults(cell, cell_suffix, logfile, splits) } finally { cleanAgent(cell.step) } @@ -490,6 +506,19 @@ def fetchDTestsSource(command, script_vars) { return script_vars } +// The prefix every test cell's `sh` script starts with: bash with pipefail, which the tee needs, and the +// variables run-tests.sh reads. fetchDTestsSource is here because it both checks out the python dtests and +// names their directory in the prefix. +def testScriptVars(command, cell) { + def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail + script_vars = "${script_vars} python_version=\'${cell.python}\'" + script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" + if ("cqlsh-test" == cell.step) { + script_vars = "${script_vars} cython=\'${cell.cython}\'" + } + return fetchDTestsSource(command, script_vars) +} + def buildJVMDTestJars(cell, script_vars, logfile) { if (cell.step.startsWith("jvm-dtest-upgrade")) { try { @@ -616,6 +645,78 @@ def _stash(cell) { stash name: "${cell.arch}_${cell.jdk}" } +// One line per cell, for the split balance table .build/ci/cell_balance.py puts in ci_summary.html. +// +// The duration is the cell's own, not the sum of its tests: the cell deadline covers the setup before the +// first test too. +// +// outcome is ok, timeout, or failed. A timed-out cell's duration is its deadline and so a lower bound, and a +// failed cell's may be short because the failure ended it. +def recordCellTime(cell, cell_suffix, splits, timeout_hours, duration_ms, outcome) { + try { + dir("build") { + writeCellTime(cell, cell_suffix, splits, timeout_hours, duration_ms, outcome) + archiveArtifacts artifacts: "test/cell-times/**", fingerprint: true + } + } catch (hudson.AbortException | IOException exc) { + echo "no cell time recorded for ${cell.step}${cell_suffix}: ${exc}" + } +} + +def processResults(cell, cell_suffix, logfile, splits) { + dir("build") { + organiseTestResultFiles(cell, cell_suffix) + if (!cell.step.startsWith("microbench")) { + junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml" + } + debugOomKiller() + compressTestResultFiles() + archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true + copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) + } +} + +// The .tsv and the .suites of one cell, written in build/ +def writeCellTime(cell, cell_suffix, splits, timeout_hours, duration_ms, outcome) { + // Both files are written before organiseTestResultFiles: here test/output/ holds only this cell's results, + // and one step later they are merged with the split gone from every path, so no report can say which cell + // ran which test again. + // + // Aggregated by a testcase's classname. The cell's whole test time is recorded too, so + // cell_balance.py can subtract it from the duration and state the setup time spent exactly. + sh label: "recording cell time...", script: """ + mkdir -p test/cell-times + suites='test/cell-times/${cell.step}${cell_suffix}.suites' + + # \\t, longest first, with the total on stderr. grep -o rather than a line-oriented + # read, because pytest puts many testcase elements on one line. A parser would be tidier; this runs + # once per cell, of which a build has over a thousand. + find test/output -type f -name '*.xml' -print0 2>/dev/null \\ + | xargs -0 -r grep -ho ']*>' 2>/dev/null \\ + | awk ' + { + cls = ""; secs = "" + # time=" is anchored on a space because name=" is a substring of classname=". Anchoring both + # costs nothing and records the trap. + if (match(\$0, /classname="[^"]*"/)) { cls = substr(\$0, RSTART + 11, RLENGTH - 12) } + if (match(\$0, /[ \\t]time="[^"]*"/)) { secs = substr(\$0, RSTART + 7, RLENGTH - 8) } + if (cls != "" && secs != "") { total[cls] += secs; grand += secs } + } + END { + for (c in total) { printf "%.3f\\t%s\\n", total[c], c } + printf "%.3f\\n", grand > "/dev/stderr" + }' 2>test/cell-times/grand \\ + | sort -rn | head -12 > "\${suites}" || true + test_seconds="\$(cat test/cell-times/grand 2>/dev/null || echo 0)" + rm -f test/cell-times/grand + + printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \\ + '${cell.step}' '${cell.arch}' '${cell.jdk}' '${cell.split}' '${splits}' \\ + '${timeout_hours}' '${duration_ms}' "\${test_seconds}" '${outcome}' \\ + > 'test/cell-times/${cell.step}${cell_suffix}.tsv' + """ +} + def organiseTestResultFiles(cell, cell_suffix) { sh label: "organise test result files...", script: """ mkdir -p test/output/${cell.step} @@ -634,9 +735,11 @@ def debugOomKiller() { } def compressTestResultFiles() { - sh label: "compress test result files...", script: """ + sh label: "compress test result files...", script: """${cpusShell} { set +x; } 2>/dev/null - find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f + # -n64, not -n1: one xz per file is a fork and an exec for a few kilobytes, and 64 per process still + # leaves more batches than parallel slots. cpus(), not nproc: see cpusShell. + find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n64 -P"\$(cpus)" xz -f echo "\$(find test/output -type f -name "*.xml.xz" | wc -l) test result files compressed" """ } @@ -646,7 +749,7 @@ def compressTestResultFiles() { ///////////////////////////////////////// def generateTestReports() { - node("cassandra-medium") { + node("cassandra-report") { cleanAgent("generateTestReports") checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]]) def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz" @@ -657,35 +760,52 @@ def generateTestReports() { // copyArtifacts takes >4hrs, hack with manual download sh label: "manual download (instead of copyArtifacts)...", script: """${script_vars} ( mkdir -p build/test - wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip - unzip -x -d build/test -q output.zip ) ${teeSuffix} + wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip + unzip -x -d build/test -q output.zip + ( wget -q ${BUILD_URL}/artifact/test/cell-times/*zip*/cell-times.zip && unzip -x -d build/test -q cell-times.zip ) || echo "no cell-times artefact to download" + ) ${teeSuffix} """ } else { - copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true + copyArtifacts filter: 'test/cell-times/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true } // merge and summarise test reports if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name "*.xml.xz" -print -quit)"', returnStatus: true) == 0) { // merge splits for each target's test report, other axes are kept separate - // TODO parallelised for loop // TODO results_details.tar.xz needs to include all logs for failed tests - sh label: "merging splits test reports...", script: """${script_vars} ( + sh label: "merging splits test reports...", script: """${script_vars}${cpusShell} ( echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l - find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f --decompress - - for target in \$(ls build/test/output/) ; do - if test -d build/test/output/\${target} ; then - mkdir -p build/test/reports/\${target} - echo "Report for \${target} (\$(find build/test/output/\${target} -name '*.xml' | wc -l) test files)" - CASSANDRA_DOCKER_ANT_OPTS="-Dbuild.test.output.dir=build/test/output/\${target} -Dbuild.test.report.dir=build/test/reports/\${target}" + find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n64 -P"\$(cpus)" xz -f --decompress + + # One ant junitreport per test target, up to three at a time: cpus() is this jnlp container's limit, so the report template's 2 cpu runs two. + report_jobs=\$(cpus) ; [ "\${report_jobs}" -le 3 ] || report_jobs=3 + report_target() { + target="\$1" + test -d "build/test/output/\${target}" || return 0 + mkdir -p "build/test/reports/\${target}" + # Held and printed as one block per target: concurrent runs otherwise interleave line by line + report_log="\$(mktemp)" + { + echo "Report for \${target} (\$(find "build/test/output/\${target}" -name '*.xml' | wc -l) test files)" + # -Xmx2g per concurrent ant jvm; unset, each takes a quarter of dind's limit. Exported inside + # the function, which xargs runs as its own process, so targets cannot see each other's value. + CASSANDRA_DOCKER_ANT_OPTS="-Xmx2g -Dbuild.test.output.dir=build/test/output/\${target} -Dbuild.test.report.dir=build/test/reports/\${target}" export CASSANDRA_DOCKER_ANT_OPTS .build/docker/_docker_run.sh debian-build.docker ci/generate-test-report.sh - fi - done + } > "\${report_log}" 2>&1 + report_status=\$? + cat "\${report_log}" ; rm -f "\${report_log}" + return \${report_status} + } + export -f report_target + ls build/test/output/ | xargs -r -n1 -P"\${report_jobs}" bash -xc 'report_target "\$0"' .build/docker/_docker_run.sh debian-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh" tar -cf build/results_details.tar -C build/test/ reports - xz -8f build/results_details.tar ) ${teeSuffix} + # -T with a count: xz's own -T0 reads nproc and would thread for the node, not this container. + # --memlimit-compress caps the total: -8 takes ~700MB per thread, and against a limit xz drops + # threads (then the preset) instead of growing until the kernel's OOM killer takes it. + xz -8f -T"\$(cpus)" --memlimit-compress=2g build/results_details.tar ) ${teeSuffix} """ dir('build/') { @@ -762,4 +882,4 @@ def emailContent() { ------------------------------------------------------------------------------- For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/ ''' -} \ No newline at end of file +} diff --git a/.jenkins/k8s/README.md b/.jenkins/k8s/README.md index 14ac7175fdd1..b8d631d7595a 100644 --- a/.jenkins/k8s/README.md +++ b/.jenkins/k8s/README.md @@ -24,15 +24,21 @@ ZONE="us-central1-c" gcloud container clusters create ${CLUSTER_NAME} --machine-type e2-standard-8 --disk-type=pd-ssd --num-nodes 1 --node-labels=cassandra.jenkins.controller=true --autoscaling-profile optimize-utilization --zone ${ZONE} # small resource nodes -gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE} +gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=20 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE} + +# report resource nodes, for the generateTestReports stage (the agent-dind-report podTemplate) +# a standard machine, not highcpu: that template's dind container has a 9G memory limit +gcloud container node-pools create agents-report --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=4 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.report=true --zone ${ZONE} # medium resource nodes # preference (by cost): n2-highcpu-8, c3-highcpu-8, n4-highcpu-8, n1-highcpu-16 -gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE} +gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=150 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE} # large resource nodes -gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE} +gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=306 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE} +# Each --max-nodes above is that size's agent.podTemplates.*.instanceCap in jenkins-deployment.yaml, one agent per node. +# Raise the pool first: `.build/run-ci --only-setup` blocks a deploy whose instanceCap exceeds the nodes its pool can hold. # For each sized resource nodes, pick any machine type that fits, those listed above should work and be the most cost-effective, but this can change region to region # See https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/Jenkinsfile#L35-L38 # and agent.podTemplates.*.resourceLimitCpu and agent.podTemplates.*.resourceLimitMemory (adding gke/eks requirements) in https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/k8s/jenkins-deployment.yaml diff --git a/.jenkins/k8s/jenkins-deployment.yaml b/.jenkins/k8s/jenkins-deployment.yaml index 6e59340a13a8..37e257da19b5 100644 --- a/.jenkins/k8s/jenkins-deployment.yaml +++ b/.jenkins/k8s/jenkins-deployment.yaml @@ -170,11 +170,16 @@ controller: enabled: true agent: disableDefaultAgent: true - maxRequestsPerHostStr: "3200" - containerCap: 300 + # Concurrent requests the plugin's client may hold to one host, the API server being its only host. The + # plugin's default of 32 is for a cloud of a few agents. + maxRequestsPerHostStr: "5120" + # The cap across every podTemplate below, and equal to the sum of their four instanceCaps, so on a cluster + # whose pools can each reach their own cap this never binds. Deliberately: a lower shared cap does not + # divide the shortfall evenly, it hands it to whichever pool holds its agents longest, which is large. + containerCap: 480 node-selector: cassandra.jenkins.agent: true - waitForPodSec: "180" + waitForPodSec: "900" # Reap agent pods the controller no longer tracks. A restart of the controller JVM clears its in-memory # agent registry, and because agent pods are bare pods with no ownerReference nothing else deletes them. # Preferred over a pod activeDeadlineSeconds, which cannot distinguish an orphan from a long (6h) build. @@ -206,13 +211,15 @@ agent: nodeSelector: 'cassandra.jenkins.agent.small=true' # 0 = no pod lifetime cap: idleMinutes reuse means a pod's age is unrelated to any one build's timeout. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' - idleMinutes: 1 - # should match the small pool's 50 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. - instanceCap: 50 - instanceCapStr: "50" + idleMinutes: 5 + # This pool's ceiling, not the cluster's; agent.containerCap above is that, and is the one to lower + # when the account cannot hold this many nodes. Must not exceed the small pool's own maximum + # (README's --max-nodes), one agent per node. No other template selects that pool. + instanceCap: 20 + instanceCapStr: "20" nodeUsageMode: "NORMAL" showRawYaml: 'true' - slaveConnectTimeout: '30' + slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp @@ -308,19 +315,126 @@ agent: volumeMounts: - name: docker-storage mountPath: /var/lib/docker + agent-dind-report: | + - name: agent-dind-report + label: agent-dind cassandra-report cassandra-amd64-report + nodeSelector: 'cassandra.jenkins.agent.report=true' + activeDeadlineSeconds: '0' + idleMinutes: 5 + instanceCap: 4 + instanceCapStr: "4" + nodeUsageMode: "NORMAL" + showRawYaml: 'true' + slaveConnectTimeout: '600' + yamlMergeStrategy: override + containers: + - name: jnlp + # https://github.com/jenkinsci/kubernetes-plugin#pipeline-support + alwaysPullImage: true + envVars: + - envVar: + key: DOCKER_TLS_CERTDIR + value: /certs/client/ + - envVar: + key: DOCKER_CERT_PATH + value: /certs/client/ + - envVar: + key: DOCKER_TLS_VERIFY + value: 'true' + - envVar: + key: DOCKER_HOST + value: tcp://localhost:2376 + - envVar: + key: JENKINS_JAVA_OPTS + value: '-Dorg.jenkinsci.plugins.durabletask.BourneShellScript.USE_BINARY_WRAPPER=true' + image: apache.jfrog.io/cassan-docker/apache/cassandra-jenkins-k8s + livenessProbe: + failureThreshold: '0' + initialDelaySeconds: '0' + periodSeconds: '0' + successThreshold: '0' + timeoutSeconds: '0' + privileged: 'true' + resourceRequestCpu: 1 + resourceLimitCpu: 2 + resourceRequestMemory: 1G + resourceLimitMemory: 2400M + # the workspace emptyDir: the downloaded output.zip, the decompressed test xml, and the tar + resourceRequestEphemeralStorage: 10Gi + resourceLimitEphemeralStorage: 20Gi + ttyEnabled: 'true' + workingDir: /home/jenkins/agent + - name: dind + alwaysPullImage: 'false' + envVars: + - envVar: + key: DOCKER_TLS_CERTDIR + value: /certs + - envVar: + key: "DOCKER_IPTABLES_LEGACY" + value: "1" + image: docker:dind + args: "--default-address-pool base=192.168.96.0/20,size=24" # overwrite docker subnet in case of overlapping + livenessProbe: + failureThreshold: '0' + initialDelaySeconds: '0' + periodSeconds: '0' + successThreshold: '0' + timeoutSeconds: '0' + privileged: 'true' + resourceRequestCpu: 2 + resourceLimitCpu: 5 + resourceRequestMemory: 3400M + # 9G against the medium template's 5G: generateTestReports now runs several ant junitreport + # jvms at once, each in a container this daemon holds. Moves with the Jenkinsfile's + # report_jobs and -Xmx. + resourceLimitMemory: 9G + # docker's images and containers, in the docker-storage emptyDir + resourceRequestEphemeralStorage: 40Gi + resourceLimitEphemeralStorage: 60Gi + ttyEnabled: 'true' + workingDir: /home/jenkins/agent + volumes: + # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit + - emptyDirVolume: + memory: 'false' + mountPath: /certs + # limit one agent pod per node for simpler operations (like orphan cleanup) + yaml: | + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: jenkins/cassius-jenkins-agent + operator: In + values: + - "true" + topologyKey: kubernetes.io/hostname + # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge. + # 60Gi bounds the images and their containers alone, of the pod's 80Gi + # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow. + volumes: + - name: docker-storage + emptyDir: + sizeLimit: 60Gi + containers: + - name: dind + volumeMounts: + - name: docker-storage + mountPath: /var/lib/docker agent-dind-medium: | - name: agent-dind-medium label: agent-dind cassandra-medium cassandra-amd64-medium nodeSelector: 'cassandra.jenkins.agent.medium=true' - # 0 = no pod lifetime cap: idleMinutes reuse means a pod's age is unrelated to any one build's timeout. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' - idleMinutes: 1 - # should match the medium pools 100 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. - instanceCap: 100 - instanceCapStr: "100" + idleMinutes: 5 + instanceCap: 150 + instanceCapStr: "150" nodeUsageMode: "NORMAL" showRawYaml: 'true' - slaveConnectTimeout: '30' + slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp @@ -420,17 +534,14 @@ agent: - name: agent-dind-large label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated nodeSelector: 'cassandra.jenkins.agent.large=true' - # 0 = no pod lifetime cap. A microbench cell runs up to 6h (timeout_hours in the Jenkinsfile) - # and idleMinutes reuse extends a pod's age past any one build, so age cannot stand in for - # health here. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' - idleMinutes: 1 - # should match the large pools 160 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. - instanceCap: 160 - instanceCapStr: "160" + idleMinutes: 5 + instanceCap: 306 + instanceCapStr: "306" nodeUsageMode: "NORMAL" showRawYaml: 'true' - slaveConnectTimeout: '30' + # Raised from 30 seconds. See the small template above. + slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp