From 7d044ad1ed4754c4513dbeb34877a3c5e482277f Mon Sep 17 00:00:00 2001 From: oddly Date: Tue, 4 Aug 2026 09:11:20 +0200 Subject: [PATCH 1/4] fix(ci): make memory-gate visible, fail loudly, and stop starving roles_calculation The molecule capacity-gate retry loop in shared/create.yml was set to retries: 360 (3 hours) but the workflows that call it have timeouts of 20-120 minutes, so a starved scenario was always killed by the workflow timeout before its own retries ran out. On top of that, Ansible's FAILED - RETRYING messages sit in Python's stdout buffer under the non-TTY GHA log, so the run looked like a silent hang for 45+ minutes before the last cancel showed up. Between them, a queue-starvation event and a genuinely broken scenario were impossible to tell apart. Cap retries at 30 (= 15 min) so the task fails loudly with the last 'No capacity: X committed + Y needed > Z available' stdout, well inside every workflow's timeout. Set PYTHONUNBUFFERED=1 in the molecule workflow env so the retry chatter surfaces in real time. Also shrink the four elasticsearch_roles_calculation containers from 4 GB to 2 GB each (16 GB -> 8 GB). elasticsearch_heap is 1 GB and the scenario only exercises the role-calculation code path, so 2 GB is plenty of headroom. Halving the memory ask is what lets the scenario clear the gate under peak concurrency instead of retrying until the workflow timeout. --- .github/workflows/molecule.yml | 6 ++++++ .../elasticsearch_roles_calculation/molecule.yml | 13 +++++++++---- molecule/shared/create.yml | 9 ++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 74ce548..c21c6d8 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -36,6 +36,12 @@ jobs: env: COLLECTION_NAMESPACE: oddly COLLECTION_NAME: elasticstack + # Ansible/molecule are Python; without PYTHONUNBUFFERED the + # "FAILED - RETRYING (N retries left)" messages that Ansible emits + # from the wait-for-capacity retry loop sit in Python's stdout + # buffer and only flush when the buffer fills, making the log look + # like a silent hang for tens of minutes. + PYTHONUNBUFFERED: '1' ANSIBLE_PIPELINING: 'true' ANSIBLE_GATHERING: smart ANSIBLE_ANY_ERRORS_FATAL: 'true' diff --git a/molecule/elasticsearch_roles_calculation/molecule.yml b/molecule/elasticsearch_roles_calculation/molecule.yml index cd8fa09..2f4863e 100644 --- a/molecule/elasticsearch_roles_calculation/molecule.yml +++ b/molecule/elasticsearch_roles_calculation/molecule.yml @@ -7,21 +7,26 @@ dependency: driver: name: default platforms: + # 2 GB per container: the scenario runs elasticsearch_heap: 1 (= 1 GB) + # and only exercises the role-count-calculation code path, not any + # ingest workload. The default 4 GB was making the 16 GB total block + # the memory-capacity gate under peak concurrency and starve the + # scenario until the 45-min workflow timeout cancelled it. - name: "es-calc1-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - elasticsearch distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + memory_mb: 2048 - name: "es-calc2-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - elasticsearch distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + memory_mb: 2048 - name: "es-calc3-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - elasticsearch distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + memory_mb: 2048 # Separate single-node "monitoring" cluster, run in a second play against # this group to catch the regression from #143 where group_by accumulated # elasticsearch_role_master across plays. @@ -29,7 +34,7 @@ platforms: groups: - elasticsearch_mon distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/shared/create.yml b/molecule/shared/create.yml index 9a5ff36..77ff50b 100644 --- a/molecule/shared/create.yml +++ b/molecule/shared/create.yml @@ -126,7 +126,14 @@ REMOTE_SCRIPT changed_when: true register: _launch_result - retries: 360 + # 30 retries × 30s = 15 min. Deliberately shorter than the workflow + # timeout (45 min for the standard molecule.yml call, 120 min for + # full_stack) so the task fails loudly with the last "No capacity: + # …" stdout instead of being silently cancelled by the workflow + # timeout — that made the queue-starvation cases very hard to + # diagnose (see PR investigating the elasticsearch_roles_calculation + # / elasticstack_default post-outage hangs). + retries: 30 delay: 30 until: _launch_result.rc == 0 From 2fa93a3ac03ed460b884fdb73ee053a95b4d2816 Mon Sep 17 00:00:00 2001 From: oddly Date: Tue, 4 Aug 2026 14:14:48 +0200 Subject: [PATCH 2/4] fix(ci): raise test_full_stack timeout to 180 min MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The elasticstack_default scenario is heavy on its own (70-105 min in isolation) and the memory-capacity gate wait tacks another 10-30 min onto the front on a busy runner. The old 120-min ceiling was cancelling otherwise-successful runs even after the retry cap fix landed in the same PR — the scenario just needs more headroom than that under load. 180 min gives comfortable margin without letting truly-stuck runs sit around forever; combined with retries: 30 (= 15 min) in create.yml, a genuinely broken run still surfaces long before the workflow cap. --- .github/workflows/test_full_stack.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 15113b3..164feb5 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -63,7 +63,13 @@ jobs: runs-on: self-hosted needs: [changes, lint_full] if: needs.changes.outputs.should_test == 'true' || github.event_name != 'pull_request' - timeout-minutes: 120 + # 180 min: elasticstack_default is the heaviest scenario and observed + # runs range 70-105 min in isolation, but the extra queue wait added + # by the memory-capacity gate on a busy runner can push a single job + # past the previous 120-min cap. The memory-gate retry cap in + # shared/create.yml is now 15 min so this ceiling is only reached + # when the scenario genuinely runs long, not when the queue starves it. + timeout-minutes: 180 env: COLLECTION_NAMESPACE: oddly From 6f23219a33e28800216b1ae9b262666d7eeb3d6c Mon Sep 17 00:00:00 2001 From: oddly Date: Tue, 4 Aug 2026 16:45:25 +0200 Subject: [PATCH 3/4] fix(ci): cap concurrent full_stack matrix jobs at 6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once the retry cap surfaced 'No capacity: 115200MB committed + 20480MB needed > 3584MB available' cleanly, it was clear the failure was legit memory pressure, not queue-observability. The 16-combo PR matrix (2 distros × 4 scenarios × 2 releases) with no max-parallel could put 8+ elasticstack_default and cert_renewal jobs on the shared 131 GB host at once, and the memory-gate then starves whichever scenarios were slowest to grab a slot. max-parallel: 6 keeps committed memory around 90-100 GB with head-room for the elasticsearch / kibana / logstash / beats role workflows that run in parallel to this one on the same host. --- .github/workflows/test_full_stack.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 164feb5..4f265d6 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -91,6 +91,13 @@ jobs: strategy: fail-fast: false + # Each scenario/distro/release combo needs 15-25 GB of container RAM on + # the shared incus-ci host. The memory-capacity gate in shared/create.yml + # eventually admits every job, but without a matrix-level cap the full + # 16-combo PR matrix would swamp the 131 GB host and starve the biggest + # scenarios (elasticstack_default, cert_renewal). 6 concurrent slots keep + # committed memory around 90-100 GB with head-room to spare. + max-parallel: 6 matrix: distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux9","debian13"]') || (inputs.distros != '' && fromJSON(inputs.distros)) || fromJSON('["rockylinux9","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]') }} scenario: From 44e9975d9c6a929097de4ec97dec30805614f31b Mon Sep 17 00:00:00 2001 From: oddly Date: Wed, 5 Aug 2026 10:04:24 +0200 Subject: [PATCH 4/4] fix(ci): route test_full_stack and test_elasticsearch_upgrade through the memory gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The heavy standalone workflows never called scripts/wait-for-memory.sh because they were built by copy-paste from an earlier revision of the molecule.yml wrapper that did not yet have the admission gate. So every molecule_full_stack_every_os / upgrade_* / etc job barged straight past the /tmp/molecule-gate reservation and only the per-launch flock in shared/create.yml was catching them — after they had already committed enough memory in aggregate to starve each other. Add the Acquire / Release memory slot steps to both workflows (mirroring the reusable molecule.yml layout) so they participate in the same reservation pool as everything else. All the scenarios these workflows run are already in the REQ table in wait-for-memory.sh, so no table changes are needed. --- .github/workflows/test_elasticsearch_upgrade.yml | 14 ++++++++++++++ .github/workflows/test_full_stack.yml | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/.github/workflows/test_elasticsearch_upgrade.yml b/.github/workflows/test_elasticsearch_upgrade.yml index ac9d6b7..7189f28 100644 --- a/.github/workflows/test_elasticsearch_upgrade.yml +++ b/.github/workflows/test_elasticsearch_upgrade.yml @@ -112,6 +112,9 @@ jobs: mkdir -p ~/.ssh ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + - name: Acquire memory slot + run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + - name: Test with molecule run: molecule test -s ${{ matrix.scenario }} --destroy=never env: @@ -125,6 +128,10 @@ jobs: MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + - name: Release memory slot + if: always() + run: bash scripts/wait-for-memory.sh release + - name: Collect and upload diagnostics if: failure() uses: ./.github/actions/collect-diagnostics @@ -227,6 +234,9 @@ jobs: mkdir -p ~/.ssh ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + - name: Acquire memory slot + run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + - name: Test with molecule run: molecule test -s ${{ matrix.scenario }} --destroy=never env: @@ -240,6 +250,10 @@ jobs: MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + - name: Release memory slot + if: always() + run: bash scripts/wait-for-memory.sh release + - name: Collect and upload diagnostics if: failure() uses: ./.github/actions/collect-diagnostics diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 4f265d6..719df7d 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -153,6 +153,14 @@ jobs: mkdir -p ~/.ssh ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + - name: Acquire memory slot + # Coordinated admission gate — reserves the scenario's memory_mb on + # the shared incus-ci host so the biggest scenarios don't trample + # each other or the parallel test_role_* / test_elasticsearch_* + # workflows. This step was missing from the standalone workflow; + # the reusable molecule.yml wrapper has always had it. + run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + - name: Converge run: molecule converge -s ${{ matrix.scenario }} env: @@ -191,6 +199,13 @@ jobs: MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + - name: Release memory slot + # Runs whether converge/verify/idempotence passed or failed, so the + # reservation is freed promptly. Stale reservations (>1h) are also + # GC'd by the next acquire under the same lock as a safety net. + if: always() + run: bash scripts/wait-for-memory.sh release + - name: Collect and upload diagnostics if: failure() uses: ./.github/actions/collect-diagnostics