diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 74ce548..c21c6d8 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -36,6 +36,12 @@ jobs: env: COLLECTION_NAMESPACE: oddly COLLECTION_NAME: elasticstack + # Ansible/molecule are Python; without PYTHONUNBUFFERED the + # "FAILED - RETRYING (N retries left)" messages that Ansible emits + # from the wait-for-capacity retry loop sit in Python's stdout + # buffer and only flush when the buffer fills, making the log look + # like a silent hang for tens of minutes. + PYTHONUNBUFFERED: '1' ANSIBLE_PIPELINING: 'true' ANSIBLE_GATHERING: smart ANSIBLE_ANY_ERRORS_FATAL: 'true' diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 15113b3..4f265d6 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -63,7 +63,13 @@ jobs: runs-on: self-hosted needs: [changes, lint_full] if: needs.changes.outputs.should_test == 'true' || github.event_name != 'pull_request' - timeout-minutes: 120 + # 180 min: elasticstack_default is the heaviest scenario and observed + # runs range 70-105 min in isolation, but the extra queue wait added + # by the memory-capacity gate on a busy runner can push a single job + # past the previous 120-min cap. The memory-gate retry cap in + # shared/create.yml is now 15 min so this ceiling is only reached + # when the scenario genuinely runs long, not when the queue starves it. + timeout-minutes: 180 env: COLLECTION_NAMESPACE: oddly @@ -85,6 +91,13 @@ jobs: strategy: fail-fast: false + # Each scenario/distro/release combo needs 15-25 GB of container RAM on + # the shared incus-ci host. The memory-capacity gate in shared/create.yml + # eventually admits every job, but without a matrix-level cap the full + # 16-combo PR matrix would swamp the 131 GB host and starve the biggest + # scenarios (elasticstack_default, cert_renewal). 6 concurrent slots keep + # committed memory around 90-100 GB with head-room to spare. + max-parallel: 6 matrix: distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux9","debian13"]') || (inputs.distros != '' && fromJSON(inputs.distros)) || fromJSON('["rockylinux9","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]') }} scenario: diff --git a/molecule/elasticsearch_roles_calculation/molecule.yml b/molecule/elasticsearch_roles_calculation/molecule.yml index cd8fa09..2f4863e 100644 --- a/molecule/elasticsearch_roles_calculation/molecule.yml +++ b/molecule/elasticsearch_roles_calculation/molecule.yml @@ -7,21 +7,26 @@ dependency: driver: name: default platforms: + # 2 GB per container: the scenario runs elasticsearch_heap: 1 (= 1 GB) + # and only exercises the role-count-calculation code path, not any + # ingest workload. The default 4 GB was making the 16 GB total block + # the memory-capacity gate under peak concurrency and starve the + # scenario until the 45-min workflow timeout cancelled it. - name: "es-calc1-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - elasticsearch distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + memory_mb: 2048 - name: "es-calc2-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - elasticsearch distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + memory_mb: 2048 - name: "es-calc3-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - elasticsearch distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + memory_mb: 2048 # Separate single-node "monitoring" cluster, run in a second play against # this group to catch the regression from #143 where group_by accumulated # elasticsearch_role_master across plays. @@ -29,7 +34,7 @@ platforms: groups: - elasticsearch_mon distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/shared/create.yml b/molecule/shared/create.yml index 9a5ff36..77ff50b 100644 --- a/molecule/shared/create.yml +++ b/molecule/shared/create.yml @@ -126,7 +126,14 @@ REMOTE_SCRIPT changed_when: true register: _launch_result - retries: 360 + # 30 retries × 30s = 15 min. Deliberately shorter than the workflow + # timeout (45 min for the standard molecule.yml call, 120 min for + # full_stack) so the task fails loudly with the last "No capacity: + # …" stdout instead of being silently cancelled by the workflow + # timeout — that made the queue-starvation cases very hard to + # diagnose (see PR investigating the elasticsearch_roles_calculation + # / elasticstack_default post-outage hangs). + retries: 30 delay: 30 until: _launch_result.rc == 0