From 4effc54f16ad9670658be02a5e34f098a9098c65 Mon Sep 17 00:00:00 2001 From: Markus Frei Date: Mon, 16 Nov 2026 07:47:18 +0100 Subject: [PATCH] perf(roles/aide, roles/librenms, roles/system_update): skip the check right after an AIDE re-baseline `aide --update` compares the whole file system like a check before it writes the new database, so the check that followed it scanned everything a second time only to find no differences, and on a small host it was the main cause of I/O pressure alerts after the nightly update. After a successful re-baseline the scripts and the role now only reset a failed state of aidecheck.service. After a finding or a failed update the check still runs. Verified with molecule (aide) on Rocky 8, 9, 10 and Debian 12, 13, and end to end with update-and-reboot on the same platforms and librenms-daily on Rocky 9: no second check, aide-logfile reports the accepted update as OK, a check afterwards is clean. --- CHANGELOG.md | 1 + .../molecule/aide/tasks/wait-for-aide.yml | 4 +- extensions/molecule/aide/verify.yml | 43 +++++++++++++++++-- roles/aide/README.md | 2 +- roles/aide/tasks/update-db.yml | 9 ++-- .../usr/local/sbin/aide-dpkg-hook.j2 | 16 +++++-- .../usr/local/sbin/librenms-daily.j2 | 14 ++++-- roles/system_update/README.md | 2 +- .../usr/local/sbin/security-update.j2 | 12 +++--- .../usr/local/sbin/update-and-reboot.j2 | 12 +++--- 10 files changed, 85 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 149b82736..50ecff9cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -167,6 +167,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +* **role:aide, role:librenms, role:system_update**: Re-baselining AIDE after an update no longer starts an extra check against the new database, which only repeated the scan the re-baseline had just done. This halves the AIDE I/O load after each nightly update, while the update is still scanned and accepted right away. * **role:collabora**: The service account `cool` has `/sbin/nologin` as its shell instead of `/bin/bash`, so no one logs in as `cool`, which also satisfies the Lynis check HOME-9304. Use `sudo -u cool ` to run a command as `cool`. * **role:postfix**: Postfix no longer answers the SMTP `VRFY` command, so a client cannot probe for valid recipient addresses, which satisfies the Lynis check MAIL-8820. Configurable with `postfix__disable_vrfy_command`. * **collection**: The collection declares ansible-core 2.16 as its minimum, as the README already did, so ansible-core 2.15 warns that it does not support the collection. diff --git a/extensions/molecule/aide/tasks/wait-for-aide.yml b/extensions/molecule/aide/tasks/wait-for-aide.yml index b154a4e6f..5c7ab07b1 100644 --- a/extensions/molecule/aide/tasks/wait-for-aide.yml +++ b/extensions/molecule/aide/tasks/wait-for-aide.yml @@ -1,6 +1,6 @@ # wait until neither a job is queued for nor a run is in progress of apt-daily-upgrade.service, -# aide-dpkg-update.service or aidecheck.service: the hook, the units and the update-db tasks of -# the role start the latter two with --no-block. units that do not exist report inactive. +# aide-dpkg-update.service or aidecheck.service: the hook and the units start the latter two +# with --no-block. units that do not exist report inactive. - name: 'systemctl list-jobs; systemctl is-active (wait until the aide units are idle)' # noqa command-instead-of-module (read-only state query in a retry loop) ansible.builtin.shell: | set -o pipefail diff --git a/extensions/molecule/aide/verify.yml b/extensions/molecule/aide/verify.yml index 7bb13f6aa..3ba61de84 100644 --- a/extensions/molecule/aide/verify.yml +++ b/extensions/molecule/aide/verify.yml @@ -692,7 +692,7 @@ name: 'linuxfabrik.lfops.aide' tasks_from: 'update-db-if-clean.yml' - - name: 'Wait for the check update-db-if-clean started (after cleaning up)' + - name: 'Wait for the aide units to be idle (after cleaning up)' ansible.builtin.include_tasks: 'tasks/wait-for-aide.yml' - name: 'systemctl show --property=Result aidecheck.service (after cleaning up)' # noqa command-instead-of-module (no module reads the Result of a finished unit) @@ -718,6 +718,11 @@ path: '/etc/aide.conf' line: '# lfops-molecule' + - name: 'systemctl show --property=ExecMainStartTimestampMonotonic aidecheck.service (before the run after a clean check)' # noqa command-instead-of-module (no module reads the start time of a unit) + ansible.builtin.command: 'systemctl show --property=ExecMainStartTimestampMonotonic --value aidecheck.service' + register: '__molecule__aide_start_before_clean_run_result' + changed_when: false + - name: 'Reset the end-of-play messages (run after a clean check)' ansible.builtin.set_fact: __shared__end_of_play_messages: [] @@ -732,18 +737,31 @@ checksum_algorithm: 'sha256' register: '__molecule__aide_db_after_clean_run_result' - - name: 'Wait for the check the handler started' + - name: 'Wait for the aide units to be idle (after the run after a clean check)' ansible.builtin.include_tasks: 'tasks/wait-for-aide.yml' + # `aide --update` compares like a check, so the update runs no second check against the + # database it just wrote + - name: 'systemctl show --property=ExecMainStartTimestampMonotonic aidecheck.service (after the run after a clean check)' # noqa command-instead-of-module (no module reads the start time of a unit) + ansible.builtin.command: 'systemctl show --property=ExecMainStartTimestampMonotonic --value aidecheck.service' + register: '__molecule__aide_start_after_clean_run_result' + changed_when: false + + - name: 'systemctl start --wait aidecheck.service (after the run after a clean check)' # noqa command-instead-of-module (wait for the check to finish and read its Result) + ansible.builtin.command: 'systemctl start --wait aidecheck.service' + changed_when: false + failed_when: false + - name: 'systemctl show --property=Result aidecheck.service (after the run after a clean check)' # noqa command-instead-of-module (no module reads the Result of a finished unit) ansible.builtin.command: 'systemctl show --property=Result --value aidecheck.service' register: '__molecule__aide_clean_run_result' changed_when: false - - name: 'Assert that the handler updated the database without telling the operator anything' + - name: 'Assert that the handler updated the database without a second check and without telling the operator anything' ansible.builtin.assert: that: - '__molecule__aide_db_after_clean_run_result["stat"]["checksum"] != __molecule__aide_db_before_clean_run_result["stat"]["checksum"]' + - '__molecule__aide_start_after_clean_run_result["stdout"] == __molecule__aide_start_before_clean_run_result["stdout"]' - '__molecule__aide_clean_run_result["stdout"] == "success"' - '__shared__end_of_play_messages | select("match", "aide: ") | list | length == 0' @@ -852,6 +870,11 @@ - name: 'Reset the logs of unattended-upgrades and the hook (clean host)' ansible.builtin.include_tasks: 'tasks/reset-uu-logs.yml' + - name: 'systemctl show --property=ExecMainStartTimestampMonotonic aidecheck.service (before the unattended upgrade on a clean host)' # noqa command-instead-of-module (no module reads the start time of a unit) + ansible.builtin.command: 'systemctl show --property=ExecMainStartTimestampMonotonic --value aidecheck.service' + register: '__molecule__aide_uu_clean_start_before_result' + changed_when: false + - name: 'systemctl start apt-daily-upgrade.service (clean host)' # noqa command-instead-of-module (runs the unattended upgrade and waits for it) ansible.builtin.command: 'systemctl start apt-daily-upgrade.service' changed_when: true @@ -859,6 +882,19 @@ - name: 'Wait for the units to finish (clean host)' ansible.builtin.include_tasks: 'tasks/wait-for-aide.yml' + # the hook checks directly, and `aide --update` compares like a check, so the upgrade + # starts no check through the unit + - name: 'systemctl show --property=ExecMainStartTimestampMonotonic aidecheck.service (after the unattended upgrade on a clean host)' # noqa command-instead-of-module (no module reads the start time of a unit) + ansible.builtin.command: 'systemctl show --property=ExecMainStartTimestampMonotonic --value aidecheck.service' + register: '__molecule__aide_uu_clean_start_after_result' + changed_when: false + + # confirms the re-baselined database + - name: 'systemctl start --wait aidecheck.service (after the unattended upgrade on a clean host)' # noqa command-instead-of-module (wait for the check to finish and read its Result) + ansible.builtin.command: 'systemctl start --wait aidecheck.service' + changed_when: false + failed_when: false + - name: 'Collect the results (clean host)' ansible.builtin.include_tasks: 'tasks/collect-uu-results.yml' @@ -870,6 +906,7 @@ - '__molecule__aide_uu_hook_result["stdout_lines"] | select("search", "aide --check") | list | length == 1' - '__molecule__aide_uu_hook_result["stdout_lines"] | select("search", "re-baselined") | list | length == 1' - '__molecule__aide_uu_update_runs_result["stdout"] | int == 1' + - '__molecule__aide_uu_clean_start_after_result["stdout"] == __molecule__aide_uu_clean_start_before_result["stdout"]' - '__molecule__aide_uu_check_result["stdout"] == "success"' fail_msg: '{{ __molecule__aide_uu_hook_result["stdout_lines"] + __molecule__aide_uu_log_result["stdout_lines"] }}' diff --git a/roles/aide/README.md b/roles/aide/README.md index f32ecf1ec..ffbff446e 100644 --- a/roles/aide/README.md +++ b/roles/aide/README.md @@ -24,7 +24,7 @@ This role is compatible with the following aide versions: * On the first run the role creates the database (`/var/lib/aide/aide.db.gz`), after it has deployed the config and the systemd units and after the pending handlers of the play have run. Depending on the size of the file system, `aide --init` can take several minutes. In a playbook that combines several roles, run aide last, as `setup_basic` does, so that the database contains what the other roles deployed. * `aidecheck.service` runs `aide --check` at a low CPU and IO priority, by `aidecheck.timer` and after every boot, once the services have started. Any finding (added, removed or changed files) makes the check exit non-zero, which leaves `aidecheck.service` in the failed state. The [aide-logfile](https://github.com/Linuxfabrik/monitoring-plugins/tree/main/check-plugins/aide-logfile) monitoring plugin evaluates the report, and monitoring failed systemd units alerts as well. The full report is written to `/var/log/aide/aide.log`, overwritten by every check. From aide 0.17 on, a summary (whether and how many files changed) additionally goes to syslog (`journalctl --unit aidecheck.service`), from where rsyslog forwards it. aide 0.16 (RHEL 8) cannot limit the syslog report, so there the report only goes to the file. The role sends no mail. * `/etc/logrotate.d/aide` rotates the report weekly with `copy` instead of the `copytruncate` the RHEL package ships, which would leave `/var/log/aide/aide.log` empty until the next check, as if the check had aborted. -* A check that is due while the [system_update](https://github.com/Linuxfabrik/lfops/tree/main/roles/system_update) role or `unattended-upgrades` installs packages waits until the update has finished. Conversely, `system_update` and the dpkg hook wait for a running check before they update the database. The role waits for it for 5 minutes at most and then aborts the run (see "Troubleshooting"). After every database update the role starts a check, so that `/var/log/aide/aide.log` shows the state of the host rather than the changes the update accepted. +* A check that is due while the [system_update](https://github.com/Linuxfabrik/lfops/tree/main/roles/system_update) role or `unattended-upgrades` installs packages waits until the update has finished. Conversely, `system_update` and the dpkg hook wait for a running check before they update the database. The role waits for it for 5 minutes at most and then aborts the run (see "Troubleshooting"). A database update runs no check against the new database: `aide --update` compares the whole file system like a check and writes its report to `/var/log/aide/aide.log`, which the [aide-logfile](https://github.com/Linuxfabrik/monitoring-plugins/tree/main/check-plugins/aide-logfile) monitoring plugin reports as accepted. The next regular check confirms the new database. * When the role changes `/etc/aide.conf`, its own units or whether `aidecheck.service` or `aidecheck.timer` is enabled, it updates the database afterwards, so that the next check does not report every path the new rules add or drop. It only does so if the last check had not reported changes: re-baselining a failing check would silently accept whatever changed on the host, an intrusion included. The same applies to a database the role finds on a host where it has not deployed `aidecheck.service` yet, for example one left by a previous AIDE setup: no check of this role has vouched for it. In both cases the database is left alone and the run tells you so. Review `/var/log/aide/aide.log` or the output of `aide --check`, and accept the current state with `--tags aide:update_db_force`. * The re-baseline after a change of the role accepts everything that changed since the last check, not only the change the role made. The shorter the check interval, the smaller that window. * Changes to monitored files made by anyone else, other LFOps roles and packages installed by hand included, are reported by the next check. The [system_update](https://github.com/Linuxfabrik/lfops/tree/main/roles/system_update) role knows about `aidecheck.service`: it runs a check before it updates packages and updates the database afterwards if that check was clean. The [librenms](https://github.com/Linuxfabrik/lfops/tree/main/roles/librenms) role does the same around the nightly self-update of LibreNMS. diff --git a/roles/aide/tasks/update-db.yml b/roles/aide/tasks/update-db.yml index ec5ed33b8..40f734d31 100644 --- a/roles/aide/tasks/update-db.yml +++ b/roles/aide/tasks/update-db.yml @@ -33,12 +33,9 @@ changed_when: true # the database now describes the host as it is, so the finding of the last check no longer -# applies, and the report `aide --update` wrote to /var/log/aide/aide.log is replaced by a check -# against the new database. --no-block: the run does not wait for the check. +# applies. no check against the new database: `aide --update` already compared the whole file +# system and wrote that report to /var/log/aide/aide.log, and a check right after it scans +# everything again only to find no differences. aide-logfile reports an accepted update as OK. - name: 'systemctl reset-failed aidecheck.service' # noqa command-instead-of-module (no module resets a failed unit) ansible.builtin.command: 'systemctl reset-failed aidecheck.service' changed_when: true - -- name: 'systemctl start --no-block aidecheck.service' # noqa command-instead-of-module (the service module cannot start without waiting) - ansible.builtin.command: 'systemctl start --no-block aidecheck.service' - changed_when: true diff --git a/roles/aide/templates/usr/local/sbin/aide-dpkg-hook.j2 b/roles/aide/templates/usr/local/sbin/aide-dpkg-hook.j2 index 1db967473..2d2400952 100644 --- a/roles/aide/templates/usr/local/sbin/aide-dpkg-hook.j2 +++ b/roles/aide/templates/usr/local/sbin/aide-dpkg-hook.j2 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # {{ ansible_managed }} -# 2026110201 +# 2026111601 # Keeps the AIDE database in step with the packages unattended-upgrades installs. Called by apt # before and after every dpkg run (DPkg::Pre-Invoke / DPkg::Post-Invoke in @@ -13,7 +13,7 @@ # * post: after a dpkg run of that process, queue aide-dpkg-update.service if the check was # clean, otherwise a check. Both are ordered after apt-daily-upgrade.service and therefore # start once unattended-upgrades has finished. -# * update: update the database and start a check against it. +# * update: update the database if the check of `pre` was clean, otherwise start a check. # # unattended-upgrades installs in package sets and runs dpkg once per set (MinimalSteps, on by # default, upgrade_in_minimal_steps() in unattended-upgrade), so apt calls pre and post once per @@ -118,10 +118,18 @@ case "$1" in else mv /var/lib/aide/aide.db.new.gz /var/lib/aide/aide.db.gz log 're-baselined the AIDE database after an unattended upgrade' + # no check against the new database: `aide --update` already compared the + # whole file system and wrote that report, and a check right after it scans + # everything again only to find no differences. aide-logfile reports an + # accepted update as OK. reset-failed, since the clean check of `pre` ran + # outside aidecheck.service, whose last run may still have failed on drift + # that has been reverted since. + systemctl reset-failed "${UNIT}.service" 2> /dev/null + exit 0 fi fi - # `aide --update` wrote its own report. a check against the new database replaces it, so - # that the report and the state of the unit reflect the host again. + # a check through the unit after a failed update, or when the unit was started by hand, + # so that the report and the state of the unit reflect the host again systemctl restart --no-block "${UNIT}.service" ;; esac diff --git a/roles/librenms/templates/usr/local/sbin/librenms-daily.j2 b/roles/librenms/templates/usr/local/sbin/librenms-daily.j2 index c8eaa3a69..3575f01e7 100644 --- a/roles/librenms/templates/usr/local/sbin/librenms-daily.j2 +++ b/roles/librenms/templates/usr/local/sbin/librenms-daily.j2 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # {{ ansible_managed }} -# 2026110901 +# 2026111601 # Runs the daily maintenance script of LibreNMS, which also updates its code (git, composer and # pip below /opt/librenms), and keeps the AIDE database in step with that update, the way @@ -67,10 +67,18 @@ else log "aide --update exited with ${update_rc}; the database was not re-baselined" else mv "$AIDE_DB_NEW" "$AIDE_DB" + # no check against the new database: `aide --update` already compared the whole file + # system and wrote that report, and a check right after it scans everything again + # only to find no differences. aide-logfile reports an accepted update as OK. + # reset-failed, since the clean check above ran outside aidecheck.service, whose last + # run may still have failed on drift that has been reverted since. + systemctl reset-failed "${AIDE_UNIT}.service" 2> /dev/null + exit 0 fi fi -# a check against the current database, so that the report and the state of the unit reflect -# the host again. --no-block, since it waits for the lock this script holds until it exits. +# a check through aidecheck.service after a finding or a failed update, so that the state of +# the unit reflects the host again. --no-block, since it waits for the lock this script holds +# until it exits. systemctl restart --no-block "${AIDE_UNIT}.service" exit 0 diff --git a/roles/system_update/README.md b/roles/system_update/README.md index 1a1dff197..ce6869a36 100644 --- a/roles/system_update/README.md +++ b/roles/system_update/README.md @@ -24,7 +24,7 @@ Reboots are not performed by the update scripts themselves. They are delegated t * **An interrupted dpkg is repaired on Debian and Ubuntu.** If an earlier dpkg run was cut short, for example by a crash or a reboot from outside the window, the regular lane runs `dpkg --configure -a` before the update, keeping the existing configuration files, instead of failing on every run until someone repairs the host by hand. * **The regular lane waits for the package manager.** If another process holds the apt or dpkg lock, for example an Ansible run or an admin in `apt`, the regular lane and its update check (`notify-and-schedule`) wait up to 10 minutes for it on Debian and Ubuntu instead of failing. If the update check still cannot refresh the package lists, it sends a "System update check failed" mail instead of announcing updates from stale lists. * **A failed update stops the run.** A metadata refresh, a removal of packages or an upgrade that exits non-zero sends a "System update failed" mail with the error output and ends the run, on both families. Nothing downstream happens: no AIDE re-baseline, no reboot request, and no success mail for a host that is left half-configured. -* **AIDE is re-baselined only when a check right before the update was clean.** If the host runs the AIDE check of the [aide](https://github.com/Linuxfabrik/lfops/tree/main/roles/aide) role (`aidecheck.timer` is active), the update runs an AIDE check before it installs anything, and if the update changed packages and that check was clean, it refreshes the AIDE database afterwards and starts the check again, so the next check does not flag every file the update touched. This applies to the regular and the security lane alike. If the check reports changes, the database is left untouched and the report is kept at `/var/log/aide/aide.log-pre-system-update` (regular lane) or `/var/log/aide/aide.log-pre-security-update` (security lane): drift that predates the update, an intrusion included, is never accepted as the new baseline. If the check cannot run at all, for example without an AIDE database, the database is left untouched as well, and a separate "AIDE check failed" mail quotes the error. +* **AIDE is re-baselined only when a check right before the update was clean.** If the host runs the AIDE check of the [aide](https://github.com/Linuxfabrik/lfops/tree/main/roles/aide) role (`aidecheck.timer` is active), the update runs an AIDE check before it installs anything, and if the update changed packages and that check was clean, it refreshes the AIDE database afterwards, so the next check does not flag every file the update touched. This applies to the regular and the security lane alike. If the check reports changes, the database is left untouched and the report is kept at `/var/log/aide/aide.log-pre-system-update` (regular lane) or `/var/log/aide/aide.log-pre-security-update` (security lane): drift that predates the update, an intrusion included, is never accepted as the new baseline. If the check cannot run at all, for example without an AIDE database, the database is left untouched as well, and a separate "AIDE check failed" mail quotes the error. ## Dependent Roles diff --git a/roles/system_update/templates/usr/local/sbin/security-update.j2 b/roles/system_update/templates/usr/local/sbin/security-update.j2 index dfd2b8a85..9d5b6b807 100644 --- a/roles/system_update/templates/usr/local/sbin/security-update.j2 +++ b/roles/system_update/templates/usr/local/sbin/security-update.j2 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # {{ ansible_managed }} -# 2026110501 +# 2026111601 # Apply Rocky Linux security hot-fixes from the dedicated `security` repository, # together with the dependencies they need from the other enabled repositories, @@ -214,10 +214,12 @@ if [ "$AIDE_WAS_CLEAN" -eq 1 ] && [ "$UPDATED" -eq 1 ]; then # /run/aide.lock serialises all aide runs of LFOps (aidecheck.service of the aide role) nice --adjustment=19 ionice --class idle flock /run/aide.lock aide --config="$AIDE_CONFIG" --update 1> /dev/null \mv "$AIDE_DB_NEW" "$AIDE_DB" - # --no-block: aidecheck.service is ordered after this script's service (After=), so a - # blocking restart would wait for this script to end, which waits for the restart. The - # check starts once this script has finished. - systemctl restart --no-block "${AIDE_UNIT}.service" + # no check against the new database: `aide --update` already compared the whole file + # system and wrote that report, and a check right after it scans everything again only to + # find no differences. aide-logfile reports an accepted update as OK. reset-failed, since + # the clean check above ran outside aidecheck.service, whose last run may still have + # failed on drift that has been reverted since. + systemctl reset-failed "${AIDE_UNIT}.service" 2> /dev/null fi # request a reboot if a core component (kernel, glibc, ...) needs it, or if running diff --git a/roles/system_update/templates/usr/local/sbin/update-and-reboot.j2 b/roles/system_update/templates/usr/local/sbin/update-and-reboot.j2 index ba5279a36..035e27859 100644 --- a/roles/system_update/templates/usr/local/sbin/update-and-reboot.j2 +++ b/roles/system_update/templates/usr/local/sbin/update-and-reboot.j2 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # {{ ansible_managed }} -# 2026110201 +# 2026111601 # Apply all available updates, then request a reboot if one is needed by dropping a # file into the /run/schedule-reboot spool. This script never reboots itself; the @@ -388,10 +388,12 @@ if [ "$AIDE_WAS_CLEAN" -eq 1 ] && [ "$UPDATED" -eq 1 ]; then # /run/aide.lock serialises all aide runs of LFOps (aidecheck.service of the aide role) nice --adjustment=19 ionice --class idle flock /run/aide.lock aide --config="$AIDE_CONFIG" --update 1> /dev/null \mv "$AIDE_DB_NEW" "$AIDE_DB" - # --no-block: aidecheck.service is ordered after this script's service (After=), so a - # blocking restart would wait for this script to end, which waits for the restart. The - # check starts once this script has finished. - systemctl restart --no-block "${AIDE_UNIT}.service" + # no check against the new database: `aide --update` already compared the whole file + # system and wrote that report, and a check right after it scans everything again only to + # find no differences. aide-logfile reports an accepted update as OK. reset-failed, since + # the clean check above ran outside aidecheck.service, whose last run may still have + # failed on drift that has been reverted since. + systemctl reset-failed "${AIDE_UNIT}.service" 2> /dev/null fi # after any update, notify about new config-file stubs (*.rpmnew / *.rpmsave on RHEL,