diff --git a/CHANGELOG.md b/CHANGELOG.md index 149b82736..4bf52e053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -251,6 +251,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +* **role:duplicity**: An exclude of a source with `divide: true` no longer fails the backups of all its other entries with exit status 14. +* **role:duplicity**: `duba` runs fewer backups at the same time when the temp dir is too small for all of them, instead of failing with exit status 35 on hosts with a small `/tmp`. * **role:lynis**: The daily audit no longer empties the AIDE report on RHEL 8, which made the aide-logfile monitoring plugin warn every morning until the next AIDE check. * **role:aide**: The check no longer reports renewed certificates in `/etc/pki/tls/certs` and `/etc/ssl/certs`, such as those of acme.sh every 60 days, while their permissions, the private keys and the trust bundles stay fully checked. * **role:grav, playbook:setup_grav**: OPcache no longer fills up with recompiled copies of Grav's file list caches and no longer throws away its whole cache every few days. diff --git a/extensions/molecule/duplicity/inventory/group_vars/systems_under_test.yml b/extensions/molecule/duplicity/inventory/group_vars/systems_under_test.yml index f2261813e..a6f6b5b77 100644 --- a/extensions/molecule/duplicity/inventory/group_vars/systems_under_test.yml +++ b/extensions/molecule/duplicity/inventory/group_vars/systems_under_test.yml @@ -73,8 +73,12 @@ duplicity__backup_sources__group_var: state: 'absent' - path: '/srv' state: 'absent' + # duplicity aborts a backup with an exclude outside of its root, so the exclude of the entry + # "excluded" must not reach the backup of "alice" - path: '/srv/duba-molecule/divided' divide: true + excludes: + - '/srv/duba-molecule/divided/excluded' state: 'present' - path: '/srv/duba-molecule/plain' divide: false diff --git a/extensions/molecule/duplicity/verify.yml b/extensions/molecule/duplicity/verify.yml index 25c71da1a..9d14de90c 100644 --- a/extensions/molecule/duplicity/verify.yml +++ b/extensions/molecule/duplicity/verify.yml @@ -160,13 +160,14 @@ - '/var/tmp/molecule-restore-latest' - '/var/tmp/molecule-restore-old' - - name: 'mkdir /srv/duba-molecule/plain /srv/duba-molecule/divided/alice/files' + - name: 'mkdir /srv/duba-molecule/plain /srv/duba-molecule/divided/{alice/files,excluded}' ansible.builtin.file: path: '{{ item }}' state: 'directory' mode: 0o755 loop: - '/srv/duba-molecule/divided/alice/files' + - '/srv/duba-molecule/divided/excluded' - '/srv/duba-molecule/plain' - name: 'Create the test data, plain/file.txt in v1' @@ -341,6 +342,32 @@ - '"will abort with exit code 35" in __molecule__duba_restore_small_tmp_result["stderr"]' fail_msg: '{{ __molecule__duba_restore_small_tmp_result | to_json }}' + # The backups running at the same time share the temp dir, and each one needs 860 MB there + # (660 MB with duplicity 3.0.0, 460 MB before), else it aborts with exit code 35. 900 MB fit + # exactly one with every duplicity version, so duba runs the backups one after the other. + - name: 'mount -t tmpfs -o size=900M tmpfs /mnt/duba-molecule-tmp' + ansible.builtin.command: 'mount -t tmpfs -o size=900M tmpfs /mnt/duba-molecule-tmp' + changed_when: false + + - name: 'TMPDIR=/mnt/duba-molecule-tmp duba' + ansible.builtin.command: '/usr/local/bin/duba' + environment: + TMPDIR: '/mnt/duba-molecule-tmp' + changed_when: false + failed_when: false + register: '__molecule__duba_backup_small_tmp_result' + + - name: 'umount /mnt/duba-molecule-tmp' + ansible.builtin.command: 'umount /mnt/duba-molecule-tmp' + changed_when: false + + - name: 'Assert that duba throttled the backups to one at a time, and all of them succeeded' + ansible.builtin.assert: + that: + - '__molecule__duba_backup_small_tmp_result["rc"] == 0' + - '"Throttled to 1 instead of" in __molecule__duba_backup_small_tmp_result["stderr"]' + fail_msg: '{{ __molecule__duba_backup_small_tmp_result | to_json }}' + # a divided source is one backup per entry, duba refuses to restore it as a whole (212) - name: 'duba restore /srv/duba-molecule/divided' ansible.builtin.command: '/usr/local/bin/duba restore /srv/duba-molecule/divided' diff --git a/roles/duplicity/README.md b/roles/duplicity/README.md index 741419dee..7f6997a38 100644 --- a/roles/duplicity/README.md +++ b/roles/duplicity/README.md @@ -10,7 +10,7 @@ Note that this role does not support running with `--check`, as it first creates ## How the Role Behaves -* The role ships `duba`, a Linuxfabrik Python wrapper for duplicity. It runs a massively parallel backup to a Swift storage backend, using `min(processor count, 6) + 1` duplicity processes. Its configuration file is `/etc/duba/duba.json`. +* The role ships `duba`, a Linuxfabrik Python wrapper for duplicity. It runs a massively parallel backup to a Swift storage backend, using `min(processor count, 6) + 1` duplicity processes. Fewer backups run at the same time if the temp dir (`TMPDIR`, default `/tmp`) lacks the space duplicity requires to start each of them (860 MB, 660 MB with duplicity 3.0.0, 460 MB before), and `duba` logs that it throttled itself. Its configuration file is `/etc/duba/duba.json`. * To start a backup, call `duba` (or `duba --config=/etc/duba/duba.json --command=backup`). See `duba --help` for details. * The password of the backend (`duplicity__swift_login["password"]` or `duplicity__sftp_password`) is not in `/etc/duba/duba.json`, but stored as the systemd credential `duba-login.json`. With systemd 251 and newer (RHEL 9 and newer, Debian 12 and newer, Ubuntu 24.04 and newer), it is encrypted in `/etc/credstore.encrypted/` with the host key `/var/lib/systemd/credential.secret`, which the default backup does not include, and `duba.service` decrypts it to `/run/credentials/duba.service/` while it runs. On RHEL 8 and Ubuntu 22.04, it stays in plain text in `/etc/credstore/`, readable by root only. Started by hand, `duba` decrypts the credential itself, so it has to run as root. The password does not show up in the process list. * On a host with a TPM2 chip, the encrypted password is also bound to the Secure Boot state. After a firmware or Secure Boot update, `duba` fails until the role runs again. @@ -186,7 +186,7 @@ duplicity__swift_login: * `excludes`: - * Optional. List of patterns that should not be included in the backup for this `path`. + * Optional. List of patterns that should not be included in the backup for this `path`. With `divide: true`, each first-level entry gets only the absolute patterns that can match below it, so `/data/archive` excludes the entry `archive` without breaking the backups of the other entries. * Type: List of strings. * Default: `[]` diff --git a/roles/duplicity/templates/usr/local/bin/duba.j2 b/roles/duplicity/templates/usr/local/bin/duba.j2 index fcc698627..ffc03b677 100644 --- a/roles/duplicity/templates/usr/local/bin/duba.j2 +++ b/roles/duplicity/templates/usr/local/bin/duba.j2 @@ -9,6 +9,7 @@ # License: The Unlicense, see https://unlicense.org/. import argparse # pylint: disable=C0413 +import fnmatch # pylint: disable=C0413 import json # pylint: disable=C0413 import logging # pylint: disable=C0413 import os # pylint: disable=C0413 @@ -20,7 +21,7 @@ import time # pylint: disable=C0413 from traceback import format_exc # pylint: disable=C0413 __author__ = 'Linuxfabrik GmbH, Zurich/Switzerland' -__version__ = '2026102905' +__version__ = '2026111602' DESCRIPTION = 'Do a massive parallel backup to a Swift storage backend with duplicity.' @@ -51,11 +52,38 @@ COMMANDS = ['backup', 'exec', 'restore'] # uses for its check of the temp dir, see warn_on_low_space(). VOLSIZE_MB = 200 +# The number of volumes a duplicity 3.x backup builds and uploads at the same time. +BACKUP_CONCURRENCY = 2 + # The secret environment variables (SWIFT_PASSWORD, FTP_PASSWORD) are not in the config file, # but in this systemd credential, see load_credential(). CREDENTIAL_NAME = 'duba-login.json' +def exclude_fits_root(pattern, root): + """Tell whether the exclude `pattern` may match something below the backup `root`. + + duplicity aborts the whole backup with exit code 14 ("The file specification ... cannot + match any files in the base directory") if an exclude cannot match the backup root or + one of its parents. With `divide: true`, every first-level entry is a backup of its + own, so an exclude for one entry would break the backups of all the others. Compares + the leading path components, `*`, `?` and `[...]` match within one component, `**` + matches anything. Patterns that are not absolute (`**/cache`, `ignorecase:...`) are + left to duplicity. Verified with duplicity 3.0.7 on Fedora 43 (glob_get_sf() and + literal_get_sf() in duplicity/selection.py). + """ + if not pattern.startswith('/'): + return True + pattern_parts = pattern.strip('/').split('/') + root_parts = root.strip('/').split('/') + for pattern_part, root_part in zip(pattern_parts, root_parts): + if '**' in pattern_part: + return True + if not fnmatch.fnmatchcase(root_part, pattern_part): + return False + return True + + def parse_args(argv): """Parse command line arguments using argparse. """ @@ -265,7 +293,7 @@ class Duba: cmd += ['backup'] if self.duplicity_version >= (3, 0, 0): - cmd += ['--concurrency', '2'] + cmd += ['--concurrency', f'{BACKUP_CONCURRENCY}'] else: cmd += ['--asynchronous-upload'] cmd += self.archive_dir_options() @@ -274,12 +302,13 @@ class Duba: cmd += ['--encrypt-key', f"{self.config.get('gpg_encrypt_local_key', '')}"] cmd += ['--encrypt-key', f"{self.config.get('gpg_encrypt_master_key', '')}"] - # global excludes - for exclude in self.config.get('backup_excludes', []): - cmd += ['--exclude', f"{exclude}"] - - # excludes for this directory - for exclude in backup_options.get('excludes', []): + # global excludes, then the excludes for this directory + excludes = self.config.get('backup_excludes', []) + backup_options.get('excludes', []) + for exclude in excludes: + if not exclude_fits_root(exclude, backup_path): + # e.g. /data/archive in the backup of /data/alice, a divided /data + self.logger.debug(f'Skipping exclude {exclude}, it is outside {backup_path}') + continue cmd += ['--exclude', f"{exclude}"] cmd += ['--exclude-other-filesystems'] @@ -324,8 +353,8 @@ class Duba: def run_cmd_in_background(self, cmd, env, running_procs, max_procs, failed_retcs): # start duplicity in the background - # * start n processes: start up to (cpu-cores + 1) nice processes to backup as - # much as possible + # * start n processes: start up to max_procs nice processes (cpu-cores + 1, fewer for + # a backup if the temp dir lacks space) to backup as much as possible # * be as nice as possible: only when the machine is busy, the script is slower # * be asynchronous: uploading volumes while packaging new ones saves a lot of time # * use compression: saves money on external storages; bottleneck is always your uplink @@ -342,11 +371,11 @@ class Duba: self.logger.exception(f'Unknown error "{e}" while calling command "{cmd}"') sys.exit(207) - # don't start more than max cpu processes + # don't start more than max_procs processes while True: time.sleep(0.5) self.reap_finished_procs(running_procs, failed_retcs) - if len(running_procs) <= max_procs: + if len(running_procs) < max_procs: # allow launching of another process break @@ -384,6 +413,62 @@ class Duba: return filelist + def temp_dir(self, env): + # duplicity builds its volumes in the temp dir of its own environment, which + # Python's tempfile takes from TMPDIR, TEMP or TMP, else /tmp + for name in ('TMPDIR', 'TEMP', 'TMP'): + if env.get(name): + return env[name] + return '/tmp' + + + def free_mb(self, path): + stat = os.statvfs(path) + return stat.f_bavail * stat.f_frsize // 2**20 + + + def backup_procs_fitting_temp_dir(self, env, max_procs): + """Return how many backups may run at the same time, at most `max_procs`. + + On start, every duplicity backup checks the free space in its temp dir and aborts + with exit code 35 if it has less than (concurrency + 2) volumes plus 30% of one, + (concurrency + 1) before duplicity 3.0.1. --asynchronous-upload counts as a + concurrency of 1. The backups running at the same time share the temp dir, so a + backup that starts later finds less space than the first one. Verified against the + duplicity source, rel.0.8.23 to rel.3.2.3.dev2 (check_resources()). + """ + if self.duplicity_version >= (3, 0, 1): + volumes = BACKUP_CONCURRENCY + 2 + elif self.duplicity_version >= (3, 0, 0): + volumes = BACKUP_CONCURRENCY + 1 + else: + volumes = 1 + 1 + need_mb = int((volumes + 0.3) * VOLSIZE_MB) + tempdir = self.temp_dir(env) + try: + free_mb = self.free_mb(tempdir) + except OSError as e: + # duplicity reports this itself + self.logger.warning(f'Cannot get the free space of {tempdir}: {e}') + return max_procs + fitting = free_mb // need_mb + if fitting >= max_procs: + return max_procs + if fitting == 0: + self.logger.error( + f'{tempdir} has {free_mb} MB free, a duplicity backup needs {need_mb} MB' + f' there and aborts with exit code 35. Free up space, or set TMPDIR in' + f' the env of the config file to another dir.' + ) + return 1 + # the backups take longer, but succeed, so nothing to act on + self.logger.info( + f'Throttled to {fitting} instead of {max_procs} backups at the same time:' + f' {tempdir} has {free_mb} MB free, a duplicity backup needs {need_mb} MB there' + ) + return fitting + + def do_backup(self): """Do a massive parallel backup using duplicity and swift as backend. After that, delete all backups older than the given retention time. @@ -394,6 +479,7 @@ class Duba: # count the number of logical processors to limit number of processes (code taken from psutil) cpu_count = min(os.sysconf('SC_NPROCESSORS_ONLN'), 6) env = self.set_env(self.config.get('env', {})) + backup_procs = self.backup_procs_fitting_temp_dir(env, cpu_count + 1) backup_failed_retcs = [] cleanup_failed_retcs = [] @@ -417,7 +503,7 @@ class Duba: self.logger.info(f"Running: {' '.join(cmd)}") self.run_cmd_in_background( - cmd, env, running_procs, cpu_count, backup_failed_retcs + cmd, env, running_procs, backup_procs, backup_failed_retcs ) # wait here for all processes to finish @@ -443,7 +529,7 @@ class Duba: self.logger.info(f"Running: {' '.join(cmd)}") self.run_cmd_in_background( - cmd, env, running_procs, cpu_count, cleanup_failed_retcs + cmd, env, running_procs, cpu_count + 1, cleanup_failed_retcs ) @@ -507,7 +593,7 @@ class Duba: return source, os.path.join(source.get('path'), entry), rest - def warn_on_low_space(self, path, target): + def warn_on_low_space(self, path, target, env): # Only warnings: the restore goes into its own directory, so running out of space # does not harm the live data. The local copy, if there still is one, is the only size # estimate there is, since the duplicity metadata does not hold the file sizes. @@ -535,12 +621,11 @@ class Duba: # Verified against the duplicity source, rel.0.8.23 to rel.3.2.1 (check_resources()). volumes = 2 if self.duplicity_version >= (3, 0, 1) else 1 need_mb = int((volumes + 0.3) * VOLSIZE_MB) - tempdir = tempfile.gettempdir() - stat = os.statvfs(tempdir) - free = stat.f_bavail * stat.f_frsize - if free < need_mb * 2**20: + tempdir = self.temp_dir(env) + free_mb = self.free_mb(tempdir) + if free_mb < need_mb: self.logger.warning( - f'{tempdir} has {free // 2**20} MB free, duplicity needs {need_mb} MB there' + f'{tempdir} has {free_mb} MB free, duplicity needs {need_mb} MB there' f' and will abort with exit code 35 (set TMPDIR to use another dir)' ) @@ -588,9 +673,9 @@ class Duba: sys.exit(0) os.makedirs(os.path.dirname(restore_path), exist_ok=True) - self.warn_on_low_space(path, os.path.dirname(restore_path)) - self.logger.info(f"Running: {' '.join(cmd)}") env = self.set_env(self.config.get('env', {})) + self.warn_on_low_space(path, os.path.dirname(restore_path), env) + self.logger.info(f"Running: {' '.join(cmd)}") try: retc = subprocess.run(cmd, env=env).returncode except OSError as e: