From d7a7142a8170cb719506c8e10ec1ee037c6d75f6 Mon Sep 17 00:00:00 2001 From: Markus Frei Date: Mon, 16 Nov 2026 09:24:34 +0100 Subject: [PATCH] feat(plugin:dashboard): redraw the screen at least once a second The screen was only redrawn on events, so the elapsed time and the ETA stood still during a long task. A daemon thread now redraws it when no event did so for a second. The screen is written with os.write() straight to the file descriptor instead of through sys.stdout: Ansible forks its workers while the thread may be drawing, Modules/_io/bufferedio.c does not reset the buffer lock after a fork, and multiprocessing flushes the std streams at the end of each worker, which would hang on a lock held at fork time. Stress-tested with a 10 ms interval, 30 hosts, 40 tasks and 10 forks on ansible-core 2.16 and 2.18. --- README.md | 2 +- plugins/callback/dashboard.py | 56 ++++++++++++++++--- tests/unit/plugins/callback/test_dashboard.py | 23 ++++++++ 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ffc5cceda..4447c4420 100644 --- a/README.md +++ b/README.md @@ -382,7 +382,7 @@ Enable it with `stdout_callback = linuxfabrik.lfops.compact` in the `ansible.cfg The goal of the `linuxfabrik.lfops.dashboard` stdout callback is a live view of a run, similar to how glances shows the load of a system: always visible, without scrolling, and adapting to the size of the terminal. -It shows the run on one fixed screen that is updated in place instead of scrolling: the current role and task, a progress bar of the play, counters of the results of the current task and of the whole run, with the ok results of the task followed by the hosts still at work on it and the number of hosts (`ok 20/5/35`), and the hosts, sorted by their most severe result (fatal, warning, changed, ignored, ok) and then by name. Each host comes with its counts of fatal (F), warning (W), changed (C) and ok (O) results and its last notable result (LAST WORST), marked with the letter of its kind. A rising O count shows that a host is still at work while LAST WORST stays the same. A host with only ok results shows its last ok result, hosts with only skipped results are counted, not listed: +It shows the run on one fixed screen that is updated in place instead of scrolling, at least once a second: the current role and task, a progress bar of the play, counters of the results of the current task and of the whole run, with the ok results of the task followed by the hosts still at work on it and the number of hosts (`ok 20/5/35`), and the hosts, sorted by their most severe result (fatal, warning, changed, ignored, ok) and then by name. Each host comes with its counts of fatal (F), warning (W), changed (C) and ok (O) results and its last notable result (LAST WORST), marked with the letter of its kind. A rising O count shows that a host is still at work while LAST WORST stays the same. A host with only ok results shows its last ok result, hosts with only skipped results are counted, not listed: ```text linuxfabrik.lfops.setup_basic 35 hosts 00:12:34 diff --git a/plugins/callback/dashboard.py b/plugins/callback/dashboard.py index 66836dcc5..2f7a48489 100644 --- a/plugins/callback/dashboard.py +++ b/plugins/callback/dashboard.py @@ -16,6 +16,7 @@ import re import subprocess # nosec B404 import sys +import threading import time from collections import Counter @@ -44,7 +45,7 @@ version_added: "10.0.0" short_description: Fixed full-screen view of a run, details at the end description: - - Shows the run on one fixed screen that is updated in place instead of scrolling. At the top the playbook, the number of hosts and the elapsed time, the current role and task, a progress bar of the play with the estimated remaining time, then counters of the results of the current task and of the whole run. The ok counter of the task is followed by the number of hosts still at work on the task and the number of hosts, for example C(ok 13/22/35). + - Shows the run on one fixed screen that is updated in place instead of scrolling, at least once a second. At the top the playbook, the number of hosts and the elapsed time, the current role and task, a progress bar of the play with the estimated remaining time, then counters of the results of the current task and of the whole run. The ok counter of the task is followed by the number of hosts still at work on the task and the number of hosts, for example C(ok 13/22/35). - Below the title, where LFOps comes from - its version, or for a git checkout the branch, the commit and the number of uncommitted files, and the source path, shortened to the initials of its parent directories, when the checkout is mounted into an execution environment - and the ansible-core version. An C(LFOPS_EE_VERSION) environment variable is shown as the EE version. - The progress bar counts the tasks of the play that are known when it starts, after C(--tags) and C(--skip-tags) are applied. A task included with C(include_tasks) or C(include_role) is one step, the bar stands still while the tasks it includes run. Tasks in C(rescue) sections and handlers are not counted. The bar counts tasks, not time, and starts over with every batch of C(serial). The estimated remaining time assumes that the steps left take as long on average as the steps so far. - Below, the hosts are listed, sorted by their most severe result (fatal, warning, changed, ignored, ok), then by name, with their counts of fatal (F), warning (W), changed (C) and ok (O) results and their last notable result (LAST WORST), or the last ok result for a host without one. Hosts with only skipped results are counted, not listed. @@ -256,6 +257,9 @@ class CallbackModule(Compact): # at most this many redraws per second, the last state is always drawn at the # end of a task _REDRAW_INTERVAL = 0.1 + # without events, the screen is redrawn after this many seconds, so the elapsed + # time and the ETA keep moving during a long task + _TICK_INTERVAL = 1.0 def __init__(self): super().__init__() @@ -291,6 +295,10 @@ def __init__(self): self._capture = [] self._details = [] self._details_section = None + # the screen is drawn by the main thread on events and by a ticker thread + # in between, see _tick() + self._screen_lock = threading.RLock() + self._ticker_stop = threading.Event() # -- screen ------------------------------------------------------------------ @@ -327,12 +335,29 @@ def _start_tui(self): self._tui = True # leave the screen and print the details also on Ctrl+C or a crash atexit.register(self._end_tui) + threading.Thread( + target=self._tick, name='lfops-dashboard-ticker', daemon=True + ).start() + + def _tick(self): + """Redraw the screen when no event did so for _TICK_INTERVAL seconds.""" + # checked four times as often, so a redraw is at most a quarter late + while not self._ticker_stop.wait(self._TICK_INTERVAL / 4): + if time.monotonic() - self._last_draw < self._TICK_INTERVAL: + continue + try: + self._draw(force=True) + except Exception: # the main thread changed the state, next tick + pass def _end_tui(self): - if not self._tui: - return - self._tui = False - self._tui_done = True + self._ticker_stop.set() + # wait for a draw of the ticker thread in progress + with self._screen_lock: + if not self._tui: + return + self._tui = False + self._tui_done = True sys.stdout.write(_SCREEN_OFF) sys.stdout.flush() if self._termios_saved is not None: @@ -405,8 +430,11 @@ def _counter_cells(counts, kinds): ] def _draw(self, force=False): - if not self._tui: - return + with self._screen_lock: + if self._tui: + self._render(force) + + def _render(self, force): now = time.monotonic() if not force and now - self._last_draw < self._REDRAW_INTERVAL: return @@ -583,8 +611,18 @@ def _draw(self, force=False): for row in range(len(screen) + 1, lines + 1): out.append(f'\033[{row};1H\033[K') out.append(f'\033[{lines};1H') - sys.stdout.write(''.join(out)) - sys.stdout.flush() + # Straight to the file descriptor, past the buffer of sys.stdout: Ansible + # forks its workers while the ticker thread may be drawing, and a fork + # while that thread holds the lock of the buffer leaves the worker hanging + # on it when it flushes its streams at exit. Verified against the CPython + # source: Modules/_io/bufferedio.c does not reset that lock after a fork, + # and multiprocessing/process.py calls _flush_std_streams() at the end of + # a worker. + if threading.current_thread() is threading.main_thread(): + sys.stdout.flush() + data = ''.join(out).encode('utf-8', 'replace') + while data: + data = data[os.write(sys.stdout.fileno(), data) :] # -- output of the run ------------------------------------------------------- diff --git a/tests/unit/plugins/callback/test_dashboard.py b/tests/unit/plugins/callback/test_dashboard.py index ce864fd71..afb8103b7 100644 --- a/tests/unit/plugins/callback/test_dashboard.py +++ b/tests/unit/plugins/callback/test_dashboard.py @@ -268,6 +268,29 @@ def test_eta_from_the_finished_steps(self): self.assertEqual(_module.eta(21, 21, 0, 200, 900), 0) +class TestTicker(unittest.TestCase): + def test_screen_redrawn_during_a_long_task(self): + # no event for 3 seconds, the elapsed time still moves on + playbook = _PLAYBOOK.replace( + " - name: 'Some changes'\n", + " - name: 'Sleep'\n" + " ansible.builtin.command: 'sleep 3'\n" + ' changed_when: false\n' + '\n' + " - name: 'Some changes'\n", + ) + returncode, raw = _run(columns=100, playbook=playbook) + self.assertEqual(returncode, 0, raw) + screen = raw[raw.index(_SCREEN_ON) : raw.rindex(_SCREEN_OFF)] + # every frame starts in the top left corner + frames = [f for f in screen.split('\x1b[1;1H') if 'Task: Sleep' in f] + elapsed = {re.search(r'hosts (\d\d:\d\d:\d\d)', f).group(1) for f in frames} + # drawn at the start of the task, and at least twice while it sleeps + self.assertGreaterEqual(len(elapsed), 3, elapsed) + # no warning about forking a multi-threaded process (Python 3.12+) + self.assertNotIn('multi-threaded', raw) + + class TestCounterRows(unittest.TestCase): def test_columns_aligned_and_apart_for_long_numbers(self): rows = _module.CallbackModule._counter_rows(