From 4493d4992f8888f9a7b6a069bb60de25055ac9de Mon Sep 17 00:00:00 2001 From: Martin Pihrt Date: Tue, 13 Oct 2026 09:06:40 +0200 Subject: [PATCH 1/3] Preserve database queue timestamps and cap storage --- CHANGELOG.md | 5 + README.md | 2 +- plugins/database_connector/README.md | 16 +- plugins/database_connector/__init__.py | 219 ++++++++++++++++-- plugins/database_connector/plugin.json | 4 +- .../templates/database_connector.html | 30 ++- tests/test_database_connector_queue.py | 98 +++++++- 7 files changed, 337 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb60641..6dd7a6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # OSPy-plugins Changelog +October 13 2026 +---------------- +(Martin Pihrt) - Database Connector v1.1.1
+The durable database queue now restores the original command timestamp for MySQL/MariaDB `CURRENT_TIMESTAMP` and timestamp-default writes, so all plug-ins using Database Connector keep measurement time after an outage. Added configurable 80 MiB warning and 100 MiB maximum queue sizes, one-time SSL e-mail alerts with an editable subject, SD-card protection by rejecting new commands past the maximum, queue size diagnostics and regression coverage. + October 11 2026 --------------- (Martin Pihrt) - Repository documentation
diff --git a/README.md b/README.md index f7f0f28..8236bf6 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ This table is generated from the `name`, `version` and `description` fields in e | [**CHMI**](plugins/chmi/) | `1.0.7` | Downloads CHMI or SHMU radar data, detects local rain and optionally controls a hardware map. | | [**CLI Control**](plugins/cli_control/) | `1.0.0` | Runs administrator-configured shell commands when OSPy station states change. | | [**Current Loop Tanks Monitor**](plugins/current_loop_tanks_monitor/) | `1.1.1` | Monitors up to four water tanks through an ADS1115 and 4-20 mA pressure sensors. | -| [**Database Connector**](plugins/database_connector/) | `1.1.0` | Provides a shared MySQL connection and backup interface for OSPy plug-ins. | +| [**Database Connector**](plugins/database_connector/) | `1.1.1` | Provides shared MySQL access, backup and durable queued writes for OSPy plug-ins. | | [**Direct 16 Relay Outputs**](plugins/relay_16/) | `1.0.0` | Controls up to 16 direct Raspberry Pi GPIO relay outputs with configurable trigger polarity. | | [**Door Opening**](plugins/door_opening/) | `1.0.0` | Activates a selected OSPy station for a bounded door or gate opening interval. | | [**E-mail Notifications**](plugins/email_notifications/) | `1.0.1` | Sends OSPy event notifications through a STARTTLS SMTP server with an optional retry queue. | diff --git a/plugins/database_connector/README.md b/plugins/database_connector/README.md index 074fbe8..76210b0 100644 --- a/plugins/database_connector/README.md +++ b/plugins/database_connector/README.md @@ -5,13 +5,9 @@ Tested in Python 3+ This extension uses a database connection via the interface: mysql-connector-python to install use: sudo pip install mysql-connector-python. https://dev.mysql.com/doc/connector-python/en/connector-python-installation.html -The plug-in includes an OSPy `plugin.json` manifest and implements the common -lifecycle and `health()` interfaces. Diagnostics reports whether the connector -is enabled and available, its version, configured server and database, and the -result of the latest real database operation. Database connections and cursors -are closed after every operation. +The plug-in includes an OSPy `plugin.json` manifest and implements the common lifecycle and `health()` interfaces. Diagnostics reports whether the connector is enabled and available, its version, configured server and database, and the result of the latest real database operation. Database connections and cursors are closed after every operation. -October 10, 2026 GitHub/OSPy-plugins/plugins/database_connector — Added an optional durable SQLite-backed database command queue. When Use buffer is enabled, SQL commands from other plug-ins are saved locally in order and replayed in the background; commands remain queued during database outages and survive OSPy restarts. The settings page shows the queued command count and provides a CSRF-protected Clear queue action. Disabling the buffer stops queue replay and new commands use the direct database path. +October 13, 2026 GitHub/OSPy-plugins/plugins/database_connector — The durable SQLite-backed queue now preserves the source-command timestamp when it replays a statement. For MySQL/MariaDB statements that use `CURRENT_TIMESTAMP` or a `TIMESTAMP DEFAULT CURRENT_TIMESTAMP` column, the restored record therefore retains the original logging time instead of the later recovery time. The queue has an adjustable warning threshold (80 MiB by default), a hard maximum (100 MiB by default), an optional alert through E-mail Notifications SSL and an editable alert subject. Plugin setup ----------- @@ -20,7 +16,13 @@ Plugin setup If checked enabled plugin is enabled. When the box is checked, the extension will be active. After filling in all the fields, ospy must be restarted! * Use buffer: - When enabled, database commands are saved in a local queue and processed in order. If the database connection is unavailable, commands remain queued and are saved after the connection is restored. This helps prevent data loss from logging in other plugins. The queue is stored in SQLite in the plug-in data directory and survives OSPy restarts. The page displays the number of queued commands and lets the administrator clear the queue manually. Clearing permanently discards every queued command. Disabling the buffer stops background replay and sends new commands through the direct database path. + When enabled, database commands are saved in a local SQLite queue and processed in order. If the database connection is unavailable, commands remain queued and are saved after the connection is restored. The queue records the time that the original database command was submitted; during replay the connector sets the MySQL/MariaDB session timestamp to that saved value, so `CURRENT_TIMESTAMP` and timestamp defaults keep the original logging time. The queue survives OSPy restarts. The page displays its command count and used size, and lets the administrator clear it manually. Clearing permanently discards every queued command. Disabling the buffer stops background replay and sends new commands through the direct database path. + +* Queue warning size and maximum size: + The warning threshold defaults to 80 MiB and the maximum defaults to 100 MiB. Both values are adjustable, and the warning must be smaller than the maximum. The connector sends one alert after crossing the warning threshold and re-arms it after the queue shrinks below the threshold. It does not evict older records: when a new command would exceed the maximum, it is rejected and a separate one-time limit alert is sent. This protects the SD card while retaining the oldest pending records in FIFO order. + +* Send queue alert e-mail through SSL and Queue alert e-mail subject: + Enable this only when the E-mail Notifications SSL plug-in is configured. Queue alerts use that plug-in's SMTP-over-SSL settings and recipients; this plug-in stores no SMTP password. The configured subject is used as the prefix for both the warning and maximum-size messages. * Host: IP address to the database server. diff --git a/plugins/database_connector/__init__.py b/plugins/database_connector/__init__.py index 9af8239..e6f9bf7 100644 --- a/plugins/database_connector/__init__.py +++ b/plugins/database_connector/__init__.py @@ -7,6 +7,7 @@ import json import traceback import time +import html import subprocess import os import mimetypes @@ -32,6 +33,10 @@ DB_COMMAND_LOG_THROTTLE = 60 DB_QUEUE_RETRY_INTERVAL = 10 QUEUE_FILENAME = 'database_queue.sqlite3' +MEBIBYTE = 1024 * 1024 +DEFAULT_QUEUE_WARNING_MIB = 80 +DEFAULT_QUEUE_MAX_MIB = 100 +QUEUE_ENTRY_OVERHEAD_BYTES = 8192 plugin_options = PluginOptions( NAME, @@ -46,6 +51,11 @@ 'sql_name': [], # a list of all sql names in the plugin data directory 'sql_size': [], # sql size in bytes 'queue_size': 0, + 'queue_bytes': 0, + 'queue_warning_mib': DEFAULT_QUEUE_WARNING_MIB, + 'queue_max_mib': DEFAULT_QUEUE_MAX_MIB, + 'queue_email_alerts': False, + 'queue_email_subject': _('OSPy Database Connector queue alert'), } ) @@ -140,6 +150,10 @@ def health(): if state['last_success'] else _('Not available') ), } + if plugin_options.get('use_buffer', False): + queued_commands, queued_bytes = db_queue_status() + details[_('Queued database commands')] = '{} ({:.2f} MiB)'.format( + queued_commands, queued_bytes / float(MEBIBYTE)) if state['last_error_message']: details[_('Last error')] = state['last_error_message'] if not plugin_options['use']: @@ -239,38 +253,177 @@ def queue_path(): return os.path.join(plugin_data_dir(), QUEUE_FILENAME) +def _queue_limit_mib(name, default): + return _positive_mib(plugin_options.get(name, default), default) + + +def _positive_mib(value, default): + try: + return max(1, min(1024, int(value))) + except (TypeError, ValueError): + return default + + +def queue_warning_bytes(): + return _queue_limit_mib('queue_warning_mib', DEFAULT_QUEUE_WARNING_MIB) * MEBIBYTE + + +def queue_max_bytes(): + warning_mib = _queue_limit_mib('queue_warning_mib', DEFAULT_QUEUE_WARNING_MIB) + maximum_mib = _queue_limit_mib('queue_max_mib', DEFAULT_QUEUE_MAX_MIB) + return max(warning_mib + 1, maximum_mib) * MEBIBYTE + + +def queue_limits_from_input(values): + warning_mib = _positive_mib( + values.get('queue_warning_mib', plugin_options.get('queue_warning_mib', DEFAULT_QUEUE_WARNING_MIB)), + DEFAULT_QUEUE_WARNING_MIB, + ) + maximum_mib = _positive_mib( + values.get('queue_max_mib', plugin_options.get('queue_max_mib', DEFAULT_QUEUE_MAX_MIB)), + DEFAULT_QUEUE_MAX_MIB, + ) + if warning_mib >= maximum_mib: + raise ValueError(_('The queue warning size must be lower than the maximum size.')) + return warning_mib, maximum_mib + + +def _queue_used_bytes(connection): + page_size = connection.execute('PRAGMA page_size').fetchone()[0] + page_count = connection.execute('PRAGMA page_count').fetchone()[0] + free_pages = connection.execute('PRAGMA freelist_count').fetchone()[0] + return max(0, page_count - free_pages) * page_size + + +def _queue_metadata(connection, key): + row = connection.execute( + 'SELECT value FROM queue_metadata WHERE key = ?', (key,) + ).fetchone() + return row[0] if row else '' + + +def _set_queue_metadata(connection, key, value): + connection.execute( + 'INSERT OR REPLACE INTO queue_metadata (key, value) VALUES (?, ?)', + (key, str(value)), + ) + + +def _queue_alert_due(connection, key, active): + if not active: + connection.execute('DELETE FROM queue_metadata WHERE key = ?', (key,)) + return False + if _queue_metadata(connection, key): + return False + _set_queue_metadata(connection, key, '1') + return True + + +def _refresh_queue_alert_state(connection, used_bytes): + _queue_alert_due(connection, 'warning_alert_sent', used_bytes >= queue_warning_bytes()) + _queue_alert_due(connection, 'limit_alert_sent', used_bytes >= queue_max_bytes()) + + def init_db_queue(): """Create the durable SQLite outbox used while the remote database is unavailable.""" path = queue_path() os.makedirs(os.path.dirname(path), exist_ok=True) - with queue_db_lock, sqlite3.connect(path, timeout=10) as connection: - connection.execute('CREATE TABLE IF NOT EXISTS pending_queries (id INTEGER PRIMARY KEY AUTOINCREMENT, sql TEXT NOT NULL, commit_after INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL)') + with queue_db_lock: + connection = sqlite3.connect(path, timeout=10) + try: + connection.execute('CREATE TABLE IF NOT EXISTS pending_queries (id INTEGER PRIMARY KEY AUTOINCREMENT, sql TEXT NOT NULL, commit_after INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL)') + connection.execute('CREATE TABLE IF NOT EXISTS queue_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)') + connection.commit() + finally: + connection.close() -def db_queue_size(): +def db_queue_status(): if not plugin_options['use_buffer']: - return 0 + return 0, 0 try: init_db_queue() - with queue_db_lock, sqlite3.connect(queue_path(), timeout=10) as connection: - return connection.execute('SELECT COUNT(*) FROM pending_queries').fetchone()[0] + with queue_db_lock: + connection = sqlite3.connect(queue_path(), timeout=10) + try: + count = connection.execute('SELECT COUNT(*) FROM pending_queries').fetchone()[0] + return count, _queue_used_bytes(connection) + finally: + connection.close() except Exception as error: log_db_error(_('Unable to read database queue') + ': {}'.format(error)) - return 0 + return 0, 0 + + +def db_queue_size(): + return db_queue_status()[0] def clear_db_queue(): init_db_queue() with queue_db_lock: - with sqlite3.connect(queue_path(), timeout=10) as connection: + connection = sqlite3.connect(queue_path(), timeout=10) + try: connection.execute('DELETE FROM pending_queries') + _refresh_queue_alert_state(connection, 0) + connection.commit() + finally: + connection.close() + + +def send_queue_email(kind, used_bytes): + """Send one alert through the configured SSL notification plug-in.""" + if not plugin_options.get('queue_email_alerts', False): + return False + try: + from plugins.email_notifications_ssl import try_mail + used_mib = used_bytes / float(MEBIBYTE) + maximum_mib = queue_max_bytes() / float(MEBIBYTE) + if kind == 'limit': + title = _('Database queue limit reached') + text = _('New database commands are being rejected until queued commands are delivered or cleared.') + else: + title = _('Database queue is large') + text = _('Database commands remain stored locally until the database becomes available.') + subject = '{} - {}'.format(plugin_options.get('queue_email_subject', NAME), title) + body_text = '{}: {:.1f} MiB / {:.0f} MiB. {}'.format(title, used_mib, maximum_mib, text) + body_html = '{}

{:.1f} MiB / {:.0f} MiB.

{}

'.format( + html.escape(title), used_mib, maximum_mib, html.escape(text)) + return bool(try_mail(body_html, body_text, attachment=None, subject=subject)) + except Exception: + log_db_error(_('Unable to send database queue alert') + ': ' + traceback.format_exc().splitlines()[-1]) + return False -def enqueue_db(sql, commit=False): +def enqueue_db(sql, commit=False, created_at=None): init_db_queue() - with queue_db_lock, sqlite3.connect(queue_path(), timeout=10) as connection: - connection.execute('INSERT INTO pending_queries (sql, commit_after, created_at) VALUES (?, ?, ?)', (str(sql), int(bool(commit)), time.time())) - return -1 + alert_kind = None + used_bytes = 0 + result = None + with queue_db_lock: + connection = sqlite3.connect(queue_path(), timeout=10) + try: + sql_text = str(sql) + used_bytes = _queue_used_bytes(connection) + required_bytes = len(sql_text.encode('utf-8')) + QUEUE_ENTRY_OVERHEAD_BYTES + if used_bytes + required_bytes > queue_max_bytes(): + if _queue_alert_due(connection, 'limit_alert_sent', True): + alert_kind = 'limit' + log_db_error(_('Database queue limit reached; command was not queued.')) + else: + queued_at = time.time() if created_at is None else float(created_at) + connection.execute('INSERT INTO pending_queries (sql, commit_after, created_at) VALUES (?, ?, ?)', (sql_text, int(bool(commit)), queued_at)) + used_bytes = _queue_used_bytes(connection) + if _queue_alert_due(connection, 'warning_alert_sent', used_bytes >= queue_warning_bytes()): + alert_kind = 'warning' + _queue_alert_due(connection, 'limit_alert_sent', used_bytes >= queue_max_bytes()) + result = -1 + connection.commit() + finally: + connection.close() + if alert_kind is not None: + send_queue_email(alert_kind, used_bytes) + return result def process_db_queue(_queue_failure=False): @@ -279,21 +432,29 @@ def process_db_queue(_queue_failure=False): try: init_db_queue() with queue_db_lock: - with sqlite3.connect(queue_path(), timeout=10) as connection: - row = connection.execute('SELECT id, sql, commit_after FROM pending_queries ORDER BY id LIMIT 1').fetchone() + connection = sqlite3.connect(queue_path(), timeout=10) + try: + row = connection.execute('SELECT id, sql, commit_after, created_at FROM pending_queries ORDER BY id LIMIT 1').fetchone() + finally: + connection.close() if row is None: if _queue_failure: return queue_stop_event.wait(1) continue - result = execute_db(row[1], commit=bool(row[2]), test=False, fetch=False, _from_queue=True, _queue_failure=True) + result = execute_db(row[1], commit=bool(row[2]), test=False, fetch=False, _from_queue=True, _queue_failure=True, _queued_at=row[3]) if result is None: if _queue_failure: log_db_error(_('Database queue replay failed; command remains queued.')) queue_stop_event.wait(DB_QUEUE_RETRY_INTERVAL) continue - with sqlite3.connect(queue_path(), timeout=10) as connection: + connection = sqlite3.connect(queue_path(), timeout=10) + try: connection.execute('DELETE FROM pending_queries WHERE id = ?', (row[0],)) + _refresh_queue_alert_state(connection, _queue_used_bytes(connection)) + connection.commit() + finally: + connection.close() if _queue_failure: return except Exception as error: @@ -314,8 +475,9 @@ def table_exists(table_name): return bool(rows) -def execute_db(sql = "", commit = False, test = False, fetch = False, _from_queue=False, _queue_failure=False): +def execute_db(sql = "", commit = False, test = False, fetch = False, _from_queue=False, _queue_failure=False, _queued_at=None): global is_installed_ok + submitted_at = time.time() if is_installed_ok: import mysql.connector from mysql.connector import errorcode @@ -330,6 +492,10 @@ def execute_db(sql = "", commit = False, test = False, fetch = False, _from_queu cur = cnx.cursor() if not test: cur.execute("USE {}".format(plugin_options['database'])) + if _from_queue and _queued_at is not None: + # MySQL/MariaDB applies this session value to CURRENT_TIMESTAMP + # and TIMESTAMP defaults used by the queued statement. + cur.execute('SET timestamp = {}'.format(int(float(_queued_at)))) rows = cur.execute(sql) if rows is not None: @@ -362,14 +528,14 @@ def execute_db(sql = "", commit = False, test = False, fetch = False, _from_queu else: log_db_error(_('Database connection/query failed') + ': ' + _('Not connected')) if plugin_options['use_buffer'] and not test and not fetch and not _from_queue: - return queue_failed_db_command(sql, commit) + return queue_failed_db_command(sql, commit, submitted_at) return None return -1 if msg is None else msg except mysql.connector.Error as err: if plugin_options['use_buffer'] and not test and not fetch and not _from_queue and is_database_connection_error(err, errorcode): - return queue_failed_db_command(sql, commit) + return queue_failed_db_command(sql, commit, submitted_at) if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: log_db_error(_('Something is wrong with your user name or password')) elif err.errno == errorcode.ER_BAD_DB_ERROR: @@ -395,7 +561,7 @@ def execute_db(sql = "", commit = False, test = False, fetch = False, _from_queu ConnectorError = () errorcode = None if ConnectorError and isinstance(err, ConnectorError) and is_database_connection_error(err, errorcode): - return queue_failed_db_command(sql, commit) + return queue_failed_db_command(sql, commit, submitted_at) log_db_error(_('Database connection/query failed') + ': {}'.format(err)) if _queue_failure: log_db_error(_('Database queue replay failed; command remains queued.')) @@ -418,10 +584,10 @@ def execute_db(sql = "", commit = False, test = False, fetch = False, _from_queu return None -def queue_failed_db_command(sql, commit): +def queue_failed_db_command(sql, commit, created_at=None): try: log_db_error(_('Database command failed; saving it to the queue for retry.')) - return enqueue_db(sql, commit=commit) + return enqueue_db(sql, commit=commit, created_at=created_at) except Exception as error: log_db_error(_('Unable to queue database command') + ': {}'.format(error)) return None @@ -522,7 +688,9 @@ def GET(self): clear_db_queue() log.info(NAME, _('Database queue cleared manually.')) - plugin_options.__setitem__('queue_size', db_queue_size()) + queue_size, queue_bytes = db_queue_status() + plugin_options.__setitem__('queue_size', queue_size) + plugin_options.__setitem__('queue_bytes', queue_bytes) return self.plugin_render.database_connector(plugin_options, log.events(NAME)) except: @@ -535,9 +703,14 @@ def POST(self): try: qdict = web.input() verify_csrf(qdict) + queue_warning_mib, queue_max_mib = queue_limits_from_input(qdict) plugin_options.web_update(qdict) plugin_options.__setitem__('use_buffer', 'use_buffer' in qdict) plugin_options.__setitem__('use', 'use' in qdict) + if 'queue_email_alerts_present' in qdict: + plugin_options.__setitem__('queue_email_alerts', 'queue_email_alerts' in qdict) + plugin_options.__setitem__('queue_warning_mib', queue_warning_mib) + plugin_options.__setitem__('queue_max_mib', queue_max_mib) if plugin_options['use_buffer'] and plugin_options['use'] and is_installed_ok: start_queue_worker() elif not plugin_options['use_buffer']: diff --git a/plugins/database_connector/plugin.json b/plugins/database_connector/plugin.json index c1240d5..d789c20 100644 --- a/plugins/database_connector/plugin.json +++ b/plugins/database_connector/plugin.json @@ -3,8 +3,8 @@ "id": "database_connector", "name": "Database Connector", "menu": "Package: Database Connector", - "version": "1.1.0", - "description": "Provides a shared MySQL connection and backup interface for OSPy plug-ins.", + "version": "1.1.1", + "description": "Provides shared MySQL access, backup and durable queued writes for OSPy plug-ins.", "author": "Martin Pihrt", "homepage": "https://github.com/martinpihrt/OSPy-plugins/tree/master/plugins/database_connector", "license": "GPL-3.0", diff --git a/plugins/database_connector/templates/database_connector.html b/plugins/database_connector/templates/database_connector.html index 8e4542f..4b43988 100644 --- a/plugins/database_connector/templates/database_connector.html +++ b/plugins/database_connector/templates/database_connector.html @@ -110,10 +110,38 @@ $_('Messages in queue'): - $plugin_options['queue_size'] + $plugin_options['queue_size'] ($_('used'): ${'%.2f' % (plugin_options['queue_bytes'] / float(1024 * 1024))} MiB) $_('Clear queue') + + $_('Queue warning size (MiB)'): + + + + + + $_('Queue maximum size (MiB)'): + + + + + + $_('Send queue alert e-mail through SSL'): + + + + + + + $_('Queue alert e-mail subject'): + + + + $_('Host'): diff --git a/tests/test_database_connector_queue.py b/tests/test_database_connector_queue.py index 13b2826..c91aa10 100644 --- a/tests/test_database_connector_queue.py +++ b/tests/test_database_connector_queue.py @@ -26,6 +26,7 @@ def web_update(self, values): self.update(values) plugins = types.ModuleType('plugins') + plugins.__path__ = [] plugins.PluginOptions = PluginOptions plugins.plugin_url = lambda page: str(page) plugins.plugin_data_dir = lambda: self.temp_dir.name @@ -81,6 +82,10 @@ def test_settings_template_reads_queue_count_from_plugin_options(self): template = (ROOT / 'plugins' / 'database_connector' / 'templates' / 'database_connector.html').read_text(encoding='utf-8') self.assertIn("$plugin_options['queue_size']", template) + self.assertIn("name='queue_warning_mib'", template) + self.assertIn("name='queue_max_mib'", template) + self.assertIn("name='queue_email_alerts'", template) + self.assertIn("name='queue_email_subject'", template) self.assertNotIn('$queue_size', template) def test_enqueue_preserves_command_order_and_commit_flag(self): @@ -132,8 +137,8 @@ def test_worker_replays_in_order_and_deletes_successful_commands(self): self.module.plugin_options['use_buffer'] = True calls = [] - def execute(sql, commit=False, test=False, fetch=False, _from_queue=False, _queue_failure=False): - calls.append((sql, commit, _from_queue)) + def execute(sql, commit=False, test=False, fetch=False, _from_queue=False, _queue_failure=False, _queued_at=None): + calls.append((sql, commit, _from_queue, _queued_at)) return -1 self.module.execute_db = execute @@ -141,9 +146,96 @@ def execute(sql, commit=False, test=False, fetch=False, _from_queue=False, _queu self.module.process_db_queue(_queue_failure=True) self.module.process_db_queue(_queue_failure=True) - self.assertEqual(calls, [('INSERT INTO sample VALUES (1)', True, True), ('INSERT INTO sample VALUES (2)', False, True)]) + self.assertEqual( + [call[:3] for call in calls], + [('INSERT INTO sample VALUES (1)', True, True), ('INSERT INTO sample VALUES (2)', False, True)], + ) + self.assertTrue(all(call[3] > 0 for call in calls)) self.assertEqual(self.module.db_queue_size(), 0) + def test_queue_rejects_a_command_larger_than_the_configured_limit(self): + self.module.plugin_options['queue_warning_mib'] = 1 + self.module.plugin_options['queue_max_mib'] = 2 + + result = self.module.enqueue_db('x' * (3 * 1024 * 1024), commit=True) + + self.assertIsNone(result) + self.assertEqual(self.module.db_queue_size(), 0) + + def test_queue_limit_form_values_keep_existing_values_when_not_rendered(self): + self.module.plugin_options['queue_warning_mib'] = 37 + self.module.plugin_options['queue_max_mib'] = 71 + + self.assertEqual(self.module.queue_limits_from_input({}), (37, 71)) + with self.assertRaisesRegex(ValueError, 'warning size'): + self.module.queue_limits_from_input({ + 'queue_warning_mib': 100, + 'queue_max_mib': 100, + }) + + def test_replay_sets_the_mysql_session_timestamp_to_the_queued_time(self): + self.module.is_installed_ok = True + executed = [] + + class Cursor: + def execute(self, sql): + executed.append(sql) + return None + + def close(self): + pass + + class Connection: + def is_connected(self): + return True + + def cursor(self): + return Cursor() + + def close(self): + pass + + def commit(self): + pass + + connector = types.SimpleNamespace( + connect=lambda **_kwargs: Connection(), + errorcode=types.SimpleNamespace(), + ) + mysql_module = types.ModuleType('mysql') + mysql_module.connector = connector + self._install_module('mysql', mysql_module) + self._install_module('mysql.connector', connector) + + result = self.module.execute_db( + 'INSERT INTO sample VALUES (1)', + commit=True, + _from_queue=True, + _queued_at=1760000000.9, + ) + + self.assertEqual(result, -1) + self.assertEqual(executed[0], 'USE ospy') + self.assertEqual(executed[1], 'SET timestamp = 1760000000') + self.assertEqual(executed[2], 'INSERT INTO sample VALUES (1)') + + def test_queue_alert_uses_the_configured_ssl_mail_subject(self): + sent = [] + email_plugin = types.ModuleType('plugins.email_notifications_ssl') + + def try_mail(body, text, attachment=None, subject=None): + sent.append((body, text, attachment, subject)) + return True + + email_plugin.try_mail = try_mail + self._install_module('plugins.email_notifications_ssl', email_plugin) + self.module.plugin_options['queue_email_alerts'] = True + self.module.plugin_options['queue_email_subject'] = 'Queue warning' + + self.assertTrue(self.module.send_queue_email('warning', 80 * 1024 * 1024)) + self.assertIn('Queue warning - Database queue is large', sent[0][3]) + self.assertIn('80.0 MiB', sent[0][1]) + def test_clear_queue_removes_all_pending_commands(self): self.module.enqueue_db('INSERT INTO sample VALUES (1)') self.module.enqueue_db('INSERT INTO sample VALUES (2)') From c8386bcc3c375ae6bdb8fbbf1586faa5c8a6965f Mon Sep 17 00:00:00 2001 From: Martin Pihrt Date: Tue, 13 Oct 2026 10:07:00 +0200 Subject: [PATCH 2/3] Fix Database Connector backup settings save --- CHANGELOG.md | 5 +++++ README.md | 2 +- plugins/database_connector/README.md | 2 +- plugins/database_connector/__init__.py | 8 +++++++- plugins/database_connector/plugin.json | 2 +- tests/test_database_connector_queue.py | 2 ++ 6 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dd7a6a..c4898df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # OSPy-plugins Changelog +October 13 2026 +---------------- +(Martin Pihrt) - Database Connector v1.1.2
+Fixed saving Database Connector settings after a database backup existed. Runtime backup names, backup sizes and queue counters are now excluded from OSPy's generic option-form parser, which incorrectly treats every list as a list of integers; existing `.sql` backup files and saved options require no migration. + October 13 2026 ---------------- (Martin Pihrt) - Database Connector v1.1.1
diff --git a/README.md b/README.md index 8236bf6..45e9b96 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ This table is generated from the `name`, `version` and `description` fields in e | [**CHMI**](plugins/chmi/) | `1.0.7` | Downloads CHMI or SHMU radar data, detects local rain and optionally controls a hardware map. | | [**CLI Control**](plugins/cli_control/) | `1.0.0` | Runs administrator-configured shell commands when OSPy station states change. | | [**Current Loop Tanks Monitor**](plugins/current_loop_tanks_monitor/) | `1.1.1` | Monitors up to four water tanks through an ADS1115 and 4-20 mA pressure sensors. | -| [**Database Connector**](plugins/database_connector/) | `1.1.1` | Provides shared MySQL access, backup and durable queued writes for OSPy plug-ins. | +| [**Database Connector**](plugins/database_connector/) | `1.1.2` | Provides shared MySQL access, backup and durable queued writes for OSPy plug-ins. | | [**Direct 16 Relay Outputs**](plugins/relay_16/) | `1.0.0` | Controls up to 16 direct Raspberry Pi GPIO relay outputs with configurable trigger polarity. | | [**Door Opening**](plugins/door_opening/) | `1.0.0` | Activates a selected OSPy station for a bounded door or gate opening interval. | | [**E-mail Notifications**](plugins/email_notifications/) | `1.0.1` | Sends OSPy event notifications through a STARTTLS SMTP server with an optional retry queue. | diff --git a/plugins/database_connector/README.md b/plugins/database_connector/README.md index 76210b0..16c3c23 100644 --- a/plugins/database_connector/README.md +++ b/plugins/database_connector/README.md @@ -7,7 +7,7 @@ https://dev.mysql.com/doc/connector-python/en/connector-python-installation.html The plug-in includes an OSPy `plugin.json` manifest and implements the common lifecycle and `health()` interfaces. Diagnostics reports whether the connector is enabled and available, its version, configured server and database, and the result of the latest real database operation. Database connections and cursors are closed after every operation. -October 13, 2026 GitHub/OSPy-plugins/plugins/database_connector — The durable SQLite-backed queue now preserves the source-command timestamp when it replays a statement. For MySQL/MariaDB statements that use `CURRENT_TIMESTAMP` or a `TIMESTAMP DEFAULT CURRENT_TIMESTAMP` column, the restored record therefore retains the original logging time instead of the later recovery time. The queue has an adjustable warning threshold (80 MiB by default), a hard maximum (100 MiB by default), an optional alert through E-mail Notifications SSL and an editable alert subject. +October 13, 2026 GitHub/OSPy-plugins/plugins/database_connector — The durable SQLite-backed queue now preserves the source-command timestamp when it replays a statement. For MySQL/MariaDB statements that use `CURRENT_TIMESTAMP` or a `TIMESTAMP DEFAULT CURRENT_TIMESTAMP` column, the restored record therefore retains the original logging time instead of the later recovery time. The queue has an adjustable warning threshold (80 MiB by default), a hard maximum (100 MiB by default), an optional alert through E-mail Notifications SSL and an editable alert subject. Version 1.1.2 excludes runtime backup-file lists and queue counters from the generic settings parser, so existing `.sql` backup names cannot make saving Database Connector settings fail. Plugin setup ----------- diff --git a/plugins/database_connector/__init__.py b/plugins/database_connector/__init__.py index e6f9bf7..86c4f14 100644 --- a/plugins/database_connector/__init__.py +++ b/plugins/database_connector/__init__.py @@ -704,7 +704,13 @@ def POST(self): qdict = web.input() verify_csrf(qdict) queue_warning_mib, queue_max_mib = queue_limits_from_input(qdict) - plugin_options.web_update(qdict) + # Backup names/sizes and queue counters are runtime values, not + # form fields. PluginOptions treats every list as integer input, + # so an existing backup filename must never be parsed here. + plugin_options.web_update( + qdict, + skipped={'sql_name', 'sql_size', 'queue_size', 'queue_bytes'}, + ) plugin_options.__setitem__('use_buffer', 'use_buffer' in qdict) plugin_options.__setitem__('use', 'use' in qdict) if 'queue_email_alerts_present' in qdict: diff --git a/plugins/database_connector/plugin.json b/plugins/database_connector/plugin.json index d789c20..3d977ee 100644 --- a/plugins/database_connector/plugin.json +++ b/plugins/database_connector/plugin.json @@ -3,7 +3,7 @@ "id": "database_connector", "name": "Database Connector", "menu": "Package: Database Connector", - "version": "1.1.1", + "version": "1.1.2", "description": "Provides shared MySQL access, backup and durable queued writes for OSPy plug-ins.", "author": "Martin Pihrt", "homepage": "https://github.com/martinpihrt/OSPy-plugins/tree/master/plugins/database_connector", diff --git a/tests/test_database_connector_queue.py b/tests/test_database_connector_queue.py index c91aa10..8577955 100644 --- a/tests/test_database_connector_queue.py +++ b/tests/test_database_connector_queue.py @@ -79,8 +79,10 @@ def _install_stub(self, name, **attributes): self._install_module(name, module) def test_settings_template_reads_queue_count_from_plugin_options(self): + source = PLUGIN_PATH.read_text(encoding='utf-8') template = (ROOT / 'plugins' / 'database_connector' / 'templates' / 'database_connector.html').read_text(encoding='utf-8') + self.assertIn("skipped={'sql_name', 'sql_size', 'queue_size', 'queue_bytes'}", source) self.assertIn("$plugin_options['queue_size']", template) self.assertIn("name='queue_warning_mib'", template) self.assertIn("name='queue_max_mib'", template) From 90f1b5f75e2cbde7546e68312bc86fb0dd177065 Mon Sep 17 00:00:00 2001 From: Martin Pihrt Date: Tue, 13 Oct 2026 10:50:27 +0200 Subject: [PATCH 3/3] Fix Database Connector settings redirect --- CHANGELOG.md | 5 +++++ README.md | 2 +- plugins/database_connector/README.md | 2 +- plugins/database_connector/__init__.py | 2 +- plugins/database_connector/plugin.json | 2 +- tests/test_database_connector_queue.py | 23 +++++++++++++++++++---- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4898df..832615b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # OSPy-plugins Changelog +October 13 2026 +---------------- +(Martin Pihrt) - Database Connector v1.1.3
+Moved the successful settings `SeeOther` redirect outside the POST exception handler, so its normal HTTP 303 response is not reported as an internal error. Added a regression test for the real settings POST flow. + October 13 2026 ---------------- (Martin Pihrt) - Database Connector v1.1.2
diff --git a/README.md b/README.md index 45e9b96..1a0447b 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ This table is generated from the `name`, `version` and `description` fields in e | [**CHMI**](plugins/chmi/) | `1.0.7` | Downloads CHMI or SHMU radar data, detects local rain and optionally controls a hardware map. | | [**CLI Control**](plugins/cli_control/) | `1.0.0` | Runs administrator-configured shell commands when OSPy station states change. | | [**Current Loop Tanks Monitor**](plugins/current_loop_tanks_monitor/) | `1.1.1` | Monitors up to four water tanks through an ADS1115 and 4-20 mA pressure sensors. | -| [**Database Connector**](plugins/database_connector/) | `1.1.2` | Provides shared MySQL access, backup and durable queued writes for OSPy plug-ins. | +| [**Database Connector**](plugins/database_connector/) | `1.1.3` | Provides shared MySQL access, backup and durable queued writes for OSPy plug-ins. | | [**Direct 16 Relay Outputs**](plugins/relay_16/) | `1.0.0` | Controls up to 16 direct Raspberry Pi GPIO relay outputs with configurable trigger polarity. | | [**Door Opening**](plugins/door_opening/) | `1.0.0` | Activates a selected OSPy station for a bounded door or gate opening interval. | | [**E-mail Notifications**](plugins/email_notifications/) | `1.0.1` | Sends OSPy event notifications through a STARTTLS SMTP server with an optional retry queue. | diff --git a/plugins/database_connector/README.md b/plugins/database_connector/README.md index 16c3c23..02de4d4 100644 --- a/plugins/database_connector/README.md +++ b/plugins/database_connector/README.md @@ -7,7 +7,7 @@ https://dev.mysql.com/doc/connector-python/en/connector-python-installation.html The plug-in includes an OSPy `plugin.json` manifest and implements the common lifecycle and `health()` interfaces. Diagnostics reports whether the connector is enabled and available, its version, configured server and database, and the result of the latest real database operation. Database connections and cursors are closed after every operation. -October 13, 2026 GitHub/OSPy-plugins/plugins/database_connector — The durable SQLite-backed queue now preserves the source-command timestamp when it replays a statement. For MySQL/MariaDB statements that use `CURRENT_TIMESTAMP` or a `TIMESTAMP DEFAULT CURRENT_TIMESTAMP` column, the restored record therefore retains the original logging time instead of the later recovery time. The queue has an adjustable warning threshold (80 MiB by default), a hard maximum (100 MiB by default), an optional alert through E-mail Notifications SSL and an editable alert subject. Version 1.1.2 excludes runtime backup-file lists and queue counters from the generic settings parser, so existing `.sql` backup names cannot make saving Database Connector settings fail. +October 13, 2026 GitHub/OSPy-plugins/plugins/database_connector — The durable SQLite-backed queue now preserves the source-command timestamp when it replays a statement. For MySQL/MariaDB statements that use `CURRENT_TIMESTAMP` or a `TIMESTAMP DEFAULT CURRENT_TIMESTAMP` column, the restored record therefore retains the original logging time instead of the later recovery time. The queue has an adjustable warning threshold (80 MiB by default), a hard maximum (100 MiB by default), an optional alert through E-mail Notifications SSL and an editable alert subject. Version 1.1.2 excludes runtime backup-file lists and queue counters from the generic settings parser, so existing `.sql` backup names cannot make saving Database Connector settings fail. Version 1.1.3 lets the successful settings POST return its normal HTTP 303 redirect instead of incorrectly handling it as an internal error. Plugin setup ----------- diff --git a/plugins/database_connector/__init__.py b/plugins/database_connector/__init__.py index 86c4f14..c81bbfd 100644 --- a/plugins/database_connector/__init__.py +++ b/plugins/database_connector/__init__.py @@ -721,12 +721,12 @@ def POST(self): start_queue_worker() elif not plugin_options['use_buffer']: queue_stop_event.set() - raise web.seeother(plugin_url(settings_page), True) except: log.error(NAME, _('Database Connector') + ':\n' + traceback.format_exc()) msg = _('An internal error was found in the system, see the error log for more information. The error is in part:') + ' ' msg += _('database_connector -> settings_page POST') return self.core_render.notice('/', msg) + raise web.seeother(plugin_url(settings_page), True) class backup_page(ProtectedPage): """Load an html page for backup""" diff --git a/plugins/database_connector/plugin.json b/plugins/database_connector/plugin.json index 3d977ee..cf1b803 100644 --- a/plugins/database_connector/plugin.json +++ b/plugins/database_connector/plugin.json @@ -3,7 +3,7 @@ "id": "database_connector", "name": "Database Connector", "menu": "Package: Database Connector", - "version": "1.1.2", + "version": "1.1.3", "description": "Provides shared MySQL access, backup and durable queued writes for OSPy plug-ins.", "author": "Martin Pihrt", "homepage": "https://github.com/martinpihrt/OSPy-plugins/tree/master/plugins/database_connector", diff --git a/tests/test_database_connector_queue.py b/tests/test_database_connector_queue.py index 8577955..e327e6b 100644 --- a/tests/test_database_connector_queue.py +++ b/tests/test_database_connector_queue.py @@ -22,8 +22,9 @@ class PluginOptions(dict): def __init__(self, _name, defaults): super().__init__(defaults) - def web_update(self, values): - self.update(values) + def web_update(self, values, skipped=None): + self.last_skipped = set(skipped or ()) + self.update({key: value for key, value in values.items() if key not in self.last_skipped}) plugins = types.ModuleType('plugins') plugins.__path__ = [] @@ -79,10 +80,8 @@ def _install_stub(self, name, **attributes): self._install_module(name, module) def test_settings_template_reads_queue_count_from_plugin_options(self): - source = PLUGIN_PATH.read_text(encoding='utf-8') template = (ROOT / 'plugins' / 'database_connector' / 'templates' / 'database_connector.html').read_text(encoding='utf-8') - self.assertIn("skipped={'sql_name', 'sql_size', 'queue_size', 'queue_bytes'}", source) self.assertIn("$plugin_options['queue_size']", template) self.assertIn("name='queue_warning_mib'", template) self.assertIn("name='queue_max_mib'", template) @@ -90,6 +89,22 @@ def test_settings_template_reads_queue_count_from_plugin_options(self): self.assertIn("name='queue_email_subject'", template) self.assertNotIn('$queue_size', template) + def test_settings_post_redirects_without_reporting_an_error(self): + class SeeOther(Exception): + pass + + self.module.web.seeother = SeeOther + self.module.web.input = lambda: {'use': 'use'} + + with self.assertRaises(SeeOther): + self.module.settings_page().POST() + + self.assertEqual( + self.module.plugin_options.last_skipped, + {'sql_name', 'sql_size', 'queue_size', 'queue_bytes'}, + ) + self.module.log.error.assert_not_called() + def test_enqueue_preserves_command_order_and_commit_flag(self): self.module.enqueue_db('INSERT INTO sample VALUES (1)', commit=True) self.module.enqueue_db('INSERT INTO sample VALUES (2)', commit=False)