From a5e318f0cda5359c616115497484674489fe891b Mon Sep 17 00:00:00 2001 From: Bob Date: Fri, 28 Aug 2026 02:42:02 +0000 Subject: [PATCH 1/5] fix(datastore): apply privacy filters on save and startup The webui saved settings.privacy_filters, but DatastoreWorker started with an empty engine and never reloaded on SetKeyValue. Insert and heartbeat paths filtered correctly against rules that were never loaded, so drop/redact rules appeared to do nothing (ActivityWatch/aw-server-rust#659). Reload the in-memory engine when that key is written or deleted, load it at worker startup, and keep RefreshPrivacyFilter as an explicit path. --- aw-datastore/src/worker.rs | 53 +++++++++++++----- aw-datastore/tests/datastore.rs | 84 +++++++++++++++++++++++++++++ aw-server/src/endpoints/settings.rs | 17 +++++- 3 files changed, 139 insertions(+), 15 deletions(-) diff --git a/aw-datastore/src/worker.rs b/aw-datastore/src/worker.rs index 8e37bac1..23e225b4 100644 --- a/aw-datastore/src/worker.rs +++ b/aw-datastore/src/worker.rs @@ -84,6 +84,12 @@ pub enum Command { Close(), } +/// Key the webui writes via POST /0/settings/privacy_filters. +/// The worker's in-memory PrivacyFilterEngine is the only thing that actually +/// filters inserts/heartbeats, so every write/delete of this key (and startup) +/// must reload the engine. RefreshPrivacyFilter exists for explicit reloads. +const PRIVACY_FILTERS_KEY: &str = "settings.privacy_filters"; + fn _unwrap_empty_response(response: Response) -> Result<(), DatastoreError> { match response { Response::Empty() => Ok(()), @@ -117,6 +123,22 @@ impl DatastoreWorker { } } + /// Replace the in-memory engine from `settings.privacy_filters`. + /// Parse errors keep the previous engine (a bad save should not unfilter + /// already-loaded rules). A missing key clears the engine so deleting the + /// setting actually disables filtering. + fn reload_privacy_engine(&mut self, ds: &DatastoreInstance, conn: &Connection) { + match ds.get_key_value(conn, PRIVACY_FILTERS_KEY) { + Ok(json_str) => match PrivacyFilterEngine::from_json(&json_str) { + Ok(engine) => self.privacy_engine = engine, + Err(e) => warn!("Failed to parse privacy_filters setting: {e}"), + }, + Err(_) => { + self.privacy_engine = PrivacyFilterEngine::new(vec![]); + } + } + } + fn work_loop(&mut self, method: DatastoreMethod) { // Open SQLite connection let mut conn = match &method { @@ -165,6 +187,11 @@ impl DatastoreWorker { let mut ds = DatastoreInstance::new(&conn, true).unwrap(); + // Load persisted privacy filters before serving inserts. The engine + // starts empty; without this, rules saved in a previous process sit + // unused until something happens to send RefreshPrivacyFilter. + self.reload_privacy_engine(&ds, &conn); + // Ensure legacy import if self.legacy_import { let transaction = match conn.transaction_with_behavior(TransactionBehavior::Immediate) { @@ -386,7 +413,12 @@ impl DatastoreWorker { Err(e) => Err(e), }, Command::SetKeyValue(key, data) => match ds.insert_key_value(tx, &key, &data) { - Ok(()) => Ok(Response::Empty()), + Ok(()) => { + if key == PRIVACY_FILTERS_KEY { + self.reload_privacy_engine(ds, tx); + } + Ok(Response::Empty()) + } Err(e) => Err(e), }, Command::GetKeyValue(key) => match ds.get_key_value(tx, &key) { @@ -394,21 +426,16 @@ impl DatastoreWorker { Err(e) => Err(e), }, Command::DeleteKeyValue(key) => match ds.delete_key_value(tx, &key) { - Ok(()) => Ok(Response::Empty()), + Ok(()) => { + if key == PRIVACY_FILTERS_KEY { + self.reload_privacy_engine(ds, tx); + } + Ok(Response::Empty()) + } Err(e) => Err(e), }, Command::RefreshPrivacyFilter() => { - // Reload privacy filter rules from settings - match ds.get_key_value(tx, "settings.privacy_filters") { - Ok(json_str) => match PrivacyFilterEngine::from_json(&json_str) { - Ok(engine) => self.privacy_engine = engine, - Err(e) => warn!("Failed to parse privacy_filters setting: {e}"), - }, - Err(_) => { - // Settings key absent — clear rules so removing the key disables filtering - self.privacy_engine = PrivacyFilterEngine::new(vec![]); - } - } + self.reload_privacy_engine(ds, tx); Ok(Response::Empty()) } Command::RenameBucket(old_id, new_id) => match ds.rename_bucket(tx, &old_id, &new_id) { diff --git a/aw-datastore/tests/datastore.rs b/aw-datastore/tests/datastore.rs index 9c05740c..43b8c97f 100644 --- a/aw-datastore/tests/datastore.rs +++ b/aw-datastore/tests/datastore.rs @@ -832,4 +832,88 @@ mod datastore_tests { let _ = fs::remove_file(&db_path); } + + fn privacy_event(title: &str) -> Event { + Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(1), + data: json_map! {"title": json!(title), "app": json!("Firefox")}, + } + } + + /// Saving privacy_filters used to persist the JSON and never load it into + /// the worker's engine (ActivityWatch/aw-server-rust#659). A drop rule + /// must actually drop a matching insert without an explicit refresh call. + #[test] + fn test_privacy_filter_applies_after_setting_saved() { + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + let drop_secret = + r#"[{"enabled":true,"field":"title","pattern":"(?i)secret","action":"drop"}]"#; + ds.set_key_value("settings.privacy_filters", drop_secret) + .unwrap(); + + ds.insert_events( + &bucket.id, + &[privacy_event("my secret file"), privacy_event("readme.md")], + ) + .unwrap(); + + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!( + events.len(), + 1, + "drop rule should discard the matching event" + ); + assert_eq!(events[0].data.get("title").unwrap(), "readme.md"); + + ds.delete_key_value("settings.privacy_filters").unwrap(); + ds.insert_events(&bucket.id, &[privacy_event("another secret")]) + .unwrap(); + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!( + events.len(), + 2, + "deleting the setting should stop filtering" + ); + } + + /// Rules must load at worker startup, not only after a later SetKeyValue. + #[test] + fn test_privacy_filter_survives_datastore_reload() { + let mut db_path = get_cache_dir().unwrap(); + db_path.push("datastore-unittest-privacy-filters.db"); + let db_path_str = db_path.to_str().unwrap().to_string(); + if db_path.exists() { + std::fs::remove_file(db_path.clone()) + .expect("Failed to remove datastore-unittest-privacy-filters.db"); + } + + let drop_secret = + r#"[{"enabled":true,"field":"title","pattern":"(?i)secret","action":"drop"}]"#; + { + let ds = Datastore::new(db_path_str.clone(), false); + create_test_bucket(&ds); + ds.set_key_value("settings.privacy_filters", drop_secret) + .unwrap(); + ds.force_commit().unwrap(); + ds.close(); + } + { + let ds = Datastore::new(db_path_str, false); + ds.insert_events("testid", &[privacy_event("top secret notes")]) + .unwrap(); + let events = ds.get_events("testid", None, None, None).unwrap(); + assert!( + events.is_empty(), + "persisted drop rule must apply after reopen, got {events:?}" + ); + ds.close(); + } + + std::fs::remove_file(&db_path) + .expect("Failed to remove datastore-unittest-privacy-filters.db"); + } } diff --git a/aw-server/src/endpoints/settings.rs b/aw-server/src/endpoints/settings.rs index d678687b..0520911b 100644 --- a/aw-server/src/endpoints/settings.rs +++ b/aw-server/src/endpoints/settings.rs @@ -124,7 +124,15 @@ pub fn setting_set( let result = datastore.set_key_value(&setting_key, &value_str); match result { - Ok(_) => Ok(Status::Created), + Ok(_) => { + // Worker also reloads on SetKeyValue of this key; this second + // RefreshPrivacyFilter is belt-and-suspenders so the HTTP path + // still works if a future writer bypasses that hook. + if setting_key == "settings.privacy_filters" { + let _ = datastore.refresh_privacy_filter(); + } + Ok(Status::Created) + } Err(err) => Err(err.into()), } } @@ -137,7 +145,12 @@ pub fn setting_delete(state: &State, key: String) -> Result<(), Htt let result = datastore.delete_key_value(&setting_key); match result { - Ok(_) => Ok(()), + Ok(_) => { + if setting_key == "settings.privacy_filters" { + let _ = datastore.refresh_privacy_filter(); + } + Ok(()) + } Err(err) => Err(err.into()), } } From 85cd5915df233ad1f87d016756309c94550310a2 Mon Sep 17 00:00:00 2001 From: Bob Date: Fri, 28 Aug 2026 02:49:18 +0000 Subject: [PATCH 2/5] test(datastore): do not fail privacy-filter reload cleanup on Windows The reopen assertion already passed on windows-latest; remove_file then hit ERROR_SHARING_VIOLATION because the worker thread still held the SQLite handle. Unique-ify the temp db and treat cleanup as best-effort. --- aw-datastore/tests/datastore.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/aw-datastore/tests/datastore.rs b/aw-datastore/tests/datastore.rs index 43b8c97f..1edbb004 100644 --- a/aw-datastore/tests/datastore.rs +++ b/aw-datastore/tests/datastore.rs @@ -884,12 +884,12 @@ mod datastore_tests { #[test] fn test_privacy_filter_survives_datastore_reload() { let mut db_path = get_cache_dir().unwrap(); - db_path.push("datastore-unittest-privacy-filters.db"); + db_path.push(format!( + "datastore-unittest-privacy-filters-{}.db", + std::process::id() + )); let db_path_str = db_path.to_str().unwrap().to_string(); - if db_path.exists() { - std::fs::remove_file(db_path.clone()) - .expect("Failed to remove datastore-unittest-privacy-filters.db"); - } + let _ = std::fs::remove_file(&db_path); let drop_secret = r#"[{"enabled":true,"field":"title","pattern":"(?i)secret","action":"drop"}]"#; @@ -913,7 +913,10 @@ mod datastore_tests { ds.close(); } - std::fs::remove_file(&db_path) - .expect("Failed to remove datastore-unittest-privacy-filters.db"); + // Windows can still hold the SQLite handle after close() while the + // worker thread unwinds (ERROR_SHARING_VIOLATION). Cleanup is best-effort. + let _ = std::fs::remove_file(&db_path); + let _ = std::fs::remove_file(db_path.with_extension("db-wal")); + let _ = std::fs::remove_file(db_path.with_extension("db-shm")); } } From f9f3ae74a02e49d1f5347130b5e98cafc878e94a Mon Sep 17 00:00:00 2001 From: Bob Date: Fri, 28 Aug 2026 03:14:10 +0000 Subject: [PATCH 3/5] fix(datastore): restore privacy engine after a failed commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A SetKeyValue/DeleteKeyValue on settings.privacy_filters reloads the in-memory engine from the still-open transaction. If the batched commit later fails, SQLite rolls back but the engine keeps the rules from the rolled-back write, so subsequent inserts filter against state that no longer exists on disk — including running unfiltered after a rolled-back delete. Reload from the durable connection on the commit-failure path so the engine matches what actually persisted. --- aw-datastore/src/worker.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aw-datastore/src/worker.rs b/aw-datastore/src/worker.rs index 23e225b4..165a4915 100644 --- a/aw-datastore/src/worker.rs +++ b/aw-datastore/src/worker.rs @@ -290,6 +290,12 @@ impl DatastoreWorker { // know to retry. Rolled-back events create a gap in the timeline; // watchers will resume sending heartbeats from current state, but the // specific batch of events is permanently lost. + // + // A privacy-setting mutation may also have replaced the in-memory + // engine from this transaction. The failed commit rolled SQLite back, + // so restore the engine from the durable connection before processing + // another event. + self.reload_privacy_engine(&ds, &conn); if let Some((sender, _)) = deferred_ack.take() { sender.respond(Err(DatastoreError::InternalError(format!( "Failed to commit datastore transaction: {err}" From 6bf2934cf805441452a0d359de928d02440c5c5f Mon Sep 17 00:00:00 2001 From: Bob Date: Fri, 28 Aug 2026 03:24:00 +0000 Subject: [PATCH 4/5] fix(datastore): only clear privacy rules when the setting is absent The reload treated every get_key_value error as "key deleted" and emptied the engine. A transient InternalError from the query would therefore disable filtering entirely and let the events these rules exist to exclude get stored. Clear only on NoSuchKey; warn and keep the current engine otherwise. --- aw-datastore/src/worker.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/aw-datastore/src/worker.rs b/aw-datastore/src/worker.rs index 165a4915..a18f9756 100644 --- a/aw-datastore/src/worker.rs +++ b/aw-datastore/src/worker.rs @@ -124,18 +124,20 @@ impl DatastoreWorker { } /// Replace the in-memory engine from `settings.privacy_filters`. - /// Parse errors keep the previous engine (a bad save should not unfilter - /// already-loaded rules). A missing key clears the engine so deleting the - /// setting actually disables filtering. + /// Only an absent key clears the engine, so deleting the setting actually + /// disables filtering. Parse errors and query errors keep the previous + /// engine: unfiltering on a bad save or a transient database error would + /// silently store the events these rules exist to keep out. fn reload_privacy_engine(&mut self, ds: &DatastoreInstance, conn: &Connection) { match ds.get_key_value(conn, PRIVACY_FILTERS_KEY) { Ok(json_str) => match PrivacyFilterEngine::from_json(&json_str) { Ok(engine) => self.privacy_engine = engine, Err(e) => warn!("Failed to parse privacy_filters setting: {e}"), }, - Err(_) => { + Err(DatastoreError::NoSuchKey(_)) => { self.privacy_engine = PrivacyFilterEngine::new(vec![]); } + Err(e) => warn!("Failed to load privacy_filters setting: {e:?}"), } } From 1b3aaa297321e032ceabb3388b1bd4b3e0574b37 Mon Sep 17 00:00:00 2001 From: Bob Date: Mon, 31 Aug 2026 15:37:44 +0000 Subject: [PATCH 5/5] fix(datastore): restore privacy engine from the pre-transaction snapshot A privacy-setting mutation reloads the engine from the still-open transaction. On commit failure, re-querying the durable connection and keeping the current engine on read error left a rolled-back delete's empty engine in place, so later inserts ran unfiltered. Restore the snapshot taken at transaction start instead. Recovery then does not depend on a second read succeeding. --- aw-datastore/src/worker.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/aw-datastore/src/worker.rs b/aw-datastore/src/worker.rs index a18f9756..51b1f168 100644 --- a/aw-datastore/src/worker.rs +++ b/aw-datastore/src/worker.rs @@ -229,6 +229,13 @@ impl DatastoreWorker { continue; } }; + // Snapshot BEFORE the request loop. SetKeyValue/DeleteKeyValue + // reload the engine from the still-open transaction so a later + // insert in the same batch is filtered. If commit fails we restore + // this snapshot — not "keep current" (that is the rolled-back + // view) and not a durable re-query (a read error would leave the + // uncommitted engine in place: fail-open after a rolled-back delete). + let privacy_engine_at_tx_start = self.privacy_engine.clone(); tx.set_drop_behavior(DropBehavior::Commit); self.uncommitted_events = 0; @@ -293,11 +300,10 @@ impl DatastoreWorker { // watchers will resume sending heartbeats from current state, but the // specific batch of events is permanently lost. // - // A privacy-setting mutation may also have replaced the in-memory - // engine from this transaction. The failed commit rolled SQLite back, - // so restore the engine from the durable connection before processing - // another event. - self.reload_privacy_engine(&ds, &conn); + // Restore the pre-transaction engine. Reloading from the durable + // connection is not enough: if that read fails, keep-on-error would + // preserve the uncommitted engine (empty after a rolled-back delete). + self.privacy_engine = privacy_engine_at_tx_start; if let Some((sender, _)) = deferred_ack.take() { sender.respond(Err(DatastoreError::InternalError(format!( "Failed to commit datastore transaction: {err}"