From 552454c1e0b736ae39af087cf162c655acbbcfaf Mon Sep 17 00:00:00 2001 From: Alex Sorafumo Date: Thu, 19 Nov 2026 19:44:05 +1100 Subject: [PATCH 1/3] feat(place/remote_debug): get logs, status and screen captures from PlaceOS apps Adds the RemoteDebug logic driver for the remote debugging interface in PlaceOS/user-interfaces#753. People with the support role request logs and send commands by device ID. Devices send entries with `post_logs` and results with `command_result`. Co-Authored-By: Claude Opus 5.5 --- drivers/place/remote_debug.cr | 299 +++++++++++++++++++++++++++ drivers/place/remote_debug_models.cr | 227 ++++++++++++++++++++ drivers/place/remote_debug_spec.cr | 127 ++++++++++++ 3 files changed, 653 insertions(+) create mode 100644 drivers/place/remote_debug.cr create mode 100644 drivers/place/remote_debug_models.cr create mode 100644 drivers/place/remote_debug_spec.cr diff --git a/drivers/place/remote_debug.cr b/drivers/place/remote_debug.cr new file mode 100644 index 0000000000..55c3e1a842 --- /dev/null +++ b/drivers/place/remote_debug.cr @@ -0,0 +1,299 @@ +require "placeos-driver" +require "./remote_debug_models" + +# Remote debugging of PlaceOS apps, such as Booking Panel, Control and Signage. +# The apps find this driver from the `remote_debug` binding of the +# organisation zone. A device reads its `device:` binding, sends logs with +# `post_logs` and sends command results with `command_result`. +# +# Interface: https://github.com/PlaceOS/user-interfaces/blob/develop/docs/remote-debug.md +# Device side: https://github.com/PlaceOS/user-interfaces/blob/develop/libs/common/src/lib/remote-debug.ts +class Place::RemoteDebug < PlaceOS::Driver + descriptive_name "PlaceOS Remote Debug" + generic_name :RemoteDebug + description <<-DESC + Get the logs, status and screen captures of PlaceOS apps by their device ID. + + To turn it on, set the `remote_debug` binding of the organisation zone to + `{"id": "", "module": "RemoteDebug"}`. + + Add a StaffAPI module to this system so the driver can remove old screen + captures from the uploads. Only support users can request logs and send commands. + DESC + + # Most entries that the driver keeps for each device + MAX_DEVICE_ENTRIES = 5000 + # Most command results that the driver keeps for each device + MAX_DEVICE_RESULTS = 50 + # Most snapshots and animations that the driver keeps for each device + MAX_DEVICE_MEDIA = 3 + # Most devices that the driver keeps data for, and most devices with a request or commands + MAX_DEVICES = 1000 + # Most commands for each device. The device reads the first 20 + MAX_COMMANDS = 20 + # Most keys in a `settings` command + MAX_SETTINGS_KEYS = 50 + # Most entries in one `post_logs` call. The device sends 100 maximum + MAX_BATCH = 100 + # Most characters in the JSON of a command result. The device sends 100,000 maximum + MAX_RESULT_LENGTH = 100_000 + # Most buffered entries that one request gets + MAX_HISTORY = 1000 + DEFAULT_DURATION_MS = 15 * 60 * 1000 + MAX_DURATION_MS = 60 * 60 * 1000 + # Time before the driver removes a command that the device did not run + COMMAND_TIMEOUT = 15.minutes + # Most time between updates of `devices` when only `last_seen` changes + SEEN_INTERVAL_MS = 60 * 1000 + DEVICE_ID = /\A[A-Za-z0-9_-]{1,64}\z/ + + # State of each device with a log request or commands + @states = {} of String => RemoteDebugDeviceState + # Devices that called the driver, by ID + @devices = {} of String => RemoteDebugDevice + @devices_published_at = 0_i64 + @logs = {} of String => RemoteLogsView + @results = {} of String => Array(RemoteDebugCommandResult) + @media = {} of String => Array(RemoteDebugMedia) + # End timers of the active log requests, by device ID + @expiry = {} of String => PlaceOS::Driver::Proxy::Scheduler::TaskWrapper + + def on_load + self[:devices] = @devices + end + + # =================================== + # Methods for the people who debug + # =================================== + + # Sets a new log request for a device. Returns the request ID + @[Security(Level::Support)] + def request_logs(device_id : String, level : RemoteLogLevel, options : RemoteLogRequestOptions? = nil) : String + check_device_id device_id + options ||= RemoteLogRequestOptions.new + request = RemoteLogRequest.new( + id: new_id, + level: level, + history: options.history.try(&.clamp(0, MAX_HISTORY)), + before: options.before, + duration: options.duration.try(&.clamp(0, MAX_DURATION_MS)), + ) + state_for(device_id).logs = request + publish_state device_id + start_expiry device_id, request.duration + request.id + end + + # Changes the level of the active log request. The request keeps its ID + @[Security(Level::Support)] + def set_level(device_id : String, level : RemoteLogLevel) : Nil + state = @states[device_id]? + logs = state.try &.logs + # The device ignores the same level, so its duration does not start again + return if state.nil? || logs.nil? || logs.level == level + state.logs = RemoteLogRequest.new(logs.id, level, logs.history, logs.before, logs.duration) + publish_state device_id + # The device starts the duration again when the level changes + start_expiry device_id, logs.duration + end + + # Ends the active log request of a device + @[Security(Level::Support)] + def stop_logs(device_id : String) : Nil + @expiry.delete(device_id).try &.cancel + state = @states[device_id]? + return unless state && state.logs + state.logs = nil + publish_state device_id + end + + # Adds a command for a device. Returns the command ID. + # `options` holds the fields of the command, such as `keys`, `path` or `width` + @[Security(Level::Support)] + def send_command(device_id : String, type : RemoteDebugCommandType, options : RemoteDebugCommandOptions? = nil) : String + check_device_id device_id + command = build_command(type, options || RemoteDebugCommandOptions.new) + count = @states[device_id]?.try(&.commands.size) || 0 + raise ArgumentError.new("device #{device_id} has #{MAX_COMMANDS} commands already") if count >= MAX_COMMANDS + state_for(device_id).commands << command + publish_state device_id + schedule.in(COMMAND_TIMEOUT) { remove_command(device_id, command.id) } + command.id + end + + # =================================== + # Methods that the devices call + # =================================== + + # Adds the entries of a device. Ignores calls for a request that is not active + def post_logs(device_id : String, request_id : String, entries : Array(RemoteLogEntry), device : RemoteDebugDevice) : Nil + logs = @states[device_id]?.try &.logs + return unless logs && logs.id == request_id && device.id == device_id + seen device_id, device + + view = @logs[device_id]? + # A new `started_at` shows that the app reloaded and `seq` started again + same_session = view && view.device.started_at == device.started_at + old = view && same_session ? view.entries : [] of RemoteLogEntry + known = old.map(&.seq).to_set + # The device sends a failed batch again, so skip the entries that we have + added = entries.first(MAX_BATCH).reject { |entry| known.includes?(entry.seq) }.uniq!(&.seq) + return if same_session && added.empty? + + merged = (old + added).sort_by!(&.seq).last(MAX_DEVICE_ENTRIES) + self["logs:#{device_id}"] = @logs[device_id] = RemoteLogsView.new(device, merged) + end + + # Keeps the result of a command and removes the command + def command_result(device_id : String, command_id : String, result : RemoteDebugResult, device : RemoteDebugDevice) : Nil + command = @states[device_id]?.try &.commands.find(&.id.==(command_id)) + return unless command && device.id == device_id + seen device_id, device + remove_command device_id, command_id + + if result.to_json.size > MAX_RESULT_LENGTH + result = RemoteDebugResult.new(ok: false, error: "Result is too large") + end + add_media(device_id, command_id, result) if command.type.media? + + results = @results[device_id]? || [] of RemoteDebugCommandResult + results << RemoteDebugCommandResult.new(command.id, command.type, result, now_ms) + self["results:#{device_id}"] = @results[device_id] = results.last(MAX_DEVICE_RESULTS) + end + + # =================================== + # Helpers + # =================================== + + protected def check_device_id(device_id : String) : Nil + raise ArgumentError.new("invalid device ID: #{device_id}") unless DEVICE_ID.matches?(device_id) + end + + protected def new_id : String + Random::Secure.hex(6) + end + + protected def now_ms : Int64 + Time.utc.to_unix_ms + end + + # The state of a device. Adds an empty state when there is none + protected def state_for(device_id : String) : RemoteDebugDeviceState + @states[device_id]? || begin + raise ArgumentError.new("#{MAX_DEVICES} devices have a request or commands already") if @states.size >= MAX_DEVICES + @states[device_id] = RemoteDebugDeviceState.new + end + end + + # Writes the `device:` binding. An empty state is removed and shows as `null` + protected def publish_state(device_id : String) : Nil + state = @states[device_id]? + if state && state.empty? + @states.delete(device_id) + state = nil + end + self["device:#{device_id}"] = state + end + + # Ends the log request of a device when its duration ends + protected def start_expiry(device_id : String, duration : Int32?) : Nil + @expiry.delete(device_id).try &.cancel + duration = (duration || DEFAULT_DURATION_MS).clamp(0, MAX_DURATION_MS) + @expiry[device_id] = schedule.in(duration.milliseconds) do + @expiry.delete(device_id) + stop_logs device_id + end + end + + protected def build_command(type : RemoteDebugCommandType, options : RemoteDebugCommandOptions) : RemoteDebugCommand + id = new_id + case type + in .settings? + keys = options.keys + raise ArgumentError.new("settings needs keys") unless keys + raise ArgumentError.new("settings has #{MAX_SETTINGS_KEYS} keys maximum") if keys.size > MAX_SETTINGS_KEYS + RemoteDebugCommand.new(id, type, keys: keys) + in .navigate? + path = options.path + raise ArgumentError.new("navigate needs a path that starts with /") unless path && path.starts_with?('/') + RemoteDebugCommand.new(id, type, path: path) + in .snapshot? + RemoteDebugCommand.new(id, type, width: options.width) + in .animation? + RemoteDebugCommand.new(id, type, width: options.width, duration: options.duration, fps: options.fps, format: options.format) + in .status?, .storage?, .reload?, .clear_cache? + RemoteDebugCommand.new(id, type) + end + end + + protected def remove_command(device_id : String, command_id : String) : Nil + commands = @states[device_id]?.try &.commands + return unless commands + count = commands.size + commands.reject!(&.id.==(command_id)) + publish_state(device_id) if commands.size != count + end + + # Keeps the upload of a `snapshot` or `animation` result in `media:`. + # Removes the uploads that the driver no longer keeps + protected def add_media(device_id : String, command_id : String, result : RemoteDebugResult) : Nil + data = result.data + return unless result.ok && data + media = begin + RemoteDebugMedia.from_json(data.to_json) + rescue error : JSON::Error + logger.warn { "invalid media from #{device_id}: #{error.message}" } + return + end + media.id = command_id + media.url = "/api/engine/v2/uploads/#{URI.encode_path_segment(media.upload_id)}/url" + media.received_at = now_ms + + list = @media[device_id]? || [] of RemoteDebugMedia + list << media + list.shift(list.size - MAX_DEVICE_MEDIA).each { |old| remove_upload old.upload_id } if list.size > MAX_DEVICE_MEDIA + self["media:#{device_id}"] = @media[device_id] = list + end + + # Removes an upload with the StaffAPI module of this system, if there is one + protected def remove_upload(upload_id : String) : Nil + return unless system.exists?(:StaffAPI) + system[:StaffAPI].upload_remove(upload_id) + rescue error + logger.warn(exception: error) { "failed to remove upload #{upload_id}" } + end + + # Adds or updates a device in `devices`. Keeps the `MAX_DEVICES` devices + # that were seen last, and removes the data of the other devices. + # Writes `devices` when a device changes, or once each minute when only `last_seen` changes + protected def seen(device_id : String, device : RemoteDebugDevice) : Nil + now = now_ms + previous = @devices[device_id]? + # Compare all the fields but `last_seen` + device.last_seen = previous.try &.last_seen + changed = previous != device + device.last_seen = now + @devices[device_id] = device + + if @devices.size > MAX_DEVICES + oldest = @devices.min_by { |_id, item| item.last_seen || 0_i64 }[0] + forget oldest + changed = true + end + + return unless changed || now - @devices_published_at >= SEEN_INTERVAL_MS + @devices_published_at = now + self[:devices] = @devices + end + + # Removes the logs, results and uploads of a device + protected def forget(device_id : String) : Nil + @devices.delete(device_id) + @logs.delete(device_id) + @results.delete(device_id) + @media.delete(device_id).try &.each { |media| remove_upload media.upload_id } + self["logs:#{device_id}"] = nil + self["results:#{device_id}"] = nil + self["media:#{device_id}"] = nil + end +end diff --git a/drivers/place/remote_debug_models.cr b/drivers/place/remote_debug_models.cr new file mode 100644 index 0000000000..a20641a78e --- /dev/null +++ b/drivers/place/remote_debug_models.cr @@ -0,0 +1,227 @@ +require "json" + +# Models of the RemoteDebug driver. They match the types in +# `libs/common/src/lib/remote-debug.ts` of PlaceOS/user-interfaces. +# See `docs/remote-debug.md` in that repository. +module Place + # Log levels, lowest first. A request gets the entries at its level and higher + enum RemoteLogLevel + Debug + Info + Warn + Error + end + + enum RemoteLogSource + Console + Error + Rejection + end + + # Log entry that a device sends with `post_logs` + struct RemoteLogEntry + include JSON::Serializable + + # Most characters in a message. The device sends 2,000 maximum + MAX_MESSAGE = 2000 + + # Number of the entry on the device. Starts at 1 when the app loads + getter seq : Int64 + # Unix epoch in milliseconds on the device clock + getter timestamp : Int64 + getter level : RemoteLogLevel + getter source : RemoteLogSource + getter message : String + + def initialize(@seq, @timestamp, @level, @source, @message) + end + + def after_initialize + @message = @message[0, MAX_MESSAGE] if @message.size > MAX_MESSAGE + end + end + + # Details of the device that the device sends with each call + struct RemoteDebugDevice + include JSON::Serializable + + getter id : String + # Short name of the app + getter app : String + getter version : String + # Git hash of the build + getter build : String + # Page address, without credentials + getter url : String + getter user_agent : String + # Unix epoch in milliseconds when the app loaded. A new value shows that the app reloaded + getter started_at : Int64 + + # Unix epoch in milliseconds of the last call. Only set in the `devices` binding + @[JSON::Field(ignore_deserialize: true)] + property last_seen : Int64? = nil + + def initialize(@id, @app, @version, @build, @url, @user_agent, @started_at) + end + end + + # Log request in the `device:` binding + struct RemoteLogRequest + include JSON::Serializable + + # Unique ID of the request. A device applies each ID one time + getter id : String + getter level : RemoteLogLevel + # Most buffered entries to send first. `0` sends only new entries + getter history : Int32? + # Send only buffered entries with a `seq` lower than this value + getter before : Int64? + # Milliseconds that the device sends new entries + getter duration : Int32? + + def initialize(@id, @level, @history = nil, @before = nil, @duration = nil) + end + end + + # Options of `request_logs` + struct RemoteLogRequestOptions + include JSON::Serializable + + getter history : Int32? + getter before : Int64? + getter duration : Int32? + + def initialize(@history = nil, @before = nil, @duration = nil) + end + end + + enum RemoteDebugCommandType + Status + Settings + Storage + Navigate + Reload + ClearCache + Snapshot + Animation + + # Whether the result of this command holds a `RemoteDebugMedia` + def media? + snapshot? || animation? + end + end + + enum RemoteDebugMediaFormat + Webp + Gif + end + + # Options of `send_command`. `keys` for `settings`, `path` for `navigate`, + # `width` for `snapshot`, and all but `keys` and `path` for `animation` + struct RemoteDebugCommandOptions + include JSON::Serializable + + getter keys : Array(String)? + getter path : String? + getter width : Int32? + getter duration : Int32? + getter fps : Int32? + getter format : RemoteDebugMediaFormat? + + def initialize(@keys = nil, @path = nil, @width = nil, @duration = nil, @fps = nil, @format = nil) + end + end + + # Command in the `device:` binding. Fields that the type does not use are not set + struct RemoteDebugCommand + include JSON::Serializable + + getter id : String + getter type : RemoteDebugCommandType + getter keys : Array(String)? + getter path : String? + getter width : Int32? + getter duration : Int32? + getter fps : Int32? + getter format : RemoteDebugMediaFormat? + + def initialize(@id, @type, @keys = nil, @path = nil, @width = nil, @duration = nil, @fps = nil, @format = nil) + end + end + + # Value of the `device:` binding. A class, as the driver changes it in place + class RemoteDebugDeviceState + include JSON::Serializable + + # Active log request. `null` when there is none + @[JSON::Field(emit_null: true)] + property logs : RemoteLogRequest? = nil + # Commands for the device, oldest first + property commands : Array(RemoteDebugCommand) = [] of RemoteDebugCommand + + def initialize + end + + def empty? + logs.nil? && commands.empty? + end + end + + # Result of a command that a device sends with `command_result` + struct RemoteDebugResult + include JSON::Serializable + + getter ok : Bool + getter data : JSON::Any? + # Reason that the command failed + getter error : String? + + def initialize(@ok, @data = nil, @error = nil) + end + end + + # Data of the `snapshot` and `animation` results, and an item of the + # `media:` binding. The device sets the fields up to `upload_id`, the + # driver sets the others. + struct RemoteDebugMedia + include JSON::Serializable + + # `image/webp` or `image/png` for a snapshot, `image/webp` or `image/gif` for an animation + getter mime : String + getter width : Int32 + getter height : Int32 + getter frames : Int32 + # Milliseconds of the animation. `0` for a snapshot + getter duration : Int32 + # Bytes of the file + getter bytes : Int64 + # ID of the PlaceOS upload + getter upload_id : String + + # ID of the command + @[JSON::Field(ignore_deserialize: true)] + property id : String? = nil + # Address of the upload + @[JSON::Field(ignore_deserialize: true)] + property url : String? = nil + @[JSON::Field(ignore_deserialize: true)] + property received_at : Int64? = nil + end + + # Item of the `results:` binding + record RemoteDebugCommandResult, + id : String, + type : RemoteDebugCommandType, + result : RemoteDebugResult, + received_at : Int64 do + include JSON::Serializable + end + + # Value of the `logs:` binding + record RemoteLogsView, + device : RemoteDebugDevice, + # Entries ordered by `seq`, oldest first + entries : Array(RemoteLogEntry) do + include JSON::Serializable + end +end diff --git a/drivers/place/remote_debug_spec.cr b/drivers/place/remote_debug_spec.cr new file mode 100644 index 0000000000..c8e69df9a7 --- /dev/null +++ b/drivers/place/remote_debug_spec.cr @@ -0,0 +1,127 @@ +require "placeos-driver/spec" +require "./remote_debug_models" + +# :nodoc: +class StaffAPIMock < DriverSpecs::MockDriver + def upload_remove(upload_id : String) : Bool + removed = self[:removed]?.try(&.as_a.map(&.as_s)) || [] of String + self[:removed] = removed << upload_id + true + end +end + +DriverSpecs.mock_driver "Place::RemoteDebug" do + system({ + StaffAPI: {StaffAPIMock}, + }) + + device_id = "DEV-TEST00000001" + device = { + id: device_id, + app: "control", + version: "1.0.0", + build: "abc", + url: "https://placeos.test/control/", + user_agent: "Test", + started_at: 1000, + } + entry = ->(seq : Int32) do + {seq: seq, timestamp: seq, level: "info", source: "console", message: "line #{seq}"} + end + + it "should set, change and clear the log request of a device" do + id = exec(:request_logs, device_id, "info", {history: 50}).get.not_nil!.as_s + status["device:#{device_id}"].should eq JSON.parse({ + logs: {id: id, level: "info", history: 50}, + commands: [] of String, + }.to_json) + + exec(:set_level, device_id, "debug").get + status["device:#{device_id}"]["logs"]["id"].should eq id + status["device:#{device_id}"]["logs"]["level"].should eq "debug" + + exec(:stop_logs, device_id).get + status["device:#{device_id}"]?.should be_nil + end + + it "should clear the log request when the duration ends" do + exec(:request_logs, device_id, "warn", {duration: 100}).get + status["device:#{device_id}"]["logs"]["level"].should eq "warn" + + sleep 300.milliseconds + status["device:#{device_id}"]?.should be_nil + end + + it "should keep posted entries once each and start again after a reload" do + id = exec(:request_logs, device_id, "info").get.not_nil!.as_s + exec(:post_logs, device_id, "old-request", [entry.call(9)], device).get + status["logs:#{device_id}"]?.should be_nil + + exec(:post_logs, device_id, id, [entry.call(2), entry.call(1)], device).get + exec(:post_logs, device_id, id, [entry.call(2), entry.call(3)], device).get + status["logs:#{device_id}"]["entries"].as_a.map(&.["seq"].as_i).should eq [1, 2, 3] + + reloaded = device.merge(started_at: 2000) + exec(:post_logs, device_id, id, [entry.call(1)], reloaded).get + status["logs:#{device_id}"].should eq JSON.parse({device: reloaded, entries: [entry.call(1)]}.to_json) + status["devices"][device_id]["started_at"].should eq 2000 + status["devices"][device_id]["last_seen"].as_i64.should be > 0 + + exec(:stop_logs, device_id).get + end + + it "should keep the log request when it adds a command, and keep the result" do + request_id = exec(:request_logs, device_id, "info").get.not_nil!.as_s + command_id = exec(:send_command, device_id, "settings", {keys: ["app.title"]}).get.not_nil!.as_s + status["device:#{device_id}"]["logs"]["id"].should eq request_id + status["device:#{device_id}"]["commands"].should eq JSON.parse([{id: command_id, type: "settings", keys: ["app.title"]}].to_json) + + result = {ok: true, data: {"app.title" => "Panel"}} + exec(:command_result, device_id, command_id, result, device).get + status["device:#{device_id}"]["commands"].as_a.should be_empty + + results = Array(Place::RemoteDebugCommandResult).from_json(status["results:#{device_id}"].to_json) + results.size.should eq 1 + results[0].id.should eq command_id + results[0].type.should eq Place::RemoteDebugCommandType::Settings + results[0].result.data.should eq JSON.parse(%({"app.title": "Panel"})) + + exec(:stop_logs, device_id).get + status["device:#{device_id}"]?.should be_nil + end + + it "should replace a result that is too large with an error" do + command_id = exec(:send_command, device_id, "status").get.not_nil!.as_s + exec(:command_result, device_id, command_id, {ok: true, data: "x" * 100_001}, device).get + + last = status["results:#{device_id}"].as_a.last + last["id"].should eq command_id + last["result"].should eq JSON.parse(%({"ok": false, "error": "Result is too large"})) + end + + it "should keep the last 3 uploads and remove the older uploads" do + ids = (1..4).map do |index| + command_id = exec(:send_command, device_id, "snapshot", {width: 800}).get.not_nil!.as_s + media = {mime: "image/png", width: 800, height: 450, frames: 1, duration: 0, bytes: 5, upload_id: "upload-#{index}"} + exec(:command_result, device_id, command_id, {ok: true, data: media}, device).get + command_id + end + + # `id`, `url` and `received_at` are set by the driver, so the model does not parse them + media = status["media:#{device_id}"].as_a + media.map(&.["upload_id"].as_s).should eq ["upload-2", "upload-3", "upload-4"] + media.map(&.["id"].as_s).should eq ids[1..] + media[0]["url"].should eq "/api/engine/v2/uploads/upload-2/url" + system(:StaffAPI)[:removed].should eq ["upload-1"] + end + + it "should not add commands that are not valid" do + expect_raises(Exception) { exec(:send_command, device_id, "navigate", {path: "x"}).get } + expect_raises(Exception) { exec(:send_command, device_id, "run_script").get } + expect_raises(Exception) { exec(:send_command, "not a device id", "status").get } + status["device:#{device_id}"]?.should be_nil + + command_id = exec(:send_command, device_id, "clear_cache").get.not_nil!.as_s + status["device:#{device_id}"]["commands"].should eq JSON.parse([{id: command_id, type: "clear_cache"}].to_json) + end +end From 5c75eac36349438a8ff3a4ce5e00aea864cfd7d4 Mon Sep 17 00:00:00 2001 From: Alex Sorafumo Date: Thu, 19 Nov 2026 22:51:16 +1100 Subject: [PATCH 2/3] feat(place/remote_debug): match latest remote debug interface - add `register` so fixed devices show in `devices` - remove `media:`; captures stay in `results:` - remove devices that do not call for 7 days, with their data - write `last_seen`-only changes to `devices` each minute - return nil from `send_command` when the device has 20 commands - remove the StaffAPI upload removal, which trusted the upload ID from the device Co-Authored-By: Claude Opus 5.5 --- drivers/place/remote_debug.cr | 93 ++++++++++++---------------- drivers/place/remote_debug_models.cr | 33 ---------- drivers/place/remote_debug_spec.cr | 39 ++++-------- 3 files changed, 53 insertions(+), 112 deletions(-) diff --git a/drivers/place/remote_debug.cr b/drivers/place/remote_debug.cr index 55c3e1a842..1e8bc5add2 100644 --- a/drivers/place/remote_debug.cr +++ b/drivers/place/remote_debug.cr @@ -4,7 +4,9 @@ require "./remote_debug_models" # Remote debugging of PlaceOS apps, such as Booking Panel, Control and Signage. # The apps find this driver from the `remote_debug` binding of the # organisation zone. A device reads its `device:` binding, sends logs with -# `post_logs` and sends command results with `command_result`. +# `post_logs` and sends command results with `command_result`. Fixed devices, +# such as panels and signage, also call `register` each 15 minutes so the admin +# views list them. # # Interface: https://github.com/PlaceOS/user-interfaces/blob/develop/docs/remote-debug.md # Device side: https://github.com/PlaceOS/user-interfaces/blob/develop/libs/common/src/lib/remote-debug.ts @@ -17,16 +19,13 @@ class Place::RemoteDebug < PlaceOS::Driver To turn it on, set the `remote_debug` binding of the organisation zone to `{"id": "", "module": "RemoteDebug"}`. - Add a StaffAPI module to this system so the driver can remove old screen - captures from the uploads. Only support users can request logs and send commands. + Only support users can request logs and send commands. DESC # Most entries that the driver keeps for each device MAX_DEVICE_ENTRIES = 5000 # Most command results that the driver keeps for each device MAX_DEVICE_RESULTS = 50 - # Most snapshots and animations that the driver keeps for each device - MAX_DEVICE_MEDIA = 3 # Most devices that the driver keeps data for, and most devices with a request or commands MAX_DEVICES = 1000 # Most commands for each device. The device reads the first 20 @@ -44,22 +43,25 @@ class Place::RemoteDebug < PlaceOS::Driver # Time before the driver removes a command that the device did not run COMMAND_TIMEOUT = 15.minutes # Most time between updates of `devices` when only `last_seen` changes - SEEN_INTERVAL_MS = 60 * 1000 - DEVICE_ID = /\A[A-Za-z0-9_-]{1,64}\z/ + SEEN_INTERVAL = 1.minute + # Time without a call before the driver removes a device and its data + STALE_AFTER = 7.days + DEVICE_ID = /\A[A-Za-z0-9_-]{1,64}\z/ # State of each device with a log request or commands @states = {} of String => RemoteDebugDeviceState # Devices that called the driver, by ID @devices = {} of String => RemoteDebugDevice - @devices_published_at = 0_i64 + # Whether `devices` has `last_seen` changes that are not written yet + @devices_changed = false @logs = {} of String => RemoteLogsView @results = {} of String => Array(RemoteDebugCommandResult) - @media = {} of String => Array(RemoteDebugMedia) # End timers of the active log requests, by device ID @expiry = {} of String => PlaceOS::Driver::Proxy::Scheduler::TaskWrapper def on_load self[:devices] = @devices + schedule.every(SEEN_INTERVAL) { sweep_devices } end # =================================== @@ -107,14 +109,15 @@ class Place::RemoteDebug < PlaceOS::Driver publish_state device_id end - # Adds a command for a device. Returns the command ID. + # Adds a command for a device. Returns the command ID, or `nil` when the + # device has `MAX_COMMANDS` commands already. # `options` holds the fields of the command, such as `keys`, `path` or `width` @[Security(Level::Support)] - def send_command(device_id : String, type : RemoteDebugCommandType, options : RemoteDebugCommandOptions? = nil) : String + def send_command(device_id : String, type : RemoteDebugCommandType, options : RemoteDebugCommandOptions? = nil) : String? check_device_id device_id command = build_command(type, options || RemoteDebugCommandOptions.new) count = @states[device_id]?.try(&.commands.size) || 0 - raise ArgumentError.new("device #{device_id} has #{MAX_COMMANDS} commands already") if count >= MAX_COMMANDS + return if count >= MAX_COMMANDS state_for(device_id).commands << command publish_state device_id schedule.in(COMMAND_TIMEOUT) { remove_command(device_id, command.id) } @@ -125,6 +128,11 @@ class Place::RemoteDebug < PlaceOS::Driver # Methods that the devices call # =================================== + # Adds or updates a fixed device in `devices`. Fixed devices call this each 15 minutes + def register(device : RemoteDebugDevice) : Nil + seen(device.id, device) if DEVICE_ID.matches?(device.id) + end + # Adds the entries of a device. Ignores calls for a request that is not active def post_logs(device_id : String, request_id : String, entries : Array(RemoteLogEntry), device : RemoteDebugDevice) : Nil logs = @states[device_id]?.try &.logs @@ -154,7 +162,6 @@ class Place::RemoteDebug < PlaceOS::Driver if result.to_json.size > MAX_RESULT_LENGTH result = RemoteDebugResult.new(ok: false, error: "Result is too large") end - add_media(device_id, command_id, result) if command.type.media? results = @results[device_id]? || [] of RemoteDebugCommandResult results << RemoteDebugCommandResult.new(command.id, command.type, result, now_ms) @@ -234,66 +241,48 @@ class Place::RemoteDebug < PlaceOS::Driver publish_state(device_id) if commands.size != count end - # Keeps the upload of a `snapshot` or `animation` result in `media:`. - # Removes the uploads that the driver no longer keeps - protected def add_media(device_id : String, command_id : String, result : RemoteDebugResult) : Nil - data = result.data - return unless result.ok && data - media = begin - RemoteDebugMedia.from_json(data.to_json) - rescue error : JSON::Error - logger.warn { "invalid media from #{device_id}: #{error.message}" } - return - end - media.id = command_id - media.url = "/api/engine/v2/uploads/#{URI.encode_path_segment(media.upload_id)}/url" - media.received_at = now_ms - - list = @media[device_id]? || [] of RemoteDebugMedia - list << media - list.shift(list.size - MAX_DEVICE_MEDIA).each { |old| remove_upload old.upload_id } if list.size > MAX_DEVICE_MEDIA - self["media:#{device_id}"] = @media[device_id] = list - end - - # Removes an upload with the StaffAPI module of this system, if there is one - protected def remove_upload(upload_id : String) : Nil - return unless system.exists?(:StaffAPI) - system[:StaffAPI].upload_remove(upload_id) - rescue error - logger.warn(exception: error) { "failed to remove upload #{upload_id}" } - end - # Adds or updates a device in `devices`. Keeps the `MAX_DEVICES` devices # that were seen last, and removes the data of the other devices. - # Writes `devices` when a device changes, or once each minute when only `last_seen` changes + # Writes `devices` when a device changes. `sweep_devices` writes the + # changes of `last_seen` only, so 1,000 fixed devices do not write the list each second protected def seen(device_id : String, device : RemoteDebugDevice) : Nil - now = now_ms previous = @devices[device_id]? # Compare all the fields but `last_seen` device.last_seen = previous.try &.last_seen changed = previous != device - device.last_seen = now + device.last_seen = now_ms @devices[device_id] = device if @devices.size > MAX_DEVICES - oldest = @devices.min_by { |_id, item| item.last_seen || 0_i64 }[0] - forget oldest + forget @devices.min_by { |_id, item| item.last_seen || 0_i64 }[0] changed = true end - return unless changed || now - @devices_published_at >= SEEN_INTERVAL_MS - @devices_published_at = now + changed ? publish_devices : (@devices_changed = true) + end + + protected def publish_devices : Nil + @devices_changed = false self[:devices] = @devices end - # Removes the logs, results and uploads of a device + # Removes the devices that did not call for `STALE_AFTER`, and writes the + # changes of `last_seen`. Runs each `SEEN_INTERVAL` + protected def sweep_devices : Nil + stale = now_ms - STALE_AFTER.total_milliseconds.to_i64 + @devices.select { |_id, item| (item.last_seen || 0_i64) < stale }.each_key do |device_id| + forget device_id + @devices_changed = true + end + publish_devices if @devices_changed + end + + # Removes the logs and results of a device protected def forget(device_id : String) : Nil @devices.delete(device_id) @logs.delete(device_id) @results.delete(device_id) - @media.delete(device_id).try &.each { |media| remove_upload media.upload_id } self["logs:#{device_id}"] = nil self["results:#{device_id}"] = nil - self["media:#{device_id}"] = nil end end diff --git a/drivers/place/remote_debug_models.cr b/drivers/place/remote_debug_models.cr index a20641a78e..271ef22c1a 100644 --- a/drivers/place/remote_debug_models.cr +++ b/drivers/place/remote_debug_models.cr @@ -104,11 +104,6 @@ module Place ClearCache Snapshot Animation - - # Whether the result of this command holds a `RemoteDebugMedia` - def media? - snapshot? || animation? - end end enum RemoteDebugMediaFormat @@ -180,34 +175,6 @@ module Place end end - # Data of the `snapshot` and `animation` results, and an item of the - # `media:` binding. The device sets the fields up to `upload_id`, the - # driver sets the others. - struct RemoteDebugMedia - include JSON::Serializable - - # `image/webp` or `image/png` for a snapshot, `image/webp` or `image/gif` for an animation - getter mime : String - getter width : Int32 - getter height : Int32 - getter frames : Int32 - # Milliseconds of the animation. `0` for a snapshot - getter duration : Int32 - # Bytes of the file - getter bytes : Int64 - # ID of the PlaceOS upload - getter upload_id : String - - # ID of the command - @[JSON::Field(ignore_deserialize: true)] - property id : String? = nil - # Address of the upload - @[JSON::Field(ignore_deserialize: true)] - property url : String? = nil - @[JSON::Field(ignore_deserialize: true)] - property received_at : Int64? = nil - end - # Item of the `results:` binding record RemoteDebugCommandResult, id : String, diff --git a/drivers/place/remote_debug_spec.cr b/drivers/place/remote_debug_spec.cr index c8e69df9a7..f48d2d1fe5 100644 --- a/drivers/place/remote_debug_spec.cr +++ b/drivers/place/remote_debug_spec.cr @@ -1,20 +1,7 @@ require "placeos-driver/spec" require "./remote_debug_models" -# :nodoc: -class StaffAPIMock < DriverSpecs::MockDriver - def upload_remove(upload_id : String) : Bool - removed = self[:removed]?.try(&.as_a.map(&.as_s)) || [] of String - self[:removed] = removed << upload_id - true - end -end - DriverSpecs.mock_driver "Place::RemoteDebug" do - system({ - StaffAPI: {StaffAPIMock}, - }) - device_id = "DEV-TEST00000001" device = { id: device_id, @@ -99,20 +86,14 @@ DriverSpecs.mock_driver "Place::RemoteDebug" do last["result"].should eq JSON.parse(%({"ok": false, "error": "Result is too large"})) end - it "should keep the last 3 uploads and remove the older uploads" do - ids = (1..4).map do |index| - command_id = exec(:send_command, device_id, "snapshot", {width: 800}).get.not_nil!.as_s - media = {mime: "image/png", width: 800, height: 450, frames: 1, duration: 0, bytes: 5, upload_id: "upload-#{index}"} - exec(:command_result, device_id, command_id, {ok: true, data: media}, device).get - command_id - end - - # `id`, `url` and `received_at` are set by the driver, so the model does not parse them - media = status["media:#{device_id}"].as_a - media.map(&.["upload_id"].as_s).should eq ["upload-2", "upload-3", "upload-4"] - media.map(&.["id"].as_s).should eq ids[1..] - media[0]["url"].should eq "/api/engine/v2/uploads/upload-2/url" - system(:StaffAPI)[:removed].should eq ["upload-1"] + it "should list a device that registers" do + fixed = device.merge(id: "DEV-FIXED0001", app: "signage") + exec(:register, fixed).get + status["devices"]["DEV-FIXED0001"]["app"].should eq "signage" + status["devices"]["DEV-FIXED0001"]["last_seen"].as_i64.should be > 0 + + exec(:register, fixed.merge(id: "not a device id")).get + status["devices"]["not a device id"]?.should be_nil end it "should not add commands that are not valid" do @@ -123,5 +104,9 @@ DriverSpecs.mock_driver "Place::RemoteDebug" do command_id = exec(:send_command, device_id, "clear_cache").get.not_nil!.as_s status["device:#{device_id}"]["commands"].should eq JSON.parse([{id: command_id, type: "clear_cache"}].to_json) + + # The admin views show that the list is full when the result is `null` + 19.times { exec(:send_command, device_id, "status").get.not_nil!.as_s } + exec(:send_command, device_id, "status").get.try(&.raw).should be_nil end end From 6c29644da7c60a11f04633a7c20a996f6880cf5c Mon Sep 17 00:00:00 2001 From: Alex Sorafumo Date: Sun, 22 Nov 2026 02:09:31 +1100 Subject: [PATCH 3/3] feat(place/remote_debug): owners, 7-day limit and 8,000 entries - keep 8,000 entries per device, so `before` gets all device entries - keep the first user that calls for each device ID, and ignore device calls for that ID from other users - remove each entry and result 7 days after the driver gets it - give each expiry timer a token, so an old timer cannot end a newer request - check the request and owner again after writes that let other calls run - ignore late batches from an app session before a reload - limit device text fields, settings keys and navigate paths - cancel command timeouts when the result arrives Co-Authored-By: Claude Opus 5.5 --- drivers/place/remote_debug.cr | 210 +++++++++++++++++++++------ drivers/place/remote_debug_models.cr | 18 ++- drivers/place/remote_debug_spec.cr | 25 +++- 3 files changed, 204 insertions(+), 49 deletions(-) diff --git a/drivers/place/remote_debug.cr b/drivers/place/remote_debug.cr index 1e8bc5add2..6c1a1962e1 100644 --- a/drivers/place/remote_debug.cr +++ b/drivers/place/remote_debug.cr @@ -19,19 +19,30 @@ class Place::RemoteDebug < PlaceOS::Driver To turn it on, set the `remote_debug` binding of the organisation zone to `{"id": "", "module": "RemoteDebug"}`. - Only support users can request logs and send commands. + Only support users can request logs and send commands. For the device + methods, the first user that calls for a device ID owns that ID. The + driver ignores device calls for that ID from other users until it + removes the device. + + The driver cannot limit who reads its bindings. The devices need access + to this system, so they can also read `devices`, `logs:` and + `results:`. DESC - # Most entries that the driver keeps for each device - MAX_DEVICE_ENTRIES = 5000 + # Most entries that the driver keeps for each device. The device keeps 8,000 + # entries, so `before` can get all of them + MAX_DEVICE_ENTRIES = 8000 # Most command results that the driver keeps for each device MAX_DEVICE_RESULTS = 50 # Most devices that the driver keeps data for, and most devices with a request or commands MAX_DEVICES = 1000 # Most commands for each device. The device reads the first 20 MAX_COMMANDS = 20 - # Most keys in a `settings` command - MAX_SETTINGS_KEYS = 50 + # Most keys in a `settings` command, and most characters in a key + MAX_SETTINGS_KEYS = 50 + MAX_SETTINGS_KEY = 200 + # Most characters in the path of a `navigate` command + MAX_PATH = 2000 # Most entries in one `post_logs` call. The device sends 100 maximum MAX_BATCH = 100 # Most characters in the JSON of a command result. The device sends 100,000 maximum @@ -44,7 +55,8 @@ class Place::RemoteDebug < PlaceOS::Driver COMMAND_TIMEOUT = 15.minutes # Most time between updates of `devices` when only `last_seen` changes SEEN_INTERVAL = 1.minute - # Time without a call before the driver removes a device and its data + # Time without a call before the driver removes a device and its data. Also + # the time that the driver keeps each entry and result after it gets them STALE_AFTER = 7.days DEVICE_ID = /\A[A-Za-z0-9_-]{1,64}\z/ @@ -54,10 +66,17 @@ class Place::RemoteDebug < PlaceOS::Driver @devices = {} of String => RemoteDebugDevice # Whether `devices` has `last_seen` changes that are not written yet @devices_changed = false + # User that first called for each device ID. Kept while the device is in + # `devices` or has a request or commands + @owners = {} of String => String @logs = {} of String => RemoteLogsView @results = {} of String => Array(RemoteDebugCommandResult) - # End timers of the active log requests, by device ID - @expiry = {} of String => PlaceOS::Driver::Proxy::Scheduler::TaskWrapper + # Token and end timer of the active log requests, by device ID. Each timer + # has a new token. The timer is `nil` until `schedule.in` returns + @expiry = {} of String => {Int64, PlaceOS::Driver::Proxy::Scheduler::TaskWrapper?} + @expiry_token = 0_i64 + # Timeout timers of the pending commands, by command ID + @command_timers = {} of String => PlaceOS::Driver::Proxy::Scheduler::TaskWrapper def on_load self[:devices] = @devices @@ -81,8 +100,9 @@ class Place::RemoteDebug < PlaceOS::Driver duration: options.duration.try(&.clamp(0, MAX_DURATION_MS)), ) state_for(device_id).logs = request - publish_state device_id + # Replace the timer before the write, as the write lets other calls run start_expiry device_id, request.duration + publish_state device_id request.id end @@ -94,15 +114,15 @@ class Place::RemoteDebug < PlaceOS::Driver # The device ignores the same level, so its duration does not start again return if state.nil? || logs.nil? || logs.level == level state.logs = RemoteLogRequest.new(logs.id, level, logs.history, logs.before, logs.duration) - publish_state device_id # The device starts the duration again when the level changes start_expiry device_id, logs.duration + publish_state device_id end # Ends the active log request of a device @[Security(Level::Support)] def stop_logs(device_id : String) : Nil - @expiry.delete(device_id).try &.cancel + cancel_expiry device_id state = @states[device_id]? return unless state && state.logs state.logs = nil @@ -119,8 +139,12 @@ class Place::RemoteDebug < PlaceOS::Driver count = @states[device_id]?.try(&.commands.size) || 0 return if count >= MAX_COMMANDS state_for(device_id).commands << command + @command_timers[command.id] = schedule.in(COMMAND_TIMEOUT) do + # Remove the timer first, so `take_command` does not cancel it + @command_timers.delete(command.id) + publish_state(device_id) if take_command(device_id, command.id) + end publish_state device_id - schedule.in(COMMAND_TIMEOUT) { remove_command(device_id, command.id) } command.id end @@ -133,11 +157,13 @@ class Place::RemoteDebug < PlaceOS::Driver seen(device.id, device) if DEVICE_ID.matches?(device.id) end - # Adds the entries of a device. Ignores calls for a request that is not active + # Adds the entries of a device. Ignores calls for a request that is not + # active, and calls from an app session before the last one def post_logs(device_id : String, request_id : String, entries : Array(RemoteLogEntry), device : RemoteDebugDevice) : Nil - logs = @states[device_id]?.try &.logs - return unless logs && logs.id == request_id && device.id == device_id - seen device_id, device + return unless device.id == device_id && accepts_logs?(device_id, request_id, device) + return unless seen(device_id, device) + # `seen` can write `devices`, which lets other calls run, so check again + return unless owned?(device_id) && accepts_logs?(device_id, request_id, device) view = @logs[device_id]? # A new `started_at` shows that the app reloaded and `seq` started again @@ -146,18 +172,30 @@ class Place::RemoteDebug < PlaceOS::Driver known = old.map(&.seq).to_set # The device sends a failed batch again, so skip the entries that we have added = entries.first(MAX_BATCH).reject { |entry| known.includes?(entry.seq) }.uniq!(&.seq) - return if same_session && added.empty? + if view && same_session && added.empty? + # Write the view again, so a retry repairs a write that failed. The + # write does nothing when the binding has this value + self["logs:#{device_id}"] = view + return + end - merged = (old + added).sort_by!(&.seq).last(MAX_DEVICE_ENTRIES) - self["logs:#{device_id}"] = @logs[device_id] = RemoteLogsView.new(device, merged) + received_at = now_ms + added.map! { |entry| entry.received_at = received_at; entry } + view = RemoteLogsView.new(device, (old + added).sort_by!(&.seq).last(MAX_DEVICE_ENTRIES)) + @logs[device_id] = view + self["logs:#{device_id}"] = view end # Keeps the result of a command and removes the command def command_result(device_id : String, command_id : String, result : RemoteDebugResult, device : RemoteDebugDevice) : Nil - command = @states[device_id]?.try &.commands.find(&.id.==(command_id)) - return unless command && device.id == device_id - seen device_id, device - remove_command device_id, command_id + pending = @states[device_id]?.try &.commands.any?(&.id.==(command_id)) + return unless pending && device.id == device_id + return unless seen(device_id, device) + # `seen` can write `devices`, which lets other calls run. Take the command + # after it, so a second call for the same command does nothing + return unless owned?(device_id) + command = take_command(device_id, command_id) + return unless command if result.to_json.size > MAX_RESULT_LENGTH result = RemoteDebugResult.new(ok: false, error: "Result is too large") @@ -165,7 +203,13 @@ class Place::RemoteDebug < PlaceOS::Driver results = @results[device_id]? || [] of RemoteDebugCommandResult results << RemoteDebugCommandResult.new(command.id, command.type, result, now_ms) - self["results:#{device_id}"] = @results[device_id] = results.last(MAX_DEVICE_RESULTS) + @results[device_id] = results = results.last(MAX_DEVICE_RESULTS) + begin + self["results:#{device_id}"] = results + ensure + # The command is not in memory now, so also remove it from the binding + publish_state device_id + end end # =================================== @@ -192,26 +236,43 @@ class Place::RemoteDebug < PlaceOS::Driver end end - # Writes the `device:` binding. An empty state is removed and shows as `null` + # Writes the `device:` binding. An empty state is removed and shows as + # `null`. Then the owner of a device that `forget` removed is also removed protected def publish_state(device_id : String) : Nil state = @states[device_id]? if state && state.empty? @states.delete(device_id) + @owners.delete(device_id) unless @devices.has_key?(device_id) state = nil end self["device:#{device_id}"] = state end - # Ends the log request of a device when its duration ends + # Ends the log request of a device when its duration ends. + # `schedule.in` can let other calls run, and its timer can run before it + # returns. So the token is kept before the call, and a timer only ends the + # request while its token is the current token protected def start_expiry(device_id : String, duration : Int32?) : Nil - @expiry.delete(device_id).try &.cancel - duration = (duration || DEFAULT_DURATION_MS).clamp(0, MAX_DURATION_MS) - @expiry[device_id] = schedule.in(duration.milliseconds) do - @expiry.delete(device_id) - stop_logs device_id + cancel_expiry device_id + token = @expiry_token += 1 + @expiry[device_id] = {token, nil} + timer = schedule.in((duration || DEFAULT_DURATION_MS).milliseconds) do + if @expiry[device_id]?.try(&.[0]) == token + @expiry.delete(device_id) + stop_logs device_id + end + end + if @expiry[device_id]?.try(&.[0]) == token + @expiry[device_id] = {token, timer} + else + timer.cancel end end + protected def cancel_expiry(device_id : String) : Nil + @expiry.delete(device_id).try(&.[1]).try(&.cancel) + end + protected def build_command(type : RemoteDebugCommandType, options : RemoteDebugCommandOptions) : RemoteDebugCommand id = new_id case type @@ -219,10 +280,12 @@ class Place::RemoteDebug < PlaceOS::Driver keys = options.keys raise ArgumentError.new("settings needs keys") unless keys raise ArgumentError.new("settings has #{MAX_SETTINGS_KEYS} keys maximum") if keys.size > MAX_SETTINGS_KEYS + raise ArgumentError.new("a settings key has #{MAX_SETTINGS_KEY} characters maximum") if keys.any?(&.size.>(MAX_SETTINGS_KEY)) RemoteDebugCommand.new(id, type, keys: keys) in .navigate? path = options.path raise ArgumentError.new("navigate needs a path that starts with /") unless path && path.starts_with?('/') + raise ArgumentError.new("navigate has a path of #{MAX_PATH} characters maximum") if path.size > MAX_PATH RemoteDebugCommand.new(id, type, path: path) in .snapshot? RemoteDebugCommand.new(id, type, width: options.width) @@ -233,19 +296,40 @@ class Place::RemoteDebug < PlaceOS::Driver end end - protected def remove_command(device_id : String, command_id : String) : Nil + # Whether the request is the active request of the device, and the call is + # not from an app session before the session of the kept entries + protected def accepts_logs?(device_id : String, request_id : String, device : RemoteDebugDevice) : Bool + return false unless @states[device_id]?.try(&.logs).try(&.id) == request_id + view = @logs[device_id]? + !(view && device.started_at < view.device.started_at) + end + + # Removes a command from the state, and stops its timeout. Does not write + # the state. Returns the command, or `nil` when it is not pending + protected def take_command(device_id : String, command_id : String) : RemoteDebugCommand? commands = @states[device_id]?.try &.commands - return unless commands - count = commands.size - commands.reject!(&.id.==(command_id)) - publish_state(device_id) if commands.size != count + index = commands.try &.index(&.id.==(command_id)) + return unless commands && index + @command_timers.delete(command_id).try &.cancel + commands.delete_at(index) + end + + # Whether the caller owns the device ID, and the device is in `devices` + protected def owned?(device_id : String) : Bool + @devices.has_key?(device_id) && @owners[device_id]? == (invoked_by_user_id || "") end - # Adds or updates a device in `devices`. Keeps the `MAX_DEVICES` devices - # that were seen last, and removes the data of the other devices. - # Writes `devices` when a device changes. `sweep_devices` writes the - # changes of `last_seen` only, so 1,000 fixed devices do not write the list each second - protected def seen(device_id : String, device : RemoteDebugDevice) : Nil + # Adds or updates a device in `devices`. Returns `false`, and does not + # change the device, when the caller is not the user that owns the device ID. + # Keeps the `MAX_DEVICES` devices that were seen last, and removes the data + # of the other devices. Writes `devices` when a device changes. + # `sweep_devices` writes the changes of `last_seen` only, so 1,000 fixed + # devices do not write the list each second + protected def seen(device_id : String, device : RemoteDebugDevice) : Bool + # Calls from other drivers have no user + user = invoked_by_user_id || "" + return false unless (@owners[device_id] ||= user) == user + previous = @devices[device_id]? # Compare all the fields but `last_seen` device.last_seen = previous.try &.last_seen @@ -259,6 +343,7 @@ class Place::RemoteDebug < PlaceOS::Driver end changed ? publish_devices : (@devices_changed = true) + true end protected def publish_devices : Nil @@ -266,20 +351,55 @@ class Place::RemoteDebug < PlaceOS::Driver self[:devices] = @devices end - # Removes the devices that did not call for `STALE_AFTER`, and writes the - # changes of `last_seen`. Runs each `SEEN_INTERVAL` + # Removes the devices that did not call for `STALE_AFTER`, and the entries + # and results that the driver got more than `STALE_AFTER` ago. Writes the + # changes of `last_seen`. Runs each `SEEN_INTERVAL`. + # Each write lets other calls run, so it reads each device again before it + # changes it protected def sweep_devices : Nil stale = now_ms - STALE_AFTER.total_milliseconds.to_i64 - @devices.select { |_id, item| (item.last_seen || 0_i64) < stale }.each_key do |device_id| + @devices.keys.each do |device_id| + device = @devices[device_id]? + next unless device && (device.last_seen || 0_i64) < stale forget device_id @devices_changed = true end publish_devices if @devices_changed + + @logs.keys.each do |device_id| + view = @logs[device_id]? + next unless view + count = view.entries.size + view.entries.reject!(&.received_at.<(stale)) + next if view.entries.size == count + if view.entries.empty? + @logs.delete(device_id) + self["logs:#{device_id}"] = nil + else + self["logs:#{device_id}"] = view + end + end + + @results.keys.each do |device_id| + results = @results[device_id]? + next unless results + count = results.size + results.reject!(&.received_at.<(stale)) + next if results.size == count + if results.empty? + @results.delete(device_id) + self["results:#{device_id}"] = nil + else + self["results:#{device_id}"] = results + end + end end - # Removes the logs and results of a device + # Removes a device and its logs and results. Keeps the owner while the + # device has a request or commands, so other users cannot send data for them protected def forget(device_id : String) : Nil @devices.delete(device_id) + @owners.delete(device_id) unless @states.has_key?(device_id) @logs.delete(device_id) @results.delete(device_id) self["logs:#{device_id}"] = nil diff --git a/drivers/place/remote_debug_models.cr b/drivers/place/remote_debug_models.cr index 271ef22c1a..633fd120e1 100644 --- a/drivers/place/remote_debug_models.cr +++ b/drivers/place/remote_debug_models.cr @@ -33,6 +33,10 @@ module Place getter source : RemoteLogSource getter message : String + # Unix epoch in milliseconds when the driver got the entry. Not in the bindings + @[JSON::Field(ignore: true)] + property received_at : Int64 = 0_i64 + def initialize(@seq, @timestamp, @level, @source, @message) end @@ -45,6 +49,9 @@ module Place struct RemoteDebugDevice include JSON::Serializable + # Most characters in each text field. The driver cuts longer values + MAX_TEXT = 500 + getter id : String # Short name of the app getter app : String @@ -63,13 +70,22 @@ module Place def initialize(@id, @app, @version, @build, @url, @user_agent, @started_at) end + + def after_initialize + @app = @app[0, MAX_TEXT] + @version = @version[0, MAX_TEXT] + @build = @build[0, MAX_TEXT] + @url = @url[0, MAX_TEXT] + @user_agent = @user_agent[0, MAX_TEXT] + end end # Log request in the `device:` binding struct RemoteLogRequest include JSON::Serializable - # Unique ID of the request. A device applies each ID one time + # Unique ID of the request. A new ID starts a new request on the device. + # `set_level` keeps the ID and changes only the level getter id : String getter level : RemoteLogLevel # Most buffered entries to send first. `0` sends only new entries diff --git a/drivers/place/remote_debug_spec.cr b/drivers/place/remote_debug_spec.cr index f48d2d1fe5..4e2bcea2b6 100644 --- a/drivers/place/remote_debug_spec.cr +++ b/drivers/place/remote_debug_spec.cr @@ -44,9 +44,17 @@ DriverSpecs.mock_driver "Place::RemoteDebug" do exec(:post_logs, device_id, "old-request", [entry.call(9)], device).get status["logs:#{device_id}"]?.should be_nil - exec(:post_logs, device_id, id, [entry.call(2), entry.call(1)], device).get - exec(:post_logs, device_id, id, [entry.call(2), entry.call(3)], device).get - status["logs:#{device_id}"]["entries"].as_a.map(&.["seq"].as_i).should eq [1, 2, 3] + exec(:post_logs, device_id, id, [entry.call(3), entry.call(2)], device).get + exec(:post_logs, device_id, id, [entry.call(3), entry.call(4)], device).get + # Only the user that first called for the device ID can add entries + exec(:post_logs, device_id, id, [entry.call(5)], device, user_id: "user-2").get + status["logs:#{device_id}"]["entries"].as_a.map(&.["seq"].as_i).should eq [2, 3, 4] + + # A request for older entries adds them before the entries that the driver has + id = exec(:request_logs, device_id, "info", {before: 2}).get.not_nil!.as_s + status["device:#{device_id}"]["logs"]["before"].should eq 2 + exec(:post_logs, device_id, id, [entry.call(1)], device).get + status["logs:#{device_id}"]["entries"].as_a.map(&.["seq"].as_i).should eq [1, 2, 3, 4] reloaded = device.merge(started_at: 2000) exec(:post_logs, device_id, id, [entry.call(1)], reloaded).get @@ -54,6 +62,10 @@ DriverSpecs.mock_driver "Place::RemoteDebug" do status["devices"][device_id]["started_at"].should eq 2000 status["devices"][device_id]["last_seen"].as_i64.should be > 0 + # A late batch from the app session before the reload does not replace the new entries + exec(:post_logs, device_id, id, [entry.call(5)], device).get + status["logs:#{device_id}"]["entries"].as_a.map(&.["seq"].as_i).should eq [1] + exec(:stop_logs, device_id).get end @@ -64,6 +76,9 @@ DriverSpecs.mock_driver "Place::RemoteDebug" do status["device:#{device_id}"]["commands"].should eq JSON.parse([{id: command_id, type: "settings", keys: ["app.title"]}].to_json) result = {ok: true, data: {"app.title" => "Panel"}} + exec(:command_result, device_id, command_id, result, device, user_id: "user-2").get + status["device:#{device_id}"]["commands"].as_a.size.should eq 1 + exec(:command_result, device_id, command_id, result, device).get status["device:#{device_id}"]["commands"].as_a.should be_empty @@ -94,6 +109,10 @@ DriverSpecs.mock_driver "Place::RemoteDebug" do exec(:register, fixed.merge(id: "not a device id")).get status["devices"]["not a device id"]?.should be_nil + + # Only the user that first called for the device ID can change it + exec(:register, fixed.merge(app: "other"), user_id: "user-2").get + status["devices"]["DEV-FIXED0001"]["app"].should eq "signage" end it "should not add commands that are not valid" do