Add on-demand hydrated DDOP snapshots for debugging

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Compile
         run: |
           mkdir build
-          cmake -S . -B build -DBUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF -DAOG_TC_VALIDATE_IOP=OFF -Wno-dev
+          cmake -S . -B build -DBUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF -DAOG_TC_VALIDATE_IOP=OFF -DAOG_TC_BUILD_TESTS=OFF -Wno-dev
           cmake --build build --config Release --target package
       - name: 'Upload Windows Installer'
         uses: actions/upload-artifact@v4
@@ -52,7 +52,7 @@ jobs:
           sudo apt-get install -y --no-install-recommends \
             build-essential cmake ninja-build
       - name: Configure (CMake)
-        run: cmake -S . -B build -G Ninja -DBUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF -DAOG_TC_VALIDATE_IOP=OFF -DCMAKE_BUILD_TYPE=Release -Wno-dev
+        run: cmake -S . -B build -G Ninja -DBUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF -DAOG_TC_VALIDATE_IOP=OFF -DAOG_TC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Wno-dev
       - name: Build
         run: cmake --build build --config Release
       - name: Stage tarball

.github/workflows/validate-iop.yml

@@ -34,7 +34,7 @@ jobs:
           -Wno-dev
 
       - name: Build
-        run: cmake --build build --config Release --target iop_validator
+        run: cmake --build build --config Release --target iop_validator ddop_hydration_test
 
-      - name: Validate object pool
+      - name: Validate object pool and run unit tests
         run: ctest --test-dir build --output-on-failure

CMakeLists.txt

@@ -146,6 +146,28 @@ endif()
 
 install(TARGETS ${PROJECT_NAME} RUNTIME DESTINATION bin COMPONENT applications)
 
+option(AOG_TC_BUILD_TESTS "Build and register the unit tests" ON)
+if(AOG_TC_BUILD_TESTS)
+  enable_testing()
+  add_executable(
+    ddop_hydration_test
+    ${CMAKE_CURRENT_LIST_DIR}/tools/ddop_hydration_test.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/ddop_hydration.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/settings.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/logging.cpp)
+  target_compile_features(ddop_hydration_test PRIVATE cxx_std_20)
+  set_target_properties(ddop_hydration_test PROPERTIES CXX_EXTENSIONS OFF)
+  target_include_directories(ddop_hydration_test
+                             PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include)
+  target_compile_definitions(
+    ddop_hydration_test PRIVATE PROJECT_VERSION="${PROJECT_VERSION}"
+                                PROJECT_NAME="${PROJECT_NAME}")
+  target_link_libraries(
+    ddop_hydration_test PRIVATE isobus::Isobus isobus::Utility
+                                nlohmann_json::nlohmann_json)
+  add_test(NAME ddop_hydration_snapshot COMMAND ddop_hydration_test)
+endif()
+
 option(AOG_TC_VALIDATE_IOP "Build and register the object pool validator test"
        ON)
 if(AOG_TC_VALIDATE_IOP)

docs/CONCURRENCY.md

@@ -57,6 +57,12 @@ registration point itself defers to an `update()` call, or invokes the listener/
   take `clientsMutex` at its top, even though (per the note above) the `TaskControllerServer`
   overrides may turn out to already be main-thread-only — the lock is cheap insurance and keeps
   every entry point consistent regardless of how the library's internals might change.
+- **Hydrated DDOP snapshot state** (`ClientState`'s shadow values, `MyTCServer::pendingHydration`) —
+  also guarded by `clientsMutex`. `on_value_command()` records values; `begin_hydration_snapshot()`
+  and `poll_hydration_snapshot()` run from `Application::update()`. The snapshot wait is polled,
+  never blocked on: `tcServer->update()` runs in the same main loop, and blocking it would stall the
+  responses being waited for. `poll_hydration_snapshot()` copies what it needs and writes files
+  after releasing the lock. The VT button listener only sets an `std::atomic<bool>`.
 
 ## What's *not* protected yet — known gap
 

include/app.hpp

@@ -8,6 +8,7 @@
  */
 #pragma once
 
+#include <atomic>
 #include <boost/asio.hpp>
 #include <cstdint>
 #include <map>
@@ -68,6 +69,7 @@ class Application
 	void sync_vt_config_once();
 	void update_vt_section_map();
 	void update_vt_status_strings(bool aogConnected);
+	void update_hydration_snapshot();
 
 	void send_vt_string_if_changed(std::uint16_t objectID, const std::string &value);
 	void send_hardware_message(const std::string &text, std::uint8_t duration, std::uint8_t color);
@@ -96,6 +98,7 @@ class Application
 	bool vtWasConnected = false;
 	bool vtConnectionWarningLogged = false;
 	bool vtCapabilitiesLogged = false;
+	std::atomic<bool> hydrationSnapshotRequested{ false }; ///< Set by the VT button listener, handled in update()
 	std::uint8_t nmea2000SequenceIdentifier = 0;
 	std::uint32_t lastJ1939SpeedTransmit = 0;
 	std::uint32_t lastTCStatusTransmit = 0;

include/ddop_hydration.hpp

@@ -0,0 +1,123 @@
+/**
+ * @brief On-demand "hydrated" DDOP snapshots for debugging and visualization
+ *
+ * A client's canonical DDOP describes structure only: DeviceProcessData objects carry no value,
+ * those arrive later as process data value commands. A hydrated snapshot is a derived copy of the
+ * canonical pool in which hydratable DeviceProcessData objects are replaced by DeviceProperty
+ * objects (same object ID, DDI, designator and presentation) holding the latest known value, so
+ * external tools such as AgIsoDDOPGenerator can show those values.
+ *
+ * Snapshots are never uploaded to a TC and never replace or modify the canonical pool.
+ */
+
+#pragma once
+
+#include "isobus/isobus/can_NAME.hpp"
+#include "isobus/isobus/isobus_device_descriptor_object_pool.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace ddop_hydration
+{
+	/// @brief How long a snapshot waits for answers to its value requests
+	constexpr std::uint32_t REQUEST_WAIT_MS = 10000;
+
+	/// @brief DDIs that are never hydrated: totals, setpoints and transient state such as work states,
+	/// section control state and actual rates.
+	bool is_always_excluded(std::uint16_t ddi);
+
+	/// @brief Whether a pool object is part of a snapshot, decided from the pool alone: every DeviceProperty,
+	/// and every DeviceProcessData that reports on change, is not a total and is not always excluded.
+	bool is_hydratable(const isobus::task_controller_object::Object &object);
+
+	/// @brief The latest process data value reported by a client for one object
+	struct ShadowValue
+	{
+		std::int32_t value = 0;
+		std::uint32_t timestamp_ms = 0;
+	};
+
+	/// @brief Latest reported process data values per object ID for one client.
+	/// Not thread-safe on its own; MyTCServer guards it with clientsMutex.
+	class ShadowValueStore
+	{
+	public:
+		void record(std::uint16_t objectID, std::int32_t value);
+		bool try_get(std::uint16_t objectID, ShadowValue &shadowValue) const;
+
+	private:
+		std::map<std::uint16_t, ShadowValue> values;
+	};
+
+	/// @brief Maps the (DDI, element number) addressing of process data messages to DDOP object IDs
+	class ProcessDataIndex
+	{
+	public:
+		void build(isobus::DeviceDescriptorObjectPool &pool);
+		bool try_get_object_id(std::uint16_t ddi, std::uint16_t elementNumber, std::uint16_t &objectID) const;
+		bool try_get_element_number(std::uint16_t objectID, std::uint16_t &elementNumber) const;
+
+	private:
+		std::map<std::pair<std::uint16_t, std::uint16_t>, std::uint16_t> ddiAndElementToObjectID;
+		std::map<std::uint16_t, std::uint16_t> objectIDToElementNumber;
+	};
+
+	/// @brief Where a snapshot object's value came from
+	enum class ValueSource : std::uint8_t
+	{
+		Pool, ///< DeviceProperty, value taken from the canonical pool as-is
+		Live, ///< Shadow value that was already known when the snapshot was requested
+		Requested, ///< Value arrived after an on-demand value request
+		NoResponse, ///< Value was requested but did not arrive in time
+		NotRequestable ///< DeviceProcessData that no device element references, so it cannot be addressed
+	};
+
+	const char *to_string(ValueSource source);
+
+	struct SnapshotEntry
+	{
+		static constexpr std::uint16_t NO_ELEMENT = 0xFFFF;
+
+		std::uint16_t objectID = 0;
+		std::uint16_t ddi = 0;
+		std::uint16_t elementNumber = NO_ELEMENT;
+		ValueSource source = ValueSource::NoResponse;
+		std::int32_t value = 0; ///< Only meaningful for Pool, Live and Requested
+	};
+
+	struct SnapshotInput
+	{
+		std::vector<std::vector<std::uint8_t>> canonicalPoolChunks; ///< The client's DDOP exactly as uploaded
+		std::uint8_t taskControllerCompatibilityLevel = 0;
+		std::uint64_t clientName = 0;
+		std::string fileStem; ///< Relative to the settings directory, e.g. "<NAME>/<label>"; the canonical pool is "<fileStem>.ddop"
+		std::vector<SnapshotEntry> entries;
+	};
+
+	struct SnapshotResult
+	{
+		bool success = false;
+		std::string ddopPath;
+		std::string metadataPath;
+		std::size_t patchedObjects = 0;
+		std::size_t missingValues = 0;
+		std::string error;
+	};
+
+	/// @brief Prefix added to the Device designator of a snapshot
+	constexpr const char *SNAPSHOT_DESIGNATOR_PREFIX = "SNAP ";
+	/// @brief Marker in snapshot file names: "<label>.SNAP-<timestamp>.ddop" next to "<label>.ddop"
+	constexpr const char *SNAPSHOT_FILENAME_MARKER = ".SNAP-";
+
+	/// @brief Clones the canonical pool, patches in the entries' values and writes the snapshot plus a JSON sidecar
+	SnapshotResult write_snapshot(const SnapshotInput &input);
+
+	/// @brief Whether a file is a hydrated snapshot (or its sidecar) rather than a canonical pool.
+	/// Anything that ever loads stored pools must skip these.
+	bool is_snapshot_file(const std::filesystem::path &path);
+} // namespace ddop_hydration

include/task_controller.hpp

@@ -9,15 +9,19 @@
 
 #pragma once
 
+#include "ddop_hydration.hpp"
 #include "isobus/isobus/isobus_data_dictionary.hpp"
 #include "isobus/isobus/isobus_device_descriptor_object_pool.hpp"
 #include "isobus/isobus/isobus_standard_data_description_indices.hpp"
 #include "isobus/isobus/isobus_task_controller_server.hpp"
 
 #include <cstdint>
 #include <map>
+#include <memory>
 #include <mutex>
+#include <optional>
 #include <queue>
+#include <string>
 
 constexpr std::uint8_t NUMBER_SECTIONS_PER_CONDENSED_MESSAGE = 16;
 
@@ -62,9 +66,19 @@ class ClientState
 	// Element work state management these act like master / override for actual sections
 	void set_element_work_state(std::uint16_t elementNumber, bool isWorking);
 	bool try_get_element_work_state(std::uint16_t elementNumber, bool &isWorking) const;
+	// Hydrated DDOP snapshot support, see ddop_hydration.hpp
+	void set_canonical_pool(std::vector<std::vector<std::uint8_t>> chunks, std::string fileStem);
+	const std::vector<std::vector<std::uint8_t>> &get_canonical_pool_chunks() const;
+	const std::string &get_canonical_file_stem() const;
+	ddop_hydration::ProcessDataIndex &get_process_data_index();
+	ddop_hydration::ShadowValueStore &get_shadow_values();
 
 private:
 	isobus::DeviceDescriptorObjectPool pool; ///< The device descriptor object pool (DDOP) for the TC
+	std::shared_ptr<const std::vector<std::vector<std::uint8_t>>> canonicalPoolChunks; ///< The DDOP exactly as uploaded, shared so get_clients() copies stay cheap
+	std::string canonicalFileStem; ///< "<NAME>/<label>"; the canonical pool is stored as "<stem>.ddop"
+	ddop_hydration::ProcessDataIndex processDataIndex; ///< (DDI, element number) -> object ID
+	ddop_hydration::ShadowValueStore shadowValues; ///< Latest reported process data values by object ID
 	bool areMeasurementCommandsSent = false; ///< Whether or not the measurement commands have been sent
 	std::map<isobus::DataDescriptionIndex, std::uint16_t> ddiToElementNumber; ///< Mapping of DDI to element number // TODO: better way to do this?
 
@@ -116,13 +130,37 @@ class MyTCServer : public isobus::TaskControllerServer
 	void update_section_states(std::vector<bool> &sectionStates);
 	void update_section_control_enabled(bool enabled);
 
+	enum class HydrationStartResult
+	{
+		Started,
+		AlreadyRunning,
+		UnknownClient
+	};
+
+	/// @brief Starts a hydrated DDOP snapshot for a client. Sends value requests for hydratable
+	/// objects without a known value; does not block. Finish it with poll_hydration_snapshot().
+	HydrationStartResult begin_hydration_snapshot(std::shared_ptr<isobus::ControlFunction> client);
+
+	/// @brief Call from the main loop. Once the request wait has elapsed, writes the snapshot and returns true.
+	/// @param[out] result The outcome of the finished snapshot, only set when this returns true
+	bool poll_hydration_snapshot(ddop_hydration::SnapshotResult &result);
+
 private:
+	struct PendingHydration
+	{
+		std::shared_ptr<isobus::ControlFunction> client;
+		std::vector<ddop_hydration::SnapshotEntry> entries;
+		std::uint32_t startedAt_ms = 0;
+		std::uint32_t wait_ms = 0;
+	};
+
 	void send_section_setpoint_states(std::shared_ptr<isobus::ControlFunction> client, std::uint8_t ddiOffset);
 	void send_section_control_state(std::shared_ptr<isobus::ControlFunction> client, bool enabled);
 	bool is_ddi_settable(std::shared_ptr<isobus::ControlFunction> client, std::uint16_t ddi);
 
 	std::map<std::shared_ptr<isobus::ControlFunction>, ClientState> clients;
 	std::map<std::shared_ptr<isobus::ControlFunction>, std::queue<std::vector<std::uint8_t>>> uploadedPools;
+	std::optional<PendingHydration> pendingHydration; ///< Guarded by clientsMutex
 
 	/// @brief Guards clients and uploadedPools.
 	///

readme.md

@@ -78,6 +78,16 @@ AOG-TaskController reads its configuration from a `settings.json` file located i
 }
 ```
 
+### Hydrated DDOP snapshots
+
+Many implement values, such as element offsets and working width, are not stored in the DDOP; the implement reports them as process data once it is running. The **Snapshot DDOP** button on the VT Implement page writes a copy of the connected implement's DDOP with those values filled in, for viewing in tools such as AgIsoDDOPGenerator.
+
+- The objects are picked from the DDOP itself: every DeviceProperty, and every DeviceProcessData that reports on change and is not a total, a setpoint, a work state, section control state or an actual rate.
+- Values that were already reported are used directly. For the rest, the TC sends value requests and waits 10 seconds for answers.
+- DeviceProcessData objects with a value are replaced by DeviceProperty objects with the same object ID, DDI and designator. The Device designator gets a `SNAP ` prefix and the structure label is changed.
+- The snapshot is stored next to the canonical pool as `<NAME>/<label>.SNAP-<timestamp>.ddop`, with a `.json` file that lists every object, its value and where the value came from.
+- Snapshots are for debugging only: never upload one to a TC. The canonical `<label>.ddop` is not changed.
+
 ### Virtual Terminal compatibility
 
 The VT object pool is embedded in the executable and automatically scales from its authored 480-pixel data mask and 80-pixel softkey designator to the connected terminal. It uses five virtual navigation softkeys. A VT with fewer than five physical keys must support softkey paging.

resources/AOG_TC.iop.h

@@ -20,6 +20,7 @@
 #define NavImplement                       5004
 #define NavDiagnostics                     5005
 #define NavConfig                          5006
+#define ImplementSnapshotButton            6000
 #define VTControlFunctionsStr              8000
 #define InputList_10000                   10000
 #define ConfigHydliftAuxN                 10001
@@ -56,6 +57,7 @@
 #define ImplementHint                     11207
 #define ImplementBoomOffsetLabel          11208
 #define ImplementBoomOffset               11209
+#define ImplementSnapshotLabel            11210
 #define DiagnosticsTitle                  11300
 #define DiagnosticsLabels                 11301
 #define DiagnosticsValues                 11302