Intigrate Migration into UnrealEngine Blueprints

.github/workflows/build.yml

@@ -200,6 +200,8 @@ jobs:
     name: Build Doxygen documentation
     needs: [compilation, build-python]
     runs-on: ubuntu-latest
+    env:
+      DOXYGEN_VERSION: "1.17.0"
     steps:
     - uses: actions/checkout@v4
       with:
@@ -215,13 +217,19 @@ jobs:
         pybind11-stubgen
         pdoc
         --user
+    - uses: actions/cache@v5
+      id: cache-doxygen
+      with:
+        path: doxygen-${{ env.DOXYGEN_VERSION }}.linux.bin.tar.gz
+        key: doxygen-${{ env.DOXYGEN_VERSION }}
+    - name: download Doxygen
+      if: steps.cache-doxygen.outputs.cache-hit != 'true'
+      run: wget https://www.doxygen.nl/files/doxygen-$DOXYGEN_VERSION.linux.bin.tar.gz
     - name: Install Doxygen
       run: |
         sudo apt-get install graphviz -y
-        wget https://www.doxygen.nl/files/doxygen-1.10.0.linux.bin.tar.gz
-        gunzip doxygen-*.tar.gz
-        tar xf doxygen-*.tar
-        cd doxygen-1.10.0/
+        tar xzf doxygen-$DOXYGEN_VERSION.linux.bin.tar.gz
+        cd doxygen-$DOXYGEN_VERSION/
         sudo make install
     - name: Setup cmake
       uses: jwlawson/actions-setup-cmake@v2

.gitignore

@@ -20,6 +20,11 @@ build/*
 bin/
 Bin/
 
+# python build artifacts
+dist/
+__pycache__/
+inkcpp_py.egg-info/
+
 # Visual Studio
 /out/build/
 .vs

inkcpp/CMakeLists.txt

@@ -83,7 +83,7 @@ endforeach()
 foreach(file IN LISTS PUBLIC_HEADERS)
 	get_filename_component(file "${file}" NAME)
 	configure_file("include/${file}"
-								 "${CMAKE_BINARY_DIR}/unreal/inkcpp/Source/inkcpp/Public/ink/${FILE}" COPYONLY)
+								 "${CMAKE_BINARY_DIR}/unreal/inkcpp/Source/inkcpp/Public/ink/${file}" COPYONLY)
 endforeach()
 foreach(file IN LISTS COLLECTION_SOURCES)
 	configure_file("${file}" "${CMAKE_BINARY_DIR}/unreal/inkcpp/Source/inkcpp/Private/ink/${file}"

inkcpp/array.h

@@ -346,13 +346,16 @@ class basic_restorable_array : public snapshot_interface
 	void clear(const T& value);
 
 	// snapshot interface
-	virtual bool                 can_be_migrated() const;
-	virtual size_t               snap(unsigned char* data, const snapper&) const;
-	virtual const unsigned char* snap_load(const unsigned char* data, const loader&);
+	bool                         can_be_migrated() const;
+	size_t                       snap(unsigned char* data, const snapper&) const;
+	virtual const unsigned char* snap_load(const unsigned char* data, const loader&) = 0;
 
 protected:
 	inline T* buffer() { return _array; }
 
+	const unsigned char* impl_snap_load_meta(const unsigned char* data);
+	const unsigned char* impl_snap_load_payload(const unsigned char* data);
+
 	void set_new_buffer(T* buffer, size_t capacity)
 	{
 		_array    = buffer;
@@ -455,6 +458,8 @@ inline void basic_restorable_array<T>::forget()
 		// Clear
 		_temp[i] = _null;
 	}
+
+	_saved = false;
 }
 
 template<typename T>
@@ -480,13 +485,18 @@ inline void basic_restorable_array<T>::clear(const T& value)
 template<typename T, size_t SIZE>
 class fixed_restorable_array : public basic_restorable_array<T>
 {
+	using base = basic_restorable_array<T>;
+
 public:
 	fixed_restorable_array(const T& initial, const T& nullValue)
 	    : basic_restorable_array<T>(_buffer, SIZE * 2, nullValue)
 	{
 		basic_restorable_array<T>::clear(initial);
 	}
 
+	const unsigned char*
+	    snap_load(const unsigned char* data, const snapshot_interface::loader&) override;
+
 private:
 	T _buffer[SIZE * 2];
 };
@@ -519,7 +529,7 @@ class allocated_restorable_array : public basic_restorable_array<T>
 		size_t new_capacity = 2 * n;
 		T*     new_buffer   = new T[new_capacity];
 		if (_buffer) {
-			for (size_t i = 0; i < base::capacity(); ++i) {
+			for (size_t i = 0; i < base::capacity() && i < n; ++i) {
 				new_buffer[i]     = _buffer[i];
 				// copy temp
 				new_buffer[i + n] = _buffer[i + base::capacity()];
@@ -543,6 +553,9 @@ class allocated_restorable_array : public basic_restorable_array<T>
 		}
 	}
 
+	const unsigned char*
+	    snap_load(const unsigned char* data, const snapshot_interface::loader&) override;
+
 private:
 	T  _initialValue;
 	T  _nullValue;
@@ -571,26 +584,55 @@ inline size_t basic_restorable_array<T>::snap(unsigned char* data, const snapper
 }
 
 template<typename T>
-inline const unsigned char*
-    basic_restorable_array<T>::snap_load(const unsigned char* data, const loader&)
+inline const unsigned char* basic_restorable_array<T>::impl_snap_load_meta(const unsigned char* data
+)
 {
 	auto ptr = data;
 	ptr      = snap_read(ptr, _saved);
 	ptr      = snap_read(ptr, _loaded_capacity);
-	if (buffer() == nullptr) {
-		static_cast<allocated_restorable_array<T>&>(*this).resize(_loaded_capacity);
-	}
-	inkAssert(
-	    _capacity >= _loaded_capacity,
-	    "New config does not allow for necessary size used by this snapshot!"
-	);
 	T null;
 	ptr = snap_read(ptr, null);
 	inkAssert(null == _null, "null value is different to snapshot!");
-	for (size_t i = 0; i < _loaded_capacity; ++i) {
+	return ptr;
+}
+
+template<typename T>
+inline const unsigned char*
+    basic_restorable_array<T>::impl_snap_load_payload(const unsigned char* data)
+{
+	inkAssert(
+	    capacity() >= loaded_capacity(),
+	    "New config does not allow for necessary size used by this snapshot!"
+	);
+	auto ptr = data;
+	for (size_t i = 0; i < loaded_capacity(); ++i) {
 		ptr = snap_read(ptr, _array[i]);
 		ptr = snap_read(ptr, _temp[i]);
 	}
 	return ptr;
 }
+
+template<typename T, size_t SIZE>
+inline const unsigned char* fixed_restorable_array<
+    T, SIZE>::snap_load(const unsigned char* data, const snapshot_interface::loader&)
+{
+	auto ptr = data;
+	ptr      = base::impl_snap_load_meta(ptr);
+	ptr      = base::impl_snap_load_payload(ptr);
+	return ptr;
+}
+
+template<typename T>
+inline const unsigned char* allocated_restorable_array<
+    T>::snap_load(const unsigned char* data, const snapshot_interface::loader&)
+{
+	auto ptr = data;
+	ptr      = base::impl_snap_load_meta(ptr);
+	if (base::buffer() == nullptr || base::capacity() < base::loaded_capacity()) {
+		resize(base::loaded_capacity());
+	}
+	ptr = base::impl_snap_load_payload(ptr);
+	return ptr;
+}
+
 } // namespace ink::runtime::internal

inkcpp/collections/restorable.h

@@ -228,7 +228,7 @@ class restorable : public snapshot_interface
 
 	// Forward iterate
 	template<typename CallbackMethod, typename IsNullPredicate>
-	void for_each(CallbackMethod callback, IsNullPredicate isNull) const
+	void for_each(CallbackMethod callback, IsNullPredicate isNull)
 	{
 		if (_pos == 0) {
 			return;

inkcpp/container_operations.cpp

@@ -40,6 +40,7 @@ void operation<Command::CHOICE_COUNT, value_type::none, void>::operator()(
     basic_eval_stack& stack, value* vals
 )
 {
+	( void ) vals;
 	stack.push(value{}.set<value_type::int32>(static_cast<int32_t>(_runner.num_choices())));
 }
 

inkcpp/globals_impl.cpp

@@ -26,33 +26,17 @@ globals_impl::globals_impl(const story_impl* story)
 	_visit_counts.resize(_num_containers);
 	if (_lists) {
 		// initialize static lists
-		init_static_list_flags();
+		_lists.init_static_list_flags(_owner->lists(), _variables);
 	}
 }
 
-void globals_impl::init_static_list_flags()
+void globals_impl::visit(uint32_t container_id, bool preserve_turns)
 {
-	const list_flag* flags = _owner->lists();
-	while (*flags != null_flag) {
-		list_table::list l = _lists.create_permament();
-		while (*flags != null_flag) {
-			list_flag flag = _lists.external_fvalue_to_internal(*flags);
-			_lists.add_inplace(l, flag);
-			++flags;
-		}
-		++flags;
-	}
-	for (const auto& flag : _lists.named_flags()) {
-		set_variable(
-		    hash_string(flag.name),
-		    value{}.set<value_type::list_flag>(list_flag{flag.flag.list_id, flag.flag.flag})
-		);
-	}
-}
-
-void globals_impl::visit(uint32_t container_id)
-{
-	_visit_counts.set(container_id, {_visit_counts[container_id].visits + 1, 0});
+	const int32_t existing_turns = _visit_counts[container_id].turns;
+	_visit_counts.set(
+	    container_id, {_visit_counts[container_id].visits + (preserve_turns ? 0 : 1),
+	                   preserve_turns ? existing_turns : 0}
+	);
 }
 
 uint32_t globals_impl::visits(uint32_t container_id) const
@@ -294,6 +278,9 @@ const unsigned char* globals_impl::snap_load(const unsigned char* ptr, const loa
 			_visit_counts.resize(_owner->num_containers());
 		}
 		_visit_counts.save();
+		for (size_t i = 0; i < old_capacity; ++i) {
+			_visit_counts.set(i, visit_count());
+		}
 	}
 
 	inkAssert(
@@ -302,8 +289,8 @@ const unsigned char* globals_impl::snap_load(const unsigned char* ptr, const loa
 	);
 	for (size_t i = 0; i < old_capacity; ++i) {
 		hash_t path;
-		ptr = snap_read(ptr, path);
-		container_t c_id;
+		ptr                      = snap_read(ptr, path);
+		container_t c_id         = ~0U;
 		ip_t        container_ip = _owner->find_offset_for(path);
 		bool        found        = container_ip != nullptr
 		          && _owner->find_container_id(
@@ -320,6 +307,7 @@ const unsigned char* globals_impl::snap_load(const unsigned char* ptr, const loa
 	}
 	if (loader.migratable) {
 		_visit_counts.forget();
+		_visit_counts.resize(_num_containers);
 	}
 	inkAssert(
 	    _num_containers == _visit_counts.capacity(),
@@ -331,15 +319,28 @@ const unsigned char* globals_impl::snap_load(const unsigned char* ptr, const loa
 	return ptr;
 }
 
-bool globals_impl::migrate_new_globals(globals_impl& new_globals, const char* list_metadata)
+bool globals_impl::migrate_new_globals(
+    const loader& loader, globals_impl& new_globals, const char* list_metadata
+)
 {
-	bool success
-	    = _variables.migrate(new_globals._variables) && ((! _lists) || _lists.migrate(list_metadata));
-	if (! success) {
+	if (! _variables.migrate(new_globals._variables)) {
 		return false;
 	}
+	if (_lists && ! loader.old_ref_table) {
+		if (! _lists.create_match_lut(
+		        list_metadata, loader.list_list_matches, loader.list_value_matches, loader.old_ref_table
+		    )) {
+			return false;
+		}
+	}
 	if (_lists) {
-		init_static_list_flags();
+		_lists.init_static_list_flags(_owner->lists(), _variables);
+		if (! _lists.migrate_variables(
+		        loader.list_old_new_map, loader.list_list_matches, loader.list_value_matches,
+		        *loader.old_ref_table, _variables
+		    )) {
+			return false;
+		}
 	}
 	return true;
 }

inkcpp/globals_impl.h

@@ -30,8 +30,6 @@ class globals_impl final
 {
 	friend snapshot_impl;
 
-	void init_static_list_flags();
-
 public:
 	size_t               snap(unsigned char* data, const snapper&) const;
 	const unsigned char* snap_load(const unsigned char* data, const loader&);
@@ -45,7 +43,9 @@ class globals_impl final
 	 * @param[in] list_metadata old list metadata to migrate list
 	 * the globals stored inside.
 	 */
-	bool                 migrate_new_globals(globals_impl& new_globals, const char* list_metadata);
+	bool                 migrate_new_globals(
+	                    const loader& loader, globals_impl& new_globals, const char* list_metadata
+	                );
 	// Initializes a new global store from the given story
 	globals_impl(const story_impl*);
 
@@ -64,8 +64,10 @@ class globals_impl final
 	void internal_observe(hash_t name, internal::callback_base* callback) override;
 
 public:
-	// Records a visit to a container
-	void visit(uint32_t container_id);
+	// Records a visit to a container.
+	// If preserve_turns is true the existing turns-since counter is kept intact
+	// (used during snapshot migration to avoid clobbering the restored value).
+	void visit(uint32_t container_id, bool preserve_turns = false);
 
 	// Checks the number of visits to a container
 	uint32_t visits(uint32_t container_id) const;

inkcpp/header.cpp

@@ -10,7 +10,7 @@
 namespace ink::internal
 {
 
-bool header::verify() const
+bool header::validate() const
 {
 	if (endian() == endian_types::none) {
 		inkFail("Header magic number was wrong!");

inkcpp/include/functional.h

@@ -204,8 +204,9 @@ class function : public function_base
 		if constexpr (traits::arity == 2) {
 			return is_same<
 			    const ink::runtime::value*, typename traits::template argument<1>::type>::value;
+		} else {
+			return false;
 		}
-		return false;
 	}
 
 	template<size_t... Is>

inkcpp/include/list.h

@@ -48,6 +48,7 @@ class list_interface
 	{
 	}
 
+	/** copy assigment operator. */
 	virtual list_interface& operator=(const list_interface&) = default;
 
 	virtual ~list_interface() {}
@@ -81,6 +82,9 @@ class list_interface
 		}
 
 	public:
+		/** copy constructor. */
+		iterator(const iterator&) = default;
+
 		/** contains flag data */
 		struct Flag {
 			const char* flag_name; ///< name of the flag
@@ -127,9 +131,8 @@ class list_interface
 #	pragma GCC diagnostic ignored "-Wunused-parameter"
 #else
 #	pragma warning(push)
-#	pragma warning(                                                                          \
-	    disable : 4100, justification : "non functional prototypes do not need the argument." \
-	)
+// non functional prototypes do not need the argument.
+#	pragma warning(disable : 4100)
 #endif
 
 	/** checks if a flag is contained in the list */
@@ -203,6 +206,7 @@ class list_interface
 
 	/** @private */
 	internal::list_table* _list_table;
+	/** @private */
 	int                   _list;
 };