diff --git a/flatdata-cpp/include/flatdata/ArchiveBuilder.h b/flatdata-cpp/include/flatdata/ArchiveBuilder.h index c84ba3df..b7fea5bc 100644 --- a/flatdata-cpp/include/flatdata/ArchiveBuilder.h +++ b/flatdata-cpp/include/flatdata/ArchiveBuilder.h @@ -52,6 +52,7 @@ class ArchiveBuilder const flatdata::ResourceStorage& storage( ) const; bool is_created( ) const; void check_created( ) const; + void clean_up_signature( flatdata::ResourceStorage& storage ); private: std::shared_ptr< flatdata::ResourceStorage > m_storage; diff --git a/flatdata-cpp/include/flatdata/FileResourceStorage.h b/flatdata-cpp/include/flatdata/FileResourceStorage.h index 699541d5..64501d7b 100644 --- a/flatdata-cpp/include/flatdata/FileResourceStorage.h +++ b/flatdata-cpp/include/flatdata/FileResourceStorage.h @@ -41,6 +41,7 @@ class FileResourceStorage : public ResourceStorage std::unique_ptr< ResourceStorage > directory( const char* key ) override; bool exists( const char* key ) override; + void remove( const char* key ) override; protected: std::shared_ptr< std::ostream > create_output_stream( const char* key ) override; @@ -150,4 +151,16 @@ FileResourceStorage::exists( const char* key ) return !ec; } +inline void +FileResourceStorage::remove( const char* key ) +{ + boost::system::error_code ec; + std::string file_path = get_path( key ); + boost::filesystem::remove( file_path.c_str( ), ec ); + if ( ec ) + { + throw std::runtime_error( std::string( "Failed to remove " ) + file_path ); + } +} + } // namespace flatdata diff --git a/flatdata-cpp/include/flatdata/MemoryResourceStorage.h b/flatdata-cpp/include/flatdata/MemoryResourceStorage.h index f8dbf72b..306368ae 100644 --- a/flatdata-cpp/include/flatdata/MemoryResourceStorage.h +++ b/flatdata-cpp/include/flatdata/MemoryResourceStorage.h @@ -32,6 +32,8 @@ class MemoryResourceStorage : public ResourceStorage public: static std::unique_ptr< MemoryResourceStorage > create( ); bool exists( const char* key ) override; + void remove( const char* key ) override; + std::unique_ptr< ResourceStorage > create_directory( const char* key ) override; std::unique_ptr< ResourceStorage > directory( const char* key ) override; @@ -259,6 +261,25 @@ MemoryResourceStorage::exists( const char* key ) || helpers::prefix_exists( m_storage->streams, path + "/" ); } +inline void +MemoryResourceStorage::remove( const char* key ) +{ + const auto path = get_path( key ); + std::lock_guard< std::mutex > lock( m_storage_mutex ); + + m_storage->streams.erase( path ); + auto found_resource = m_storage->resources.find( path ); + if ( found_resource != m_storage->resources.end( ) ) + { +#ifdef DEBUG_DATA_ACCESS_STATISTICS + auto& string = found_resource->second; + DebugDataAccessStatistics::deregister_mapping( MemoryDescriptor( + reinterpret_cast< const unsigned char* >( string->c_str( ) ), string->size( ) ) ); +#endif + m_storage->resources.erase( found_resource ); + } +} + template < typename ResourceSerializer > std::string MemoryResourceStorage::dump_resources( bool dump_schemas, ResourceSerializer&& f ) const diff --git a/flatdata-cpp/include/flatdata/ResourceStorage.h b/flatdata-cpp/include/flatdata/ResourceStorage.h index 2085167f..5aefa2c9 100644 --- a/flatdata-cpp/include/flatdata/ResourceStorage.h +++ b/flatdata-cpp/include/flatdata/ResourceStorage.h @@ -95,6 +95,11 @@ class ResourceStorage : private boost::noncopyable * @brief Check if resource exists */ virtual bool exists( const char* key ) = 0; + virtual void remove( const char* key ) = 0; + + void remove_resource( const char* resource_name, const char* expected_schema ); + void remove_multivector_index_resource( const char* resource_name, + const char* expected_schema ); protected: /** diff --git a/flatdata-cpp/include/flatdata/TarArchiveResourceStorage.h b/flatdata-cpp/include/flatdata/TarArchiveResourceStorage.h index 870fc0a5..9e31b031 100644 --- a/flatdata-cpp/include/flatdata/TarArchiveResourceStorage.h +++ b/flatdata-cpp/include/flatdata/TarArchiveResourceStorage.h @@ -32,6 +32,7 @@ class TarArchiveResourceStorage : public ResourceStorage std::unique_ptr< ResourceStorage > create_directory( const char* key ) override; std::unique_ptr< ResourceStorage > directory( const char* key ) override; bool exists( const char* key ) override; + void remove( const char* key ) override; protected: std::shared_ptr< std::ostream > create_output_stream( const char* key ) override; @@ -123,4 +124,11 @@ TarArchiveResourceStorage::exists( const char* key ) return m_storage->read( get_path( key ).c_str( ) ).data( ) != nullptr; } +inline void +TarArchiveResourceStorage::remove( const char* /* key */ ) +{ + // Removing parts of TAR files is not supported + throw std::runtime_error( "Removing from TAR files is not supported" ); +} + } // namespace flatdata diff --git a/flatdata-cpp/include/flatdata/internal/ResourceStorage.inl b/flatdata-cpp/include/flatdata/internal/ResourceStorage.inl index 0520da23..a8be4cdd 100644 --- a/flatdata-cpp/include/flatdata/internal/ResourceStorage.inl +++ b/flatdata-cpp/include/flatdata/internal/ResourceStorage.inl @@ -207,4 +207,44 @@ ResourceStorage::create_multi_vector( const char* resource_name, const char* sch internal::ResourceReader( this, resource_name, schema ) ) ); } +inline void +ResourceStorage::remove_resource( const char* resource_name, const char* expected_schema ) +{ + const std::string schema_name = std::string( resource_name ) + ".schema"; + const bool resource_exists = exists( resource_name ); + const bool schema_exists = exists( schema_name.c_str( ) ); + + if ( !resource_exists && !schema_exists ) + { + return; + } + + if ( resource_exists != schema_exists ) + { + throw std::runtime_error( std::string( "Resource or schema missing for " ) + + resource_name ); + } + + auto schema = read_resource( ( schema_name.c_str( ) ) ); + std::string stored_schema( reinterpret_cast< const char* >( schema.data( ) ), + schema.size_in_bytes( ) ); + + if ( stored_schema != expected_schema ) + { + throw std::runtime_error( std::string( "unexpected schema for " ) + resource_name ); + } + remove( resource_name ); + remove( schema_name.c_str( ) ); +} + +inline void +ResourceStorage::remove_multivector_index_resource( const char* resource_name, + const char* expected_schema ) +{ + std::string index_name = std::string( resource_name ) + internal::INDEX_SUFFIX; + std::string index_schema = internal::multivector_index_schema( expected_schema ); + + remove_resource( index_name.c_str( ), index_schema.c_str( ) ); +} + } // namespace flatdata diff --git a/flatdata-cpp/src/ArchiveBuilder.cpp b/flatdata-cpp/src/ArchiveBuilder.cpp index 4333b5d9..cbd3f9d3 100644 --- a/flatdata-cpp/src/ArchiveBuilder.cpp +++ b/flatdata-cpp/src/ArchiveBuilder.cpp @@ -67,4 +67,11 @@ ArchiveBuilder::initialize( ) return static_cast< bool >( signature ); } +void +ArchiveBuilder::clean_up_signature( flatdata::ResourceStorage& storage ) +{ + const auto signature_name = internal::signature_name( name( ) ); + storage.remove_resource( signature_name.c_str( ), schema( ) ); +} + } // namespace flatdata diff --git a/flatdata-cpp/test/ResourceStorageTest.cpp b/flatdata-cpp/test/ResourceStorageTest.cpp index 0b5c148d..6be503af 100644 --- a/flatdata-cpp/test/ResourceStorageTest.cpp +++ b/flatdata-cpp/test/ResourceStorageTest.cpp @@ -81,5 +81,28 @@ TEST_CASE( "schema is checked for written vector", "[ResourceStorage]" ) ( a->read< ArrayView< AStruct > >( resource_name, correct_schema ).is_initialized( ) ) ); } +TEST_CASE( "resource doesn't exist after removing it", "[ResourceStorage]" ) +{ + std::unique_ptr< ResourceStorage > a = MemoryResourceStorage::create( ); + auto create = [ & ] { + auto v = a->create_external_vector< AStruct >( resource_name, correct_schema ); + v.close( ); + }; + REQUIRE_NOTHROW( create( ) ); + + REQUIRE_FALSE( + ( a->read< ArrayView< AStruct > >( resource_name, incorrect_schema ).is_initialized( ) ) ); + REQUIRE( + ( a->read< ArrayView< AStruct > >( resource_name, correct_schema ).is_initialized( ) ) ); + + std::string schema_name = std::string( resource_name ) + ".schema"; + REQUIRE( a->exists( resource_name ) ); + REQUIRE( a->exists( schema_name.c_str( ) ) ); + + a->remove_resource( resource_name, correct_schema ); + REQUIRE_FALSE( a->exists( resource_name ) ); + REQUIRE_FALSE( a->exists( schema_name.c_str( ) ) ); +} + } // namespace test } // namespace flatdata diff --git a/flatdata-generator/flatdata/generator/templates/cpp/archive.jinja2 b/flatdata-generator/flatdata/generator/templates/cpp/archive.jinja2 index 97b5e7de..913cdc02 100644 --- a/flatdata-generator/flatdata/generator/templates/cpp/archive.jinja2 +++ b/flatdata-generator/flatdata/generator/templates/cpp/archive.jinja2 @@ -51,7 +51,7 @@ class {{ builder_name }} : public flatdata::ArchiveBuilder { public: /// Creates Archive builder - static {{ builder_name }} open( std::shared_ptr< flatdata::ResourceStorage > storage ); + static {{ builder_name }} open( std::shared_ptr< flatdata::ResourceStorage > storage, bool force_overwrite = false ); /// Archive schema static const char* schema_definition( ); @@ -65,19 +65,21 @@ public: /// Resources using {{ r|typedef_name }} = {{ resource.writer_type(r) }}; using {{ r|typedef_name("Reader") }} = {{ resource.reader_type(r) }}; {% if r|resource_provides_incremental_builder %} - {{ r|typedef_name }} start_{{ r.name }}( ); + {{ r|typedef_name }} start_{{ r.name }}( bool force_overwrite = false ); {% endif %} {% if r|provides_setter %} - bool set_{{ r.name }}( {{ r|typedef_name("Reader") }} data ); + bool set_{{ r.name }}( {{ r|typedef_name("Reader") }} data, bool force_overwrite = false ); {% endif %} - + static void clean_up_{{ r.name }}( flatdata::ResourceStorage& storage ); {% endfor %} {% for r in archive.resources|archive_resources %} using {{ r|typedef_name }} = {{ resource.writer_type(r) }}; - {{ r|typedef_name }}& {{ r.name }}( ); + {{ r|typedef_name }}& {{ r.name }}( bool force_overwrite = false ); {% endfor %} + void clean_up( flatdata::ResourceStorage& storage ); + private: {{ builder_name }}( std::shared_ptr< flatdata::ResourceStorage > storage ); @@ -205,9 +207,13 @@ inline inline {{ builder_name }} -{{ builder_name }}::open(std::shared_ptr< flatdata::ResourceStorage > storage ) +{{ builder_name }}::open(std::shared_ptr< flatdata::ResourceStorage > storage, bool force_overwrite ) { {{ builder_name }} result( storage ); + if ( force_overwrite ) + { + result.clean_up( *storage ); + } if ( !result.initialize( ) ) { return {{ builder_name }}( ); @@ -217,31 +223,69 @@ inline {{ builder_name }} {% for r in archive.resources|simple_resources %} {% if r|resource_provides_incremental_builder %} -inline auto {{ builder_name }}::start_{{ r.name }}( ) -> {{ r|typedef_name }} +inline auto {{ builder_name }}::start_{{ r.name }}( bool force_overwrite ) -> {{ r|typedef_name }} { - check_created( ); + if ( force_overwrite ) + { + clean_up_{{ r.name }}( storage( ) ); + } + else + { + check_created( ); + } return storage( ).{{ resource.incremental_write_call(r) }}( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ ); } {% endif %} {% if r|provides_setter %} inline bool -{{ builder_name }}::set_{{ r.name }}( {{ r|typedef_name("Reader") }} data ) +{{ builder_name }}::set_{{ r.name }}( {{ r|typedef_name("Reader") }} data, bool force_overwrite ) { - check_created( ); + if ( force_overwrite) + { + clean_up_{{ r.name }}( storage( ) ); + } + else + { + check_created( ); + } return storage( ).write< {{ r|typedef_name("Reader") }} >( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__, data ); } {% endif %} +inline void +{{ builder_name }}::clean_up_{{ r.name }}( flatdata::ResourceStorage& storage ) +{ + storage.remove_resource( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ ); + {% if r|is_multivector %} + storage.remove_multivector_index_resource( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ ); + {% endif %} +} {% endfor %} {% for r in archive.resources|archive_resources %} inline auto -{{ builder_name }}::{{ r.name }}( ) -> {{ r|typedef_name }}& +{{ builder_name }}::{{ r.name }}( bool force_overwrite ) -> {{ r|typedef_name }}& { if ( !m_{{ r.name }}.is_open( ) ) { - m_{{ r.name }} = {{ r|typedef_name }}::open( this->storage( ).create_directory( "{{ r.name }}" ) ); + m_{{ r.name }} = {{ r|typedef_name }}::open( this->storage( ).create_directory( "{{ r.name }}" ), force_overwrite ); } return m_{{ r.name }}; } {% endfor %} + +inline void +{{ builder_name }}::clean_up( flatdata::ResourceStorage& storage ) +{ +{% for r in archive.resources|simple_resources %} + {{ builder_name }}::clean_up_{{ r.name }}( storage ); +{% endfor %} +{% for r in archive.resources|archive_resources %} + if ( auto subarchive_storage = storage.directory( "{{ r.name }}" ) ) + { + m_{{ r.name }}.clean_up( *subarchive_storage ); + } +{% endfor %} + clean_up_signature( storage ); +} + {% endmacro %} diff --git a/flatdata-generator/flatdata/generator/templates/rust/archive.jinja2 b/flatdata-generator/flatdata/generator/templates/rust/archive.jinja2 index a3867cbb..00789e2c 100644 --- a/flatdata-generator/flatdata/generator/templates/rust/archive.jinja2 +++ b/flatdata-generator/flatdata/generator/templates/rust/archive.jinja2 @@ -256,7 +256,31 @@ impl {{archive.name}}Builder { } {% endif %} + #[inline] + pub fn clean_up_{{ r.name }}_storage(storage: flatdata::StorageHandle) -> Result<(), ::std::io::Error> { + {% if [r] | subarchive_resources %} + {% set t = fully_qualified_name(archive, r.target.node) %} + {{t}}Builder::clean_up(storage.subdir("{{r.name}}")) + {% else %} + {% if [r] | multivector_resources %} + storage.remove_multivector_index_resource("{{r.name}}", schema::{{ archive_ns }}::resources::{{ r.name | upper }})?; + {% endif %} + storage.remove_resource("{{r.name}}", schema::{{ archive_ns }}::resources::{{ r.name | upper }}) + {% endif %} + } + #[inline] + pub fn clean_up_{{ r.name }}(&self) -> Result<(), ::std::io::Error> { + Self::clean_up_{{ r.name }}_storage(self.storage.clone()) + } {% endfor %} + + #[inline] + pub fn clean_up(storage: flatdata::StorageHandle) -> Result<(), ::std::io::Error> { + {% for r in archive.resources | supported_resources %} + Self::clean_up_{{ r.name }}_storage(storage.clone())?; + {% endfor %} + storage.remove_resource("{{archive.name}}.archive", schema::{{ archive_ns }}::{{ archive.name | camel_to_snake_case | upper }}) + } } {% endif %} diff --git a/flatdata-rs/lib/src/filestorage.rs b/flatdata-rs/lib/src/filestorage.rs index 8e1e14d9..523f81a6 100644 --- a/flatdata-rs/lib/src/filestorage.rs +++ b/flatdata-rs/lib/src/filestorage.rs @@ -77,6 +77,16 @@ impl ResourceStorage for FileResourceStorage { self.path.join(resource_name).exists() } + fn remove(&self, resource_name: &str) -> Result<(), io::Error> { + let resource_path = self.path.join(resource_name); + fs::remove_file(&resource_path).map_err(|e| { + io::Error::new( + e.kind(), + String::from(resource_path.to_str().unwrap_or(resource_name)), + ) + }) + } + fn read_resource(&self, resource_name: &str) -> Result<&[u8], io::Error> { let resource_path = self.path.join(resource_name); if !resource_path.exists() { diff --git a/flatdata-rs/lib/src/memstorage.rs b/flatdata-rs/lib/src/memstorage.rs index 04fdee21..64eeb797 100644 --- a/flatdata-rs/lib/src/memstorage.rs +++ b/flatdata-rs/lib/src/memstorage.rs @@ -104,6 +104,17 @@ impl ResourceStorage for MemoryResourceStorage { .contains_key(&resource_path) } + fn remove(&self, resource_name: &str) -> Result<(), io::Error> { + let resource_path = self.path.join(resource_name); + self.storage.streams.lock().unwrap().remove(&resource_path); + self.storage + .resources + .lock() + .unwrap() + .remove(&resource_path); + Ok(()) + } + fn read_resource(&self, resource_name: &str) -> Result<&[u8], io::Error> { let resource_path = self.path.join(resource_name); if !self diff --git a/flatdata-rs/lib/src/storage.rs b/flatdata-rs/lib/src/storage.rs index d3c4eabf..0c3bb00f 100644 --- a/flatdata-rs/lib/src/storage.rs +++ b/flatdata-rs/lib/src/storage.rs @@ -50,6 +50,44 @@ pub trait ResourceStorage: std::fmt::Debug { write_schema(schema, &mut stream) } + /// remove resource and the corresponding schema + /// fail if resource or schema is missing or the existing schema doesn't match the expected + /// schema + #[doc(hidden)] + fn remove_resource(&self, resource_name: &str, expected_schema: &str) -> Result<(), io::Error> { + let schema_name = format!("{}.schema", resource_name); + let resource_exists = self.exists(resource_name); + let schema_exists = self.exists(&schema_name); + if !resource_exists && !schema_exists { + return Ok(()); + } else if resource_exists != schema_exists { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("resource or schema missing for {}", resource_name), + )); + } + let stored_schema = self.read_resource(&schema_name)?; + if stored_schema != expected_schema.as_bytes() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unexpected schema for {}", resource_name), + )); + } + self.remove(resource_name)?; + self.remove(&schema_name) + } + + /// remove multivector index resource + #[doc(hidden)] + fn remove_multivector_index_resource( + &self, + resource_name: &str, + expected_schema: &str, + ) -> Result<(), io::Error> { + let index_name = format!("{}_index", resource_name); + let index_schema = format!("index({})", expected_schema); + self.remove_resource(&index_name, &index_schema) + } // // Virtual // @@ -60,6 +98,9 @@ pub trait ResourceStorage: std::fmt::Debug { /// Returns `true` if resource exists in the storage. fn exists(&self, resource_name: &str) -> bool; + /// remove the resource with key `resouce_name` + fn remove(&self, resource_name: &str) -> Result<(), io::Error>; + /// Reads a resource in storage and returns a pointer to its raw data. /// /// This is a low level facility for opening and reading resources. Cf. diff --git a/flatdata-rs/lib/src/tarstorage.rs b/flatdata-rs/lib/src/tarstorage.rs index 6809fc81..2b4be66b 100644 --- a/flatdata-rs/lib/src/tarstorage.rs +++ b/flatdata-rs/lib/src/tarstorage.rs @@ -112,6 +112,12 @@ impl ResourceStorage for TarArchiveResourceStorage { .is_some() } + fn remove(&self, _resource_name: &str) -> Result<(), io::Error> { + Err(io::Error::other( + "Removing parts of tar archives is not supported", + )) + } + fn read_resource(&self, resource_name: &str) -> Result<&[u8], io::Error> { let resource_path = self.sub_path.join(resource_name); if let Some(data) = self.storage.read(&resource_path) { diff --git a/flatdata-rs/lib/src/test/mod.rs b/flatdata-rs/lib/src/test/mod.rs index 01869db3..fdb46825 100644 --- a/flatdata-rs/lib/src/test/mod.rs +++ b/flatdata-rs/lib/src/test/mod.rs @@ -18,6 +18,8 @@ // TODO: Implement missing docs // TODO: Implement missing debugs #![allow(dead_code, missing_debug_implementations, missing_docs)] +// The generated file defines `mod test`, which is included inside `mod test`. +#![allow(clippy::module_inception)] include!("test_generated.rs"); diff --git a/flatdata-rs/tests/coappearances/src/lib.rs b/flatdata-rs/tests/coappearances/src/lib.rs index 20f4aa44..27db72a4 100644 --- a/flatdata-rs/tests/coappearances/src/lib.rs +++ b/flatdata-rs/tests/coappearances/src/lib.rs @@ -177,11 +177,9 @@ fn copy_coappearances_archive( // open for writing let archive_path = env::temp_dir().join(to_path); println!("Will create archive in: {}", archive_path.to_str().unwrap()); - if archive_path.exists() { - fs::remove_dir_all(&archive_path).expect("could not remove already existing archive"); - } fs::create_dir_all(&archive_path).expect("could not create archive dir"); let storage = flatdata::FileResourceStorage::new(archive_path.clone()); + coappearances::GraphBuilder::clean_up(storage.clone()).expect("Cleanup failed"); let gb = coappearances::GraphBuilder::new(storage).expect("could not create archive"); // copy data