cleanup support

flatdata-cpp/include/flatdata/ArchiveBuilder.h

@@ -52,6 +52,7 @@ class ArchiveBuilder
     const flatdata::ResourceStorage& storage( ) const;
     bool is_created( ) const;
     void check_created( ) const;
+    void clean_up_signature( flatdata::ResourceStorage& storage );
 
 private:
     std::shared_ptr< flatdata::ResourceStorage > m_storage;

flatdata-cpp/include/flatdata/FileResourceStorage.h

@@ -41,6 +41,7 @@ class FileResourceStorage : public ResourceStorage
     std::unique_ptr< ResourceStorage > directory( const char* key ) override;
 
     bool exists( const char* key ) override;
+    void remove( const char* key ) override;
 
 protected:
     std::shared_ptr< std::ostream > create_output_stream( const char* key ) override;
@@ -150,4 +151,16 @@ FileResourceStorage::exists( const char* key )
     return !ec;
 }
 
+inline void
+FileResourceStorage::remove( const char* key )
+{
+    boost::system::error_code ec;
+    std::string file_path = get_path( key );
+    boost::filesystem::remove( file_path.c_str( ), ec );
+    if ( ec )
+    {
+        throw std::runtime_error( std::string( "Failed to remove " ) + file_path );
+    }
+}
+
 }  // namespace flatdata

flatdata-cpp/include/flatdata/MemoryResourceStorage.h

@@ -32,6 +32,8 @@ class MemoryResourceStorage : public ResourceStorage
 public:
     static std::unique_ptr< MemoryResourceStorage > create( );
     bool exists( const char* key ) override;
+    void remove( const char* key ) override;
+
     std::unique_ptr< ResourceStorage > create_directory( const char* key ) override;
     std::unique_ptr< ResourceStorage > directory( const char* key ) override;
 
@@ -259,6 +261,25 @@ MemoryResourceStorage::exists( const char* key )
            || helpers::prefix_exists( m_storage->streams, path + "/" );
 }
 
+inline void
+MemoryResourceStorage::remove( const char* key )
+{
+    const auto path = get_path( key );
+    std::lock_guard< std::mutex > lock( m_storage_mutex );
+
+    m_storage->streams.erase( path );
+    auto found_resource = m_storage->resources.find( path );
+    if ( found_resource != m_storage->resources.end( ) )
+    {
+#ifdef DEBUG_DATA_ACCESS_STATISTICS
+        auto& string = found_resource->second;
+        DebugDataAccessStatistics::deregister_mapping( MemoryDescriptor(
+            reinterpret_cast< const unsigned char* >( string->c_str( ) ), string->size( ) ) );
+#endif
+        m_storage->resources.erase( found_resource );
+    }
+}
+
 template < typename ResourceSerializer >
 std::string
 MemoryResourceStorage::dump_resources( bool dump_schemas, ResourceSerializer&& f ) const

flatdata-cpp/include/flatdata/ResourceStorage.h

@@ -95,6 +95,11 @@ class ResourceStorage : private boost::noncopyable
      * @brief Check if resource exists
      */
     virtual bool exists( const char* key ) = 0;
+    virtual void remove( const char* key ) = 0;
+
+    void remove_resource( const char* resource_name, const char* expected_schema );
+    void remove_multivector_index_resource( const char* resource_name,
+                                            const char* expected_schema );
 
 protected:
     /**

flatdata-cpp/include/flatdata/TarArchiveResourceStorage.h

@@ -32,6 +32,7 @@ class TarArchiveResourceStorage : public ResourceStorage
     std::unique_ptr< ResourceStorage > create_directory( const char* key ) override;
     std::unique_ptr< ResourceStorage > directory( const char* key ) override;
     bool exists( const char* key ) override;
+    void remove( const char* key ) override;
 
 protected:
     std::shared_ptr< std::ostream > create_output_stream( const char* key ) override;
@@ -123,4 +124,11 @@ TarArchiveResourceStorage::exists( const char* key )
     return m_storage->read( get_path( key ).c_str( ) ).data( ) != nullptr;
 }
 
+inline void
+TarArchiveResourceStorage::remove( const char* /* key */ )
+{
+    // Removing parts of TAR files is not supported
+    throw std::runtime_error( "Removing from TAR files is not supported" );
+}
+
 }  // namespace flatdata

flatdata-cpp/include/flatdata/internal/ResourceStorage.inl

@@ -207,4 +207,44 @@ ResourceStorage::create_multi_vector( const char* resource_name, const char* sch
                                 internal::ResourceReader( this, resource_name, schema ) ) );
 }
 
+inline void
+ResourceStorage::remove_resource( const char* resource_name, const char* expected_schema )
+{
+    const std::string schema_name = std::string( resource_name ) + ".schema";
+    const bool resource_exists = exists( resource_name );
+    const bool schema_exists = exists( schema_name.c_str( ) );
+
+    if ( !resource_exists && !schema_exists )
+    {
+        return;
+    }
+
+    if ( resource_exists != schema_exists )
+    {
+        throw std::runtime_error( std::string( "Resource or schema missing for " )
+                                  + resource_name );
+    }
+
+    auto schema = read_resource( ( schema_name.c_str( ) ) );
+    std::string stored_schema( reinterpret_cast< const char* >( schema.data( ) ),
+                               schema.size_in_bytes( ) );
+
+    if ( stored_schema != expected_schema )
+    {
+        throw std::runtime_error( std::string( "unexpected schema for " ) + resource_name );
+    }
+    remove( resource_name );
+    remove( schema_name.c_str( ) );
+}
+
+inline void
+ResourceStorage::remove_multivector_index_resource( const char* resource_name,
+                                                    const char* expected_schema )
+{
+    std::string index_name = std::string( resource_name ) + internal::INDEX_SUFFIX;
+    std::string index_schema = internal::multivector_index_schema( expected_schema );
+
+    remove_resource( index_name.c_str( ), index_schema.c_str( ) );
+}
+
 }  // namespace flatdata

flatdata-cpp/src/ArchiveBuilder.cpp

@@ -67,4 +67,11 @@ ArchiveBuilder::initialize( )
     return static_cast< bool >( signature );
 }
 
+void
+ArchiveBuilder::clean_up_signature( flatdata::ResourceStorage& storage )
+{
+    const auto signature_name = internal::signature_name( name( ) );
+    storage.remove_resource( signature_name.c_str( ), schema( ) );
+}
+
 }  // namespace flatdata

flatdata-cpp/test/ResourceStorageTest.cpp

@@ -81,5 +81,28 @@ TEST_CASE( "schema is checked for written vector", "[ResourceStorage]" )
         ( a->read< ArrayView< AStruct > >( resource_name, correct_schema ).is_initialized( ) ) );
 }
 
+TEST_CASE( "resource doesn't exist after removing it", "[ResourceStorage]" )
+{
+    std::unique_ptr< ResourceStorage > a = MemoryResourceStorage::create( );
+    auto create = [ & ] {
+        auto v = a->create_external_vector< AStruct >( resource_name, correct_schema );
+        v.close( );
+    };
+    REQUIRE_NOTHROW( create( ) );
+
+    REQUIRE_FALSE(
+        ( a->read< ArrayView< AStruct > >( resource_name, incorrect_schema ).is_initialized( ) ) );
+    REQUIRE(
+        ( a->read< ArrayView< AStruct > >( resource_name, correct_schema ).is_initialized( ) ) );
+
+    std::string schema_name = std::string( resource_name ) + ".schema";
+    REQUIRE( a->exists( resource_name ) );
+    REQUIRE( a->exists( schema_name.c_str( ) ) );
+
+    a->remove_resource( resource_name, correct_schema );
+    REQUIRE_FALSE( a->exists( resource_name ) );
+    REQUIRE_FALSE( a->exists( schema_name.c_str( ) ) );
+}
+
 }  // namespace test
 }  // namespace flatdata

flatdata-generator/flatdata/generator/templates/cpp/archive.jinja2

@@ -51,7 +51,7 @@ class {{ builder_name }} : public flatdata::ArchiveBuilder
 {
 public:
     /// Creates Archive builder
-    static {{ builder_name }} open( std::shared_ptr< flatdata::ResourceStorage > storage );
+    static {{ builder_name }} open( std::shared_ptr< flatdata::ResourceStorage > storage, bool force_overwrite = false );
     /// Archive schema
     static const char* schema_definition( );
 
@@ -65,19 +65,21 @@ public:  /// Resources
     using {{ r|typedef_name }} = {{ resource.writer_type(r) }};
     using {{ r|typedef_name("Reader") }} = {{ resource.reader_type(r) }};
     {% if r|resource_provides_incremental_builder %}
-    {{ r|typedef_name }} start_{{ r.name }}( );
+    {{ r|typedef_name }} start_{{ r.name }}( bool force_overwrite = false );
     {% endif %}
     {% if r|provides_setter %}
-    bool set_{{ r.name }}( {{ r|typedef_name("Reader") }} data );
+    bool set_{{ r.name }}( {{ r|typedef_name("Reader") }} data, bool force_overwrite = false );
     {% endif %}
-
+    static void clean_up_{{ r.name }}( flatdata::ResourceStorage& storage );
     {% endfor %}
 
     {% for r in archive.resources|archive_resources %}
     using {{ r|typedef_name }} = {{ resource.writer_type(r) }};
-    {{ r|typedef_name }}& {{ r.name }}( );
+    {{ r|typedef_name }}& {{ r.name }}( bool force_overwrite = false );
     {% endfor %}
 
+    void clean_up( flatdata::ResourceStorage& storage );
+
 private:
     {{ builder_name }}( std::shared_ptr< flatdata::ResourceStorage > storage );
 
@@ -205,9 +207,13 @@ inline
 
 
 inline {{ builder_name }}
-{{ builder_name }}::open(std::shared_ptr< flatdata::ResourceStorage > storage )
+{{ builder_name }}::open(std::shared_ptr< flatdata::ResourceStorage > storage, bool force_overwrite )
 {
     {{ builder_name }} result( storage );
+    if ( force_overwrite )
+    {
+        result.clean_up( *storage );
+    }
     if ( !result.initialize( ) )
     {
         return {{ builder_name }}( );
@@ -217,31 +223,69 @@ inline {{ builder_name }}
 
 {% for r in archive.resources|simple_resources %}
 {%  if r|resource_provides_incremental_builder %}
-inline auto {{ builder_name }}::start_{{ r.name }}( ) -> {{ r|typedef_name }}
+inline auto {{ builder_name }}::start_{{ r.name }}( bool force_overwrite ) -> {{ r|typedef_name }}
 {
-    check_created( );
+    if ( force_overwrite )
+    {
+        clean_up_{{ r.name }}( storage( ) );
+    }
+    else
+    {
+        check_created( );
+    }
     return storage( ).{{ resource.incremental_write_call(r) }}( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ );
 }
 {% endif %}
 {% if r|provides_setter %}
 inline bool
-{{ builder_name }}::set_{{ r.name }}( {{ r|typedef_name("Reader") }} data )
+{{ builder_name }}::set_{{ r.name }}( {{ r|typedef_name("Reader") }} data, bool force_overwrite )
 {
-    check_created( );
+    if ( force_overwrite)
+    {
+        clean_up_{{ r.name }}( storage( ) );
+    }
+    else
+    {
+        check_created( );
+    }
     return storage( ).write< {{ r|typedef_name("Reader") }} >( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__, data );
 }
 {% endif %}
+inline void
+{{ builder_name }}::clean_up_{{ r.name }}( flatdata::ResourceStorage& storage )
+{
+    storage.remove_resource( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ );
+    {% if r|is_multivector %}
+    storage.remove_multivector_index_resource( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ );
+    {% endif %}
+}
 {% endfor %}
 
 {% for r in archive.resources|archive_resources %}
 inline auto
-{{ builder_name }}::{{ r.name }}( ) -> {{ r|typedef_name }}&
+{{ builder_name }}::{{ r.name }}( bool force_overwrite ) -> {{ r|typedef_name }}&
 {
     if ( !m_{{ r.name }}.is_open( ) )
     {
-        m_{{ r.name }} = {{ r|typedef_name }}::open( this->storage( ).create_directory( "{{ r.name }}" ) );
+        m_{{ r.name }} = {{ r|typedef_name }}::open( this->storage( ).create_directory( "{{ r.name }}" ), force_overwrite );
     }
     return m_{{ r.name }};
 }
 {% endfor %}
+
+inline void
+{{ builder_name }}::clean_up( flatdata::ResourceStorage& storage )
+{
+{% for r in archive.resources|simple_resources %}
+    {{ builder_name }}::clean_up_{{ r.name }}( storage );
+{% endfor %}
+{% for r in archive.resources|archive_resources %}
+    if ( auto subarchive_storage = storage.directory( "{{ r.name }}" ) )
+    {
+        m_{{ r.name }}.clean_up( *subarchive_storage );
+    }
+{% endfor %}
+    clean_up_signature( storage );
+}
+
 {% endmacro %}

flatdata-generator/flatdata/generator/templates/rust/archive.jinja2

@@ -256,7 +256,31 @@ impl {{archive.name}}Builder {
     }
 
     {% endif %}
+    #[inline]
+    pub fn clean_up_{{ r.name }}_storage(storage: flatdata::StorageHandle) -> Result<(), ::std::io::Error> {
+        {% if [r] | subarchive_resources %}
+        {% set t = fully_qualified_name(archive, r.target.node) %}
+        {{t}}Builder::clean_up(storage.subdir("{{r.name}}"))
+        {% else %}
+        {% if [r] | multivector_resources %}
+        storage.remove_multivector_index_resource("{{r.name}}", schema::{{ archive_ns }}::resources::{{ r.name | upper }})?;
+        {% endif %}
+        storage.remove_resource("{{r.name}}", schema::{{ archive_ns }}::resources::{{ r.name | upper }})
+        {% endif %}
+    }
+    #[inline]
+    pub fn clean_up_{{ r.name }}(&self) -> Result<(), ::std::io::Error> {
+        Self::clean_up_{{ r.name }}_storage(self.storage.clone())
+    }
     {% endfor %}
+
+    #[inline]
+    pub fn clean_up(storage: flatdata::StorageHandle) -> Result<(), ::std::io::Error> {
+        {% for r in archive.resources | supported_resources %}
+        Self::clean_up_{{ r.name }}_storage(storage.clone())?;
+        {% endfor %}
+        storage.remove_resource("{{archive.name}}.archive", schema::{{ archive_ns }}::{{ archive.name | camel_to_snake_case | upper }})
+    }
 }
 {% endif %}
 

flatdata-rs/lib/src/filestorage.rs

@@ -77,6 +77,16 @@ impl ResourceStorage for FileResourceStorage {
         self.path.join(resource_name).exists()
     }
 
+    fn remove(&self, resource_name: &str) -> Result<(), io::Error> {
+        let resource_path = self.path.join(resource_name);
+        fs::remove_file(&resource_path).map_err(|e| {
+            io::Error::new(
+                e.kind(),
+                String::from(resource_path.to_str().unwrap_or(resource_name)),
+            )
+        })
+    }
+
     fn read_resource(&self, resource_name: &str) -> Result<&[u8], io::Error> {
         let resource_path = self.path.join(resource_name);
         if !resource_path.exists() {

flatdata-rs/lib/src/memstorage.rs

@@ -104,6 +104,17 @@ impl ResourceStorage for MemoryResourceStorage {
                 .contains_key(&resource_path)
     }
 
+    fn remove(&self, resource_name: &str) -> Result<(), io::Error> {
+        let resource_path = self.path.join(resource_name);
+        self.storage.streams.lock().unwrap().remove(&resource_path);
+        self.storage
+            .resources
+            .lock()
+            .unwrap()
+            .remove(&resource_path);
+        Ok(())
+    }
+
     fn read_resource(&self, resource_name: &str) -> Result<&[u8], io::Error> {
         let resource_path = self.path.join(resource_name);
         if !self