From 0fc1f801854d974275a9efc6adc8442740485fe4 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Mon, 19 Oct 2026 12:13:18 +0300 Subject: [PATCH 01/54] common: spaceallocator: identify outdated items by ID and version Signed-off-by: Mykola Solianko --- .../common/spaceallocator/spaceallocator.hpp | 2 +- .../spaceallocator/tests/spaceallocator.cpp | 141 ++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/src/core/common/spaceallocator/spaceallocator.hpp b/src/core/common/spaceallocator/spaceallocator.hpp index a1f076198..b0a345450 100644 --- a/src/core/common/spaceallocator/spaceallocator.hpp +++ b/src/core/common/spaceallocator/spaceallocator.hpp @@ -210,7 +210,7 @@ class Partition { LockGuard lock {mMutex}; for (auto& outdatedItem : mOutdatedItems) { - if (outdatedItem.mID == item.mID) { + if (outdatedItem.mID == item.mID && outdatedItem.mVersion == item.mVersion) { outdatedItem = item; return ErrorEnum::eNone; diff --git a/src/core/common/spaceallocator/tests/spaceallocator.cpp b/src/core/common/spaceallocator/tests/spaceallocator.cpp index f1dfb746c..1583cb179 100644 --- a/src/core/common/spaceallocator/tests/spaceallocator.cpp +++ b/src/core/common/spaceallocator/tests/spaceallocator.cpp @@ -501,4 +501,145 @@ TEST_F(SpaceallocatorTest, ResizeSpaceInsufficientSpace) ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); } +TEST_F(SpaceallocatorTest, OutdatedItemsMultipleVersions) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(0, ErrorEnum::eNone))); + + const Time now = Time::Now(); + const size_t itemSize = 128 * cKilobyte; + + // both versions of the same item should be tracked independently + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "2.0.0", now.Add(-1 * Time::cHours)).IsNone()); + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", now.Add(-2 * Time::cHours)).IsNone()); + + { + InSequence seq; + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(itemSize, ErrorEnum::eNone))); + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("2.0.0"))) + .WillOnce(Return(RetWithError(itemSize, ErrorEnum::eNone))); + } + + // the oldest version is removed first + + auto [space1, err1] = mSpaceAllocator.AllocateSpace(itemSize); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + ASSERT_TRUE(space1->Accept().IsNone()); + + // then the newer one + + auto [space2, err2] = mSpaceAllocator.AllocateSpace(itemSize); + ASSERT_TRUE(err2.IsNone()); + ASSERT_NE(space2.Get(), nullptr); + ASSERT_TRUE(space2->Accept().IsNone()); + + // no outdated items left + + auto [space3, err3] = mSpaceAllocator.AllocateSpace(itemSize); + EXPECT_EQ(err3, ErrorEnum::eNoMemory); + EXPECT_EQ(space3.Get(), nullptr); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RestoreOutdatedItemVersion) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(0, ErrorEnum::eNone))); + + const Time now = Time::Now(); + const size_t itemSize = 128 * cKilobyte; + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", now.Add(-2 * Time::cHours)).IsNone()); + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "2.0.0", now.Add(-1 * Time::cHours)).IsNone()); + + // restoring one version should keep the other one outdated + + ASSERT_TRUE(mSpaceAllocator.RestoreOutdatedItem("item1", "1.0.0").IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("2.0.0"))) + .WillOnce(Return(RetWithError(itemSize, ErrorEnum::eNone))); + + auto [space1, err1] = mSpaceAllocator.AllocateSpace(itemSize); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + ASSERT_TRUE(space1->Accept().IsNone()); + + // restored version must not be removed + + auto [space2, err2] = mSpaceAllocator.AllocateSpace(itemSize); + EXPECT_EQ(err2, ErrorEnum::eNoMemory); + EXPECT_EQ(space2.Get(), nullptr); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, ReAddOutdatedItemUpdatesExisting) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(0, ErrorEnum::eNone))); + + const Time now = Time::Now(); + const size_t itemSize = 128 * cKilobyte; + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", now.Add(-1 * Time::cHours)).IsNone()); + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item2", "1.0.0", now.Add(-2 * Time::cHours)).IsNone()); + + // re-adding the same ID and version updates the timestamp instead of adding a duplicate + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", now.Add(-3 * Time::cHours)).IsNone()); + + { + InSequence seq; + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(itemSize, ErrorEnum::eNone))); + EXPECT_CALL(mRemover, RemoveItem(String("item2"), String("1.0.0"))) + .WillOnce(Return(RetWithError(itemSize, ErrorEnum::eNone))); + } + + auto [space1, err1] = mSpaceAllocator.AllocateSpace(itemSize); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + ASSERT_TRUE(space1->Accept().IsNone()); + + auto [space2, err2] = mSpaceAllocator.AllocateSpace(itemSize); + ASSERT_TRUE(err2.IsNone()); + ASSERT_NE(space2.Get(), nullptr); + ASSERT_TRUE(space2->Accept().IsNone()); + + // no duplicate of item1 is left + + auto [space3, err3] = mSpaceAllocator.AllocateSpace(itemSize); + EXPECT_EQ(err3, ErrorEnum::eNoMemory); + EXPECT_EQ(space3.Get(), nullptr); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + } // namespace aos::spaceallocator From 6fe36b2bfdbc17c9c91bf8d57002f50f9162b4c5 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 21 Oct 2026 18:33:55 +0300 Subject: [PATCH 02/54] common: spaceallocator: add RemoveOutdatedItem Signed-off-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- .../spaceallocator/itf/spaceallocator.hpp | 13 + .../common/spaceallocator/spaceallocator.hpp | 52 +++ .../spaceallocator/tests/spaceallocator.cpp | 390 ++++++++++++++++++ .../common/tests/mocks/spaceallocatormock.hpp | 1 + .../common/tests/stubs/spaceallocatorstub.hpp | 7 + 5 files changed, 463 insertions(+) diff --git a/src/core/common/spaceallocator/itf/spaceallocator.hpp b/src/core/common/spaceallocator/itf/spaceallocator.hpp index 5814f5249..a9c8d7ce8 100644 --- a/src/core/common/spaceallocator/itf/spaceallocator.hpp +++ b/src/core/common/spaceallocator/itf/spaceallocator.hpp @@ -111,6 +111,19 @@ class SpaceAllocatorItf { */ virtual Error RestoreOutdatedItem(const String& id, const String& version) = 0; + /** + * Removes the oldest outdated item. + * + * The freed size is already accounted by the space allocator: the caller must not call FreeSpace for it. If the + * item remover fails, the item is still dropped from the outdated items list and the remover error is returned. + * The item remover is called with the partition lock taken (shared by all space allocators on the same mount + * point): the item remover must not call any space allocator of this partition, and this method must not be + * called while holding a mutex the item remover might need. + * + * @return RetWithError: freed size, eNotFound if there is no outdated item to remove. + */ + virtual RetWithError RemoveOutdatedItem() = 0; + /** * Allocates done. * diff --git a/src/core/common/spaceallocator/spaceallocator.hpp b/src/core/common/spaceallocator/spaceallocator.hpp index b0a345450..3d7ebced2 100644 --- a/src/core/common/spaceallocator/spaceallocator.hpp +++ b/src/core/common/spaceallocator/spaceallocator.hpp @@ -260,6 +260,41 @@ class Partition { } } + /** + * Removes the oldest outdated item of the space allocator. + * + * The item is removed from the outdated items list whatever the result of the item remover. The freed size is + * accounted in the partition only: the allocator free callback is not called, the caller must account the freed + * size in the space allocator. + * + * @param spaceAllocator space allocator the item belongs to. + * @return RetWithError: freed size, eNotFound if there is no outdated item to remove. + */ + RetWithError RemoveOutdatedItem(const SpaceAllocatorItf* spaceAllocator) + { + LockGuard lock {mMutex}; + + mOutdatedItems.Sort([](const OutdatedItem& a, const OutdatedItem& b) { return a.mTimestamp < b.mTimestamp; }); + + auto oldestItem = mOutdatedItems.FindIf( + [spaceAllocator](const OutdatedItem& item) { return item.mSpaceAllocator == spaceAllocator; }); + if (oldestItem == mOutdatedItems.end()) { + return {0, Error(ErrorEnum::eNotFound, "no outdated item")}; + } + + auto [size, err] = oldestItem->mRemover->RemoveItem(oldestItem->mID, oldestItem->mVersion); + + mOutdatedItems.Erase(oldestItem); + + if (!err.IsNone()) { + return {0, err}; + } + + mAvailableSize += size; + + return {size, ErrorEnum::eNone}; + } + static constexpr size_t cMaxNumOutdatedItems = AOS_CONFIG_SPACEALLOCATOR_MAX_OUTDATED_ITEMS; StaticString mMountPoint; @@ -543,6 +578,23 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { return ErrorEnum::eNone; } + /** + * Removes the oldest outdated item. + * + * @return RetWithError: freed size, eNotFound if there is no outdated item to remove. + */ + RetWithError RemoveOutdatedItem() override + { + auto [size, err] = mPartition->RemoveOutdatedItem(this); + if (!err.IsNone()) { + return {0, err}; + } + + Free(size); + + return {size, ErrorEnum::eNone}; + } + private: Error ResizeSpace(size_t oldSize, size_t newSize) { diff --git a/src/core/common/spaceallocator/tests/spaceallocator.cpp b/src/core/common/spaceallocator/tests/spaceallocator.cpp index 1583cb179..62c97acff 100644 --- a/src/core/common/spaceallocator/tests/spaceallocator.cpp +++ b/src/core/common/spaceallocator/tests/spaceallocator.cpp @@ -4,6 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + #include #include @@ -642,4 +644,392 @@ TEST_F(SpaceallocatorTest, ReAddOutdatedItemUpdatesExisting) ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); } +TEST_F(SpaceallocatorTest, RemoveOutdatedItemNoItems) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(_, _)).Times(0); + + auto [size, err] = mSpaceAllocator.RemoveOutdatedItem(); + EXPECT_EQ(err, ErrorEnum::eNotFound); + EXPECT_EQ(size, 0U); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RemoveOutdatedItemOldestFirst) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + const Time now = Time::Now(); + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", now.Add(-1 * Time::cHours)).IsNone()); + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item2", "1.0.0", now.Add(-3 * Time::cHours)).IsNone()); + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item3", "1.0.0", now.Add(-2 * Time::cHours)).IsNone()); + + { + InSequence seq; + + EXPECT_CALL(mRemover, RemoveItem(String("item2"), String("1.0.0"))) + .WillOnce(Return(RetWithError(32 * cKilobyte, ErrorEnum::eNone))); + EXPECT_CALL(mRemover, RemoveItem(String("item3"), String("1.0.0"))) + .WillOnce(Return(RetWithError(64 * cKilobyte, ErrorEnum::eNone))); + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(128 * cKilobyte, ErrorEnum::eNone))); + } + + // items are removed one by one starting from the oldest one + + auto [size1, err1] = mSpaceAllocator.RemoveOutdatedItem(); + ASSERT_TRUE(err1.IsNone()); + EXPECT_EQ(size1, 32 * cKilobyte); + + auto [size2, err2] = mSpaceAllocator.RemoveOutdatedItem(); + ASSERT_TRUE(err2.IsNone()); + EXPECT_EQ(size2, 64 * cKilobyte); + + auto [size3, err3] = mSpaceAllocator.RemoveOutdatedItem(); + ASSERT_TRUE(err3.IsNone()); + EXPECT_EQ(size3, 128 * cKilobyte); + + // no outdated items left + + auto [size4, err4] = mSpaceAllocator.RemoveOutdatedItem(); + EXPECT_EQ(err4, ErrorEnum::eNotFound); + EXPECT_EQ(size4, 0U); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RemoveOutdatedItemRemoverErrorPartition) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + // no allocator limit: only the partition accounting is checked + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(30 * cKilobyte, ErrorEnum::eNone))); + + // space1 is not accepted on purpose: the partition available size is not re-read from the file system + + auto [space1, err1] = mSpaceAllocator.AllocateSpace(25 * cKilobyte); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", Time::Now()).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(64 * cKilobyte, ErrorEnum::eFailed))); + + // remover error is returned and the failed item is dropped from the outdated items + + auto [size1, err2] = mSpaceAllocator.RemoveOutdatedItem(); + EXPECT_EQ(err2, ErrorEnum::eFailed); + EXPECT_EQ(size1, 0U); + + auto [size2, err3] = mSpaceAllocator.RemoveOutdatedItem(); + EXPECT_EQ(err3, ErrorEnum::eNotFound); + EXPECT_EQ(size2, 0U); + + // 5K is available: 10K doesn't fit, the size returned with the error must not be accounted + + auto [space2, err4] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + EXPECT_EQ(err4, ErrorEnum::eNoMemory); + EXPECT_EQ(space2.Get(), nullptr); + + ASSERT_TRUE(space1->Accept().IsNone()); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RemoveOutdatedItemRemoverErrorAllocator) +{ + constexpr size_t cTotalSize = 100 * cKilobyte; + + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + // allocator limit is 50K + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, 50, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetTotalSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetDirSize(mPath)) + .WillRepeatedly(Return(RetWithError(20 * cKilobyte, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + + // space1 is not accepted on purpose: the allocated size is not re-read from the file system + + auto [space1, err1] = mSpaceAllocator.AllocateSpace(25 * cKilobyte); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", Time::Now()).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(64 * cKilobyte, ErrorEnum::eFailed))); + + // remover error is returned and the failed item is dropped from the outdated items + + auto [size1, err2] = mSpaceAllocator.RemoveOutdatedItem(); + EXPECT_EQ(err2, ErrorEnum::eFailed); + EXPECT_EQ(size1, 0U); + + auto [size2, err3] = mSpaceAllocator.RemoveOutdatedItem(); + EXPECT_EQ(err3, ErrorEnum::eNotFound); + EXPECT_EQ(size2, 0U); + + // 45K is allocated: 10K more exceeds the limit, the size returned with the error must not be accounted + + auto [space2, err4] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + EXPECT_EQ(err4, ErrorEnum::eNoMemory); + EXPECT_EQ(space2.Get(), nullptr); + + ASSERT_TRUE(space1->Accept().IsNone()); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RemoveOutdatedItemOtherAllocatorUntouched) +{ + SpaceAllocator<5> allocator1; + SpaceAllocator<5> allocator2; + + StrictMock remover2; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .Times(2) + .WillRepeatedly(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(allocator1.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + ASSERT_TRUE(allocator2.Init(mAllocator, mPath, mPlatformFS, mLimit, &remover2).IsNone()); + + const Time now = Time::Now(); + + // allocator2 item is older but belongs to another allocator sharing the partition + + ASSERT_TRUE(allocator1.AddOutdatedItem("item1", "1.0.0", now.Add(-1 * Time::cHours)).IsNone()); + ASSERT_TRUE(allocator2.AddOutdatedItem("item2", "1.0.0", now.Add(-2 * Time::cHours)).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(32 * cKilobyte, ErrorEnum::eNone))); + + auto [size1, err1] = allocator1.RemoveOutdatedItem(); + ASSERT_TRUE(err1.IsNone()); + EXPECT_EQ(size1, 32 * cKilobyte); + + auto [size2, err2] = allocator1.RemoveOutdatedItem(); + EXPECT_EQ(err2, ErrorEnum::eNotFound); + EXPECT_EQ(size2, 0U); + + // allocator2 item is still there + + EXPECT_CALL(remover2, RemoveItem(String("item2"), String("1.0.0"))) + .WillOnce(Return(RetWithError(64 * cKilobyte, ErrorEnum::eNone))); + + auto [size3, err3] = allocator2.RemoveOutdatedItem(); + ASSERT_TRUE(err3.IsNone()); + EXPECT_EQ(size3, 64 * cKilobyte); + + auto [size4, err4] = allocator2.RemoveOutdatedItem(); + EXPECT_EQ(err4, ErrorEnum::eNotFound); + EXPECT_EQ(size4, 0U); + + ASSERT_TRUE(allocator1.Close().IsNone()); + ASSERT_TRUE(allocator2.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RemoveOutdatedItemFreesSpace) +{ + constexpr size_t cTotalSize = 100 * cKilobyte; + + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + // allocator limit is 50K + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, 50, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetTotalSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetDirSize(mPath)) + .WillRepeatedly(Return(RetWithError(20 * cKilobyte, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + + auto [space1, err1] = mSpaceAllocator.AllocateSpace(25 * cKilobyte); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + + // 45K is allocated: 10K more exceeds the limit + + auto [space2, err2] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + EXPECT_EQ(err2, ErrorEnum::eNoMemory); + EXPECT_EQ(space2.Get(), nullptr); + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", Time::Now()).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(15 * cKilobyte, ErrorEnum::eNone))); + + auto [size, err] = mSpaceAllocator.RemoveOutdatedItem(); + ASSERT_TRUE(err.IsNone()); + EXPECT_EQ(size, 15 * cKilobyte); + + // freed size is taken into account without FreeSpace call + + auto [space3, err3] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + ASSERT_TRUE(err3.IsNone()); + ASSERT_NE(space3.Get(), nullptr); + + // 40K is allocated: 15K more exceeds the limit, the freed size must not be accounted twice + + auto [space4, err4] = mSpaceAllocator.AllocateSpace(15 * cKilobyte); + EXPECT_EQ(err4, ErrorEnum::eNoMemory); + EXPECT_EQ(space4.Get(), nullptr); + + // space1 is accepted only here on purpose: while it is not accepted, the allocated size is not re-read from the + // file system, so the allocator accounting is checked + + ASSERT_TRUE(space3->Accept().IsNone()); + ASSERT_TRUE(space1->Accept().IsNone()); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RemoveOutdatedItemFreesPartitionSpace) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + // no allocator limit: only the partition accounting is checked + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(30 * cKilobyte, ErrorEnum::eNone))); + + // space1 is not accepted on purpose: the partition available size is not re-read from the file system + + auto [space1, err1] = mSpaceAllocator.AllocateSpace(25 * cKilobyte); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + + // 5K is available: 10K doesn't fit + + auto [space2, err2] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + EXPECT_EQ(err2, ErrorEnum::eNoMemory); + EXPECT_EQ(space2.Get(), nullptr); + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", Time::Now()).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(15 * cKilobyte, ErrorEnum::eNone))); + + auto [size, err] = mSpaceAllocator.RemoveOutdatedItem(); + ASSERT_TRUE(err.IsNone()); + EXPECT_EQ(size, 15 * cKilobyte); + + // 20K is available now + + auto [space3, err3] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + ASSERT_TRUE(err3.IsNone()); + ASSERT_NE(space3.Get(), nullptr); + + // 10K is available: 15K doesn't fit, the freed size must not be accounted twice + + auto [space4, err4] = mSpaceAllocator.AllocateSpace(15 * cKilobyte); + EXPECT_EQ(err4, ErrorEnum::eNoMemory); + EXPECT_EQ(space4.Get(), nullptr); + + ASSERT_TRUE(space3->Accept().IsNone()); + ASSERT_TRUE(space1->Accept().IsNone()); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RemoveOutdatedItemAfterRestore) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", Time::Now()).IsNone()); + ASSERT_TRUE(mSpaceAllocator.RestoreOutdatedItem("item1", "1.0.0").IsNone()); + + // restored item must not be removed + + EXPECT_CALL(mRemover, RemoveItem(_, _)).Times(0); + + auto [size, err] = mSpaceAllocator.RemoveOutdatedItem(); + EXPECT_EQ(err, ErrorEnum::eNotFound); + EXPECT_EQ(size, 0U); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, RemoveOutdatedItemAfterEviction) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + const Time now = Time::Now(); + + // item0 is the oldest one + + for (size_t i = 0; i < Partition::cMaxNumOutdatedItems; i++) { + const auto id = "item" + std::to_string(i); + + ASSERT_TRUE(mSpaceAllocator + .AddOutdatedItem(id.c_str(), "1.0.0", + now.Add(-static_cast(Partition::cMaxNumOutdatedItems - i) * Time::cMinutes)) + .IsNone()); + } + + { + InSequence seq; + + // the outdated items list is full: the oldest item is evicted on adding a new one + EXPECT_CALL(mRemover, RemoveItem(String("item0"), String("1.0.0"))) + .WillOnce(Return(RetWithError(32 * cKilobyte, ErrorEnum::eNone))); + // the next oldest item is removed after eviction + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce(Return(RetWithError(64 * cKilobyte, ErrorEnum::eNone))); + } + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("itemNew", "1.0.0", now).IsNone()); + + auto [size, err] = mSpaceAllocator.RemoveOutdatedItem(); + ASSERT_TRUE(err.IsNone()); + EXPECT_EQ(size, 64 * cKilobyte); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + } // namespace aos::spaceallocator diff --git a/src/core/common/tests/mocks/spaceallocatormock.hpp b/src/core/common/tests/mocks/spaceallocatormock.hpp index dd424de80..26eb6950f 100644 --- a/src/core/common/tests/mocks/spaceallocatormock.hpp +++ b/src/core/common/tests/mocks/spaceallocatormock.hpp @@ -41,6 +41,7 @@ class SpaceAllocatorMock : public SpaceAllocatorItf { MOCK_METHOD(void, FreeSpace, (size_t), (override)); MOCK_METHOD(Error, AddOutdatedItem, (const String&, const String&, const Time&), (override)); MOCK_METHOD(Error, RestoreOutdatedItem, (const String&, const String&), (override)); + MOCK_METHOD(RetWithError, RemoveOutdatedItem, (), (override)); MOCK_METHOD(Error, AllocateDone, (), (override)); }; diff --git a/src/core/common/tests/stubs/spaceallocatorstub.hpp b/src/core/common/tests/stubs/spaceallocatorstub.hpp index 09b3689a1..82888249e 100644 --- a/src/core/common/tests/stubs/spaceallocatorstub.hpp +++ b/src/core/common/tests/stubs/spaceallocatorstub.hpp @@ -131,6 +131,13 @@ class SpaceAllocatorStub : public SpaceAllocatorItf { return ErrorEnum::eNone; } + /** + * Removes the oldest outdated item. + * + * @return RetWithError. + */ + RetWithError RemoveOutdatedItem() override { return {0, ErrorEnum::eNotFound}; } + /** * Allocates done. * From 5884c276d410d7f523bb64b6dceee50eca541475 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 21 Oct 2026 20:42:20 +0300 Subject: [PATCH 03/54] common: imagemanager: add common image manager skeleton Signed-off-by: Mykola Solianko Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/common/CMakeLists.txt | 5 +- src/core/common/imagemanager/CMakeLists.txt | 57 ++++ src/core/common/imagemanager/config.hpp | 28 ++ src/core/common/imagemanager/imagemanager.cpp | 126 ++++++++ src/core/common/imagemanager/imagemanager.hpp | 269 ++++++++++++++++++ .../imagemanager/itf/blobinfoprovider.hpp | 56 ++++ .../imagemanager/itf/itemstatusprovider.hpp | 77 +++++ src/core/common/imagemanager/itf/storage.hpp | 168 +++++++++++ .../common/imagemanager/tests/CMakeLists.txt | 35 +++ .../imagemanager/tests/imagemanager.cpp | 85 ++++++ .../imagemanager/tests/stubs/storagestub.hpp | 174 +++++++++++ src/core/common/tests/mocks/CMakeLists.txt | 2 + .../tests/mocks/blobinfoprovidermock.hpp | 34 +++ .../tests/mocks/itemstatuslistenermock.hpp | 27 ++ 14 files changed, 1142 insertions(+), 1 deletion(-) create mode 100644 src/core/common/imagemanager/CMakeLists.txt create mode 100644 src/core/common/imagemanager/config.hpp create mode 100644 src/core/common/imagemanager/imagemanager.cpp create mode 100644 src/core/common/imagemanager/imagemanager.hpp create mode 100644 src/core/common/imagemanager/itf/blobinfoprovider.hpp create mode 100644 src/core/common/imagemanager/itf/itemstatusprovider.hpp create mode 100644 src/core/common/imagemanager/itf/storage.hpp create mode 100644 src/core/common/imagemanager/tests/CMakeLists.txt create mode 100644 src/core/common/imagemanager/tests/imagemanager.cpp create mode 100644 src/core/common/imagemanager/tests/stubs/storagestub.hpp create mode 100644 src/core/common/tests/mocks/blobinfoprovidermock.hpp create mode 100644 src/core/common/tests/mocks/itemstatuslistenermock.hpp diff --git a/src/core/common/CMakeLists.txt b/src/core/common/CMakeLists.txt index fa0a6b812..a83edee0c 100644 --- a/src/core/common/CMakeLists.txt +++ b/src/core/common/CMakeLists.txt @@ -31,7 +31,9 @@ set(INSTALL_HEADERS aos_core_common_version ) -set(INSTALL_LIBRARIES aos_core_common_crypto aos_core_common_monitoring aos_core_common_pkcs11 aos_core_common_tools) +set(INSTALL_LIBRARIES aos_core_common_crypto aos_core_common_imagemanager aos_core_common_monitoring + aos_core_common_pkcs11 aos_core_common_tools +) if(WITH_TEST) list(APPEND INSTALL_HEADERS aos_core_common_tests_mocks aos_core_common_tests_stubs) @@ -61,6 +63,7 @@ add_subdirectory(crypto) add_subdirectory(downloader) add_subdirectory(logging) add_subdirectory(iamclient) +add_subdirectory(imagemanager) add_subdirectory(instancestatusprovider) add_subdirectory(monitoring) add_subdirectory(networkmanager) diff --git a/src/core/common/imagemanager/CMakeLists.txt b/src/core/common/imagemanager/CMakeLists.txt new file mode 100644 index 000000000..ed347a15b --- /dev/null +++ b/src/core/common/imagemanager/CMakeLists.txt @@ -0,0 +1,57 @@ +# +# Copyright (C) 2026 EPAM Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# ###################################################################################################################### +# Target name +# ###################################################################################################################### + +set(TARGET_NAME imagemanager) + +# ###################################################################################################################### +# Sources +# ###################################################################################################################### + +set(SOURCES imagemanager.cpp) + +# ###################################################################################################################### +# Headers +# ###################################################################################################################### + +set(HEADERS config.hpp imagemanager.hpp itf/blobinfoprovider.hpp itf/itemstatusprovider.hpp itf/storage.hpp) + +# ###################################################################################################################### +# Libraries +# ###################################################################################################################### + +set(LIBRARIES aos::core::common::crypto aos::core::common::ocispec aos::core::common::spaceallocator + aos::core::common::tools +) + +# ###################################################################################################################### +# Target +# ###################################################################################################################### + +add_module( + TARGET_NAME + ${TARGET_NAME} + LOG_MODULE + STACK_USAGE + ${AOS_STACK_USAGE} + SOURCES + ${SOURCES} + HEADERS + ${HEADERS} + LIBRARIES + ${LIBRARIES} +) + +# ###################################################################################################################### +# Tests +# ###################################################################################################################### + +if(WITH_TEST) + add_subdirectory(tests) +endif() diff --git a/src/core/common/imagemanager/config.hpp b/src/core/common/imagemanager/config.hpp new file mode 100644 index 000000000..adb12e1fe --- /dev/null +++ b/src/core/common/imagemanager/config.hpp @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_IMAGEMANAGER_CONFIG_HPP_ +#define AOS_CORE_COMMON_IMAGEMANAGER_CONFIG_HPP_ + +#include +#include +#include + +namespace aos::common::imagemanager { + +/** + * Image manager config. + */ +struct Config { + StaticString mImagePath; + size_t mPartLimit {}; + Duration mItemCacheTTL {}; + Duration mRemoveOutdatedPeriod {}; +}; + +} // namespace aos::common::imagemanager + +#endif diff --git a/src/core/common/imagemanager/imagemanager.cpp b/src/core/common/imagemanager/imagemanager.cpp new file mode 100644 index 000000000..5a59fa8df --- /dev/null +++ b/src/core/common/imagemanager/imagemanager.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "imagemanager.hpp" + +namespace aos::common::imagemanager { + +/*********************************************************************************************************************** + * Public + **********************************************************************************************************************/ + +Error ImageManager::Start() +{ + LOG_DBG() << "Start image manager"; + + return ErrorEnum::eNotSupported; +} + +Error ImageManager::Stop() +{ + LOG_DBG() << "Stop image manager"; + + return ErrorEnum::eNotSupported; +} + +Error ImageManager::GetItemsStatuses(Array& statuses) +{ + (void)statuses; + + LOG_DBG() << "Get items statuses"; + + return ErrorEnum::eNotSupported; +} + +Error ImageManager::SubscribeListener(ItemStatusListenerItf& listener) +{ + (void)listener; + + LOG_DBG() << "Subscribe listener"; + + return ErrorEnum::eNotSupported; +} + +Error ImageManager::UnsubscribeListener(ItemStatusListenerItf& listener) +{ + (void)listener; + + LOG_DBG() << "Unsubscribe listener"; + + return ErrorEnum::eNotSupported; +} + +Error ImageManager::GetBlobPath(const String& digest, String& path) const +{ + (void)path; + + LOG_DBG() << "Get blob path" << Log::Field("digest", digest); + + return ErrorEnum::eNotSupported; +} + +/*********************************************************************************************************************** + * Protected + **********************************************************************************************************************/ + +Error ImageManager::Init(const Config& config, DownloadBlobInfoProviderItf& blobInfoProvider, StorageItf& storage, + spaceallocator::SpaceAllocatorItf& spaceAllocator, fs::FileInfoProviderItf& fileInfoProvider, + oci::OCISpecItf& ociSpec, fs::FSPlatformItf& fsPlatform, AllocatorItf& allocator) +{ + LOG_DBG() << "Init image manager"; + + mConfig = config; + mBlobInfoProvider = &blobInfoProvider; + mStorage = &storage; + mSpaceAllocator = &spaceAllocator; + mFileInfoProvider = &fileInfoProvider; + mOCISpec = &ociSpec; + mFSPlatform = &fsPlatform; + mAllocator = &allocator; + + LOG_DBG() << "Config" << Log::Field("imagePath", mConfig.mImagePath) << Log::Field("partLimit", mConfig.mPartLimit) + << Log::Field("itemCacheTTL", mConfig.mItemCacheTTL) + << Log::Field("removeOutdatedPeriod", mConfig.mRemoveOutdatedPeriod); + + return ErrorEnum::eNone; +} + +Error ImageManager::InstallItem(const InstallItemInfo& itemInfo) +{ + LOG_DBG() << "Install item" << Log::Field("itemID", itemInfo.mItemID) << Log::Field("version", itemInfo.mVersion) + << Log::Field("rootDigest", itemInfo.mRootDigest); + + return ErrorEnum::eNotSupported; +} + +Error ImageManager::RemoveItem(const String& itemID, const String& version) +{ + LOG_DBG() << "Remove item" << Log::Field("itemID", itemID) << Log::Field("version", version); + + return ErrorEnum::eNotSupported; +} + +Error ImageManager::Cancel() +{ + LOG_DBG() << "Cancel"; + + return ErrorEnum::eNotSupported; +} + +/*********************************************************************************************************************** + * Private + **********************************************************************************************************************/ + +RetWithError ImageManager::RemoveOutdatedItem(const String& itemID, const String& version) +{ + LOG_DBG() << "Remove outdated item" << Log::Field("itemID", itemID) << Log::Field("version", version); + + return {0, ErrorEnum::eNotSupported}; +} + +} // namespace aos::common::imagemanager diff --git a/src/core/common/imagemanager/imagemanager.hpp b/src/core/common/imagemanager/imagemanager.hpp new file mode 100644 index 000000000..65a614d15 --- /dev/null +++ b/src/core/common/imagemanager/imagemanager.hpp @@ -0,0 +1,269 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_IMAGEMANAGER_IMAGEMANAGER_HPP_ +#define AOS_CORE_COMMON_IMAGEMANAGER_IMAGEMANAGER_HPP_ + +#include +#include +#include +#include +#include + +#include "itf/blobinfoprovider.hpp" +#include "itf/itemstatusprovider.hpp" +#include "itf/storage.hpp" + +#include "config.hpp" + +namespace aos::common::imagemanager { + +/** + * Common image manager. + * + * Implements the part of the image manager shared by CM and SM. CM and SM image managers are derived from this class + * and specialize blob download and layer handling. + */ +class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderItf, private NonCopyable { +public: + /** + * Destructor. + */ + virtual ~ImageManager() = default; + + /** + * Starts image manager. + * + * @return Error. + */ + virtual Error Start(); + + /** + * Stops image manager. Cancels all items being installed and waits until all InstallItem and RemoveItem calls in + * progress return. + * + * @return Error. + */ + virtual Error Stop(); + + /** + * Retrieves deployable items statuses. Reports one status per item version in the storage. + * + * @param[out] statuses list of deployable items statuses. + * @return Error. + */ + Error GetItemsStatuses(Array& statuses) override; + + /** + * Subscribes status notifications. + * + * @param listener status listener. + * @return Error: eAlreadyExist if the listener is already subscribed. + */ + Error SubscribeListener(ItemStatusListenerItf& listener) override; + + /** + * Unsubscribes from status notifications. + * + * @param listener status listener. + * @return Error: eNotFound if the listener is not subscribed. + */ + Error UnsubscribeListener(ItemStatusListenerItf& listener) override; + + /** + * Returns local blob path. + * + * @param digest blob digest. + * @param[out] path local blob path. + * @return Error. + */ + Error GetBlobPath(const String& digest, String& path) const override; + + /** + * Returns item remover which should be passed to the space allocator to remove outdated items. + * + * @return spaceallocator::ItemRemoverItf&. + */ + spaceallocator::ItemRemoverItf& GetItemRemover() { return mItemRemover; } + +protected: + /** + * Install item info. + */ + struct InstallItemInfo { + StaticString mItemID; + UpdateItemType mType; + StaticString mVersion; + StaticString mRootDigest; + bool mIsRootIndex {}; + }; + + /** + * Initializes image manager. Derived classes call it from their own Init. + * + * @param config image manager config. + * @param downloadBlobInfoProvider download blob info provider. + * @param storage image manager storage. + * @param spaceAllocator space allocator. + * @param fileInfoProvider file info provider. + * @param ociSpec OCI spec interface. + * @param fsPlatform file system platform interface. + * @param allocator allocator for temporary objects. + * @return Error. + */ + Error Init(const Config& config, DownloadBlobInfoProviderItf& downloadBlobInfoProvider, StorageItf& storage, + spaceallocator::SpaceAllocatorItf& spaceAllocator, fs::FileInfoProviderItf& fileInfoProvider, + oci::OCISpecItf& ociSpec, fs::FSPlatformItf& fsPlatform, AllocatorItf& allocator); + + /** + * Installs item. Waits while the same item version is being installed or removed, or while + * cMaxNumConcurrentItems items are being processed. + * + * @param itemInfo item info. + * @return Error. + */ + Error InstallItem(const InstallItemInfo& itemInfo); + + /** + * Removes item. Sets the item state to eRemoved in the storage, keeping the item blobs, and adds the item as + * outdated to the space allocator. + * + * @param itemID item ID. + * @param version item version. + * @return Error. + */ + Error RemoveItem(const String& itemID, const String& version); + + /** + * Cancels all items being installed. Waits until all InstallItem and RemoveItem calls in progress return. + * + * @return Error. + */ + Error Cancel(); + + /** + * Returns space allocator. + * + * @return spaceallocator::SpaceAllocatorItf&. + */ + spaceallocator::SpaceAllocatorItf& GetSpaceAllocator() const { return *mSpaceAllocator; } + +private: + static constexpr auto cBlobsFolder = "blobs"; + static constexpr auto cBlobHashSize = Max(crypto::cSHA256Size, crypto::cSHA512Size); + static constexpr auto cMaxNumListeners = 4; + static constexpr auto cMaxNumStoredItems = cMaxNumUpdateItems * 2; + static constexpr auto cMaxNumItemBlobs = 1 + oci::cMaxNumManifests * (3 + oci::cMaxNumLayers); + static constexpr auto cRetryTimeout = Time::cSeconds * 2; + + /** + * Downloads blob. Called by InstallItem, derived classes implement it but must not call it directly. + * + * @param path destination blob path. + * @param blobInfo blob information. + * @return Error. + */ + virtual Error DownloadBlob(const String& path, const BlobInfo& blobInfo) = 0; + + /** + * Cancels ongoing blob download. + * + * @param digest blob digest. + * @return Error: eNotFound if no download is in progress for the blob. + */ + virtual Error CancelDownload(const String& digest) = 0; + + /** + * Validates installed layer. Called by the image manager after the layer blob is validated, derived classes can + * override it to validate unpacked layer data. Returns eNotSupported by default, which the image manager treats as + * nothing to do (skip). + * + * @param path layer blob path. + * @param descriptor layer descriptor. + * @param diffID diff ID, empty if item has no image config. + * @return Error. + */ + virtual Error ValidateLayer(const String& path, const oci::ContentDescriptor& descriptor, const String& diffID) + { + (void)path; + (void)descriptor; + (void)diffID; + + return ErrorEnum::eNotSupported; + } + + /** + * Unpacks installed layer. Called by the image manager after the layer blob is installed, derived classes can + * override it to unpack layer data. Returns eNotSupported by default, which the image manager treats as nothing to + * do (skip). + * + * @param path layer blob path. + * @param descriptor layer descriptor. + * @param diffID diff ID, empty if item has no image config. + * @return Error. + */ + virtual Error UnpackLayer(const String& path, const oci::ContentDescriptor& descriptor, const String& diffID) + { + (void)path; + (void)descriptor; + (void)diffID; + + return ErrorEnum::eNotSupported; + } + + /** + * Removes installed layer. Called by the image manager before the layer blob is removed, derived classes can + * override it to remove unpacked layer data. Returns eNotSupported by default, which the image manager treats as + * nothing to do (skip). + * + * @param path layer blob path. + * @param descriptor layer descriptor. + * @param diffID diff ID, empty if item has no image config. + * @return RetWithError: size of the unpacked layer data freed. + */ + virtual RetWithError RemoveLayer( + const String& path, const oci::ContentDescriptor& descriptor, const String& diffID) + { + (void)path; + (void)descriptor; + (void)diffID; + + return {0, ErrorEnum::eNotSupported}; + } + + class ItemRemover : public spaceallocator::ItemRemoverItf { + public: + explicit ItemRemover(ImageManager& imageManager) + : mImageManager(imageManager) + { + } + + RetWithError RemoveItem(const String& itemID, const String& version) override + { + return mImageManager.RemoveOutdatedItem(itemID, version); + } + + private: + ImageManager& mImageManager; + }; + + RetWithError RemoveOutdatedItem(const String& itemID, const String& version); + + Config mConfig; + DownloadBlobInfoProviderItf* mBlobInfoProvider {}; + StorageItf* mStorage {}; + spaceallocator::SpaceAllocatorItf* mSpaceAllocator {}; + fs::FileInfoProviderItf* mFileInfoProvider {}; + oci::OCISpecItf* mOCISpec {}; + fs::FSPlatformItf* mFSPlatform {}; + AllocatorItf* mAllocator {}; + + ItemRemover mItemRemover {*this}; +}; + +} // namespace aos::common::imagemanager + +#endif diff --git a/src/core/common/imagemanager/itf/blobinfoprovider.hpp b/src/core/common/imagemanager/itf/blobinfoprovider.hpp new file mode 100644 index 000000000..d3428d7b7 --- /dev/null +++ b/src/core/common/imagemanager/itf/blobinfoprovider.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_IMAGEMANAGER_ITF_BLOBINFOPROVIDER_HPP_ +#define AOS_CORE_COMMON_IMAGEMANAGER_ITF_BLOBINFOPROVIDER_HPP_ + +#include + +namespace aos::common::imagemanager { + +/** + * Interface that provides download blob info. + */ +class DownloadBlobInfoProviderItf { +public: + /** + * Destructor. + */ + virtual ~DownloadBlobInfoProviderItf() = default; + + /** + * Returns blob info. + * + * @param digest blob digest. + * @param[out] blobInfo blob info. + * @return Error. + */ + virtual Error GetBlobInfo(const String& digest, BlobInfo& blobInfo) = 0; +}; + +/** + * Interface that provides local blob info. + */ +class LocalBlobInfoProviderItf { +public: + /** + * Destructor. + */ + virtual ~LocalBlobInfoProviderItf() = default; + + /** + * Returns local blob path. + * + * @param digest blob digest. + * @param[out] path local blob path. + * @return Error. + */ + virtual Error GetBlobPath(const String& digest, String& path) const = 0; +}; + +} // namespace aos::common::imagemanager + +#endif diff --git a/src/core/common/imagemanager/itf/itemstatusprovider.hpp b/src/core/common/imagemanager/itf/itemstatusprovider.hpp new file mode 100644 index 000000000..1c0e22b4c --- /dev/null +++ b/src/core/common/imagemanager/itf/itemstatusprovider.hpp @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_IMAGEMANAGER_ITF_ITEMSTATUSPROVIDER_HPP_ +#define AOS_CORE_COMMON_IMAGEMANAGER_ITF_ITEMSTATUSPROVIDER_HPP_ + +#include + +namespace aos::common::imagemanager { + +/** + * Interface for receiving notification about update image statuses. + */ +class ItemStatusListenerItf { +public: + /** + * Destructor. + */ + virtual ~ItemStatusListenerItf() = default; + + /** + * Notifies about item status change. + * + * @param status update item status. + */ + virtual void OnItemStatusChanged(const UpdateItemStatus& status) = 0; + + /** + * Notifies about item removal from cache. + * + * @param id removed item id. + * @param version removed item version. + */ + virtual void OnItemRemovedFromCache(const String& id, const String& version) = 0; +}; + +/** + * Interface to provide deployable item statuses. + */ +class ItemStatusProviderItf { +public: + /** + * Destructor. + */ + virtual ~ItemStatusProviderItf() = default; + + /** + * Retrieves deployable items statuses. + * + * @param[out] statuses list of deployable items statuses. + * @return Error. + */ + virtual Error GetItemsStatuses(Array& statuses) = 0; + + /** + * Subscribes status notifications. + * + * @param listener status listener. + * @return Error. + */ + virtual Error SubscribeListener(ItemStatusListenerItf& listener) = 0; + + /** + * Unsubscribes from status notifications. + * + * @param listener status listener. + * @return Error. + */ + virtual Error UnsubscribeListener(ItemStatusListenerItf& listener) = 0; +}; + +} // namespace aos::common::imagemanager + +#endif diff --git a/src/core/common/imagemanager/itf/storage.hpp b/src/core/common/imagemanager/itf/storage.hpp new file mode 100644 index 000000000..892fc87b5 --- /dev/null +++ b/src/core/common/imagemanager/itf/storage.hpp @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_IMAGEMANAGER_ITF_STORAGE_HPP_ +#define AOS_CORE_COMMON_IMAGEMANAGER_ITF_STORAGE_HPP_ + +#include +#include +#include + +namespace aos::common::imagemanager { + +/** + * Deployable item info. + */ +struct DeployableItemInfo : public UpdateItemStatus { + StaticString mRootDigest; + Time mTimestamp; + + /** + * Compares deployable item info. + * + * @param lhs left object. + * @param rhs right object. + * @return bool. + */ + friend bool operator==(const DeployableItemInfo& lhs, const DeployableItemInfo& rhs) + { + return (static_cast(lhs) == static_cast(rhs)) + && lhs.mRootDigest == rhs.mRootDigest && lhs.mTimestamp == rhs.mTimestamp; + } + + /** + * Compares deployable item info. + * + * @param lhs left object. + * @param rhs right object. + * @return bool. + */ + friend bool operator!=(const DeployableItemInfo& lhs, const DeployableItemInfo& rhs) { return !(lhs == rhs); } +}; + +/** + * Item blob info. + */ +struct ItemBlobInfo : public oci::ContentDescriptor { + bool mIsLayer {}; + StaticString mDiffID; + + /** + * Compares item blob info. + * + * @param lhs left object. + * @param rhs right object. + * @return bool. + */ + friend bool operator==(const ItemBlobInfo& lhs, const ItemBlobInfo& rhs) + { + return (static_cast(lhs) == static_cast(rhs)) + && lhs.mIsLayer == rhs.mIsLayer && lhs.mDiffID == rhs.mDiffID; + } + + /** + * Compares item blob info. + * + * @param lhs left object. + * @param rhs right object. + * @return bool. + */ + friend bool operator!=(const ItemBlobInfo& lhs, const ItemBlobInfo& rhs) { return !(lhs == rhs); } +}; + +/** + * Image manager storage interface. + */ +class StorageItf { +public: + /** + * Destructor. + */ + virtual ~StorageItf() = default; + + /** + * Adds deployable item. + * + * @param item deployable item info. + * @return Error: eAlreadyExist if item with the same ID and version exists. + */ + virtual Error AddDeployableItem(const DeployableItemInfo& item) = 0; + + /** + * Updates deployable item. Blob references of the item are kept. + * + * @param item deployable item info. + * @return Error: eNotFound if item doesn't exist. + */ + virtual Error UpdateDeployableItem(const DeployableItemInfo& item) = 0; + + /** + * Removes deployable item and all its blob references. + * + * @param itemID item ID. + * @param version item version. + * @return Error: eNotFound if item doesn't exist. + */ + virtual Error RemoveDeployableItem(const String& itemID, const String& version) = 0; + + /** + * Returns deployable item. + * + * @param itemID item ID. + * @param version item version. + * @param[out] item deployable item info. + * @return Error: eNotFound if item doesn't exist. + */ + virtual Error GetDeployableItem(const String& itemID, const String& version, DeployableItemInfo& item) = 0; + + /** + * Returns all deployable items. + * + * @param[out] items deployable items. + * @return Error. + */ + virtual Error GetAllDeployableItems(Array& items) = 0; + + /** + * Returns number of stored deployable items in all states. + * + * @return RetWithError. + */ + virtual RetWithError GetDeployableItemsCount() = 0; + + /** + * Adds blob reference to deployable item. Does nothing if the reference already exists. + * + * @param itemID item ID. + * @param version item version. + * @param blob blob info. + * @return Error: eNotFound if item doesn't exist. + */ + virtual Error AddItemBlob(const String& itemID, const String& version, const ItemBlobInfo& blob) = 0; + + /** + * Returns blobs referenced by deployable item. + * + * @param itemID item ID. + * @param version item version. + * @param[out] blobs blobs info. + * @return Error: eNotFound if item doesn't exist. + */ + virtual Error GetItemBlobs(const String& itemID, const String& version, Array& blobs) = 0; + + /** + * Checks if blob is referenced by any existing deployable item, regardless of the item state. Blob references + * left without an item are ignored. + * + * @param digest blob digest. + * @return RetWithError. + */ + virtual RetWithError IsBlobUsed(const String& digest) = 0; +}; + +} // namespace aos::common::imagemanager + +#endif diff --git a/src/core/common/imagemanager/tests/CMakeLists.txt b/src/core/common/imagemanager/tests/CMakeLists.txt new file mode 100644 index 000000000..09df6587d --- /dev/null +++ b/src/core/common/imagemanager/tests/CMakeLists.txt @@ -0,0 +1,35 @@ +# +# Copyright (C) 2026 EPAM Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME imagemanager_test) + +# ###################################################################################################################### +# Sources +# ###################################################################################################################### + +set(SOURCES imagemanager.cpp) + +# ###################################################################################################################### +# Libraries +# ###################################################################################################################### + +set(LIBRARIES aos::core::common::imagemanager aos::core::common::tests::mocks aos::core::common::tests::utils + GTest::gmock_main +) + +# ###################################################################################################################### +# Target +# ###################################################################################################################### + +add_test( + TARGET_NAME + ${TARGET_NAME} + LOG_MODULE + SOURCES + ${SOURCES} + LIBRARIES + ${LIBRARIES} +) diff --git a/src/core/common/imagemanager/tests/imagemanager.cpp b/src/core/common/imagemanager/tests/imagemanager.cpp new file mode 100644 index 000000000..d447e633e --- /dev/null +++ b/src/core/common/imagemanager/tests/imagemanager.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "stubs/storagestub.hpp" + +using namespace testing; + +namespace aos::common::imagemanager { + +namespace { + +/*********************************************************************************************************************** + * Consts + **********************************************************************************************************************/ + +constexpr auto cTestImagePath = "/tmp/common_imagemanager_test/images"; + +/*********************************************************************************************************************** + * Test image manager + **********************************************************************************************************************/ + +class TestImageManager : public ImageManager { +public: + using ImageManager::Init; + using ImageManager::InstallItemInfo; + + Error Install(const InstallItemInfo& itemInfo) { return InstallItem(itemInfo); } + Error Remove(const String& itemID, const String& version) { return RemoveItem(itemID, version); } + + MOCK_METHOD(Error, DownloadBlob, (const String& path, const BlobInfo& blobInfo), (override)); + MOCK_METHOD(Error, CancelDownload, (const String& digest), (override)); +}; + +} // namespace + +/*********************************************************************************************************************** + * Suite + **********************************************************************************************************************/ + +class CommonImageManagerTest : public Test { +protected: + static void SetUpTestSuite() { tests::utils::InitLog(); } + + HeapAllocator mAllocator; + NiceMock mBlobInfoProvider; + StorageStub mStorage; + NiceMock mSpaceAllocator; + NiceMock mFileInfoProvider; + NiceMock mOCISpec; + NiceMock mFSPlatform; + NiceMock mListener; + NiceMock mImageManager; +}; + +/*********************************************************************************************************************** + * Tests + **********************************************************************************************************************/ + +TEST_F(CommonImageManagerTest, Init) +{ + Config config {cTestImagePath, 0, 10 * Time::cSeconds, 5 * Time::cSeconds}; + + auto err = mImageManager.Init( + config, mBlobInfoProvider, mStorage, mSpaceAllocator, mFileInfoProvider, mOCISpec, mFSPlatform, mAllocator); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + +} // namespace aos::common::imagemanager diff --git a/src/core/common/imagemanager/tests/stubs/storagestub.hpp b/src/core/common/imagemanager/tests/stubs/storagestub.hpp new file mode 100644 index 000000000..e24932f77 --- /dev/null +++ b/src/core/common/imagemanager/tests/stubs/storagestub.hpp @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_IMAGEMANAGER_TESTS_STUBS_STORAGESTUB_HPP_ +#define AOS_CORE_COMMON_IMAGEMANAGER_TESTS_STUBS_STORAGESTUB_HPP_ + +#include +#include +#include + +#include + +namespace aos::common::imagemanager { + +/** + * Storage stub. + */ +class StorageStub : public StorageItf { +public: + Error AddDeployableItem(const DeployableItemInfo& item) override + { + std::lock_guard lock {mMutex}; + + if (Find(item.mItemID, item.mVersion) != mItems.end()) { + return ErrorEnum::eAlreadyExist; + } + + mItems.push_back({item, {}}); + + return ErrorEnum::eNone; + } + + Error UpdateDeployableItem(const DeployableItemInfo& item) override + { + std::lock_guard lock {mMutex}; + + auto it = Find(item.mItemID, item.mVersion); + if (it == mItems.end()) { + return ErrorEnum::eNotFound; + } + + it->mInfo = item; + + return ErrorEnum::eNone; + } + + Error RemoveDeployableItem(const String& itemID, const String& version) override + { + std::lock_guard lock {mMutex}; + + auto it = Find(itemID, version); + if (it == mItems.end()) { + return ErrorEnum::eNotFound; + } + + mItems.erase(it); + + return ErrorEnum::eNone; + } + + Error GetDeployableItem(const String& itemID, const String& version, DeployableItemInfo& item) override + { + std::lock_guard lock {mMutex}; + + auto it = Find(itemID, version); + if (it == mItems.end()) { + return ErrorEnum::eNotFound; + } + + item = it->mInfo; + + return ErrorEnum::eNone; + } + + Error GetAllDeployableItems(Array& items) override + { + std::lock_guard lock {mMutex}; + + for (const auto& item : mItems) { + if (auto err = items.PushBack(item.mInfo); !err.IsNone()) { + return err; + } + } + + return ErrorEnum::eNone; + } + + RetWithError GetDeployableItemsCount() override + { + std::lock_guard lock {mMutex}; + + return mItems.size(); + } + + Error AddItemBlob(const String& itemID, const String& version, const ItemBlobInfo& blob) override + { + std::lock_guard lock {mMutex}; + + auto it = Find(itemID, version); + if (it == mItems.end()) { + return ErrorEnum::eNotFound; + } + + if (std::none_of(it->mBlobs.begin(), it->mBlobs.end(), + [&](const ItemBlobInfo& itemBlob) { return itemBlob.mDigest == blob.mDigest; })) { + it->mBlobs.push_back(blob); + } + + return ErrorEnum::eNone; + } + + Error GetItemBlobs(const String& itemID, const String& version, Array& blobs) override + { + std::lock_guard lock {mMutex}; + + auto it = Find(itemID, version); + if (it == mItems.end()) { + return ErrorEnum::eNotFound; + } + + for (const auto& blob : it->mBlobs) { + if (auto err = blobs.PushBack(blob); !err.IsNone()) { + return err; + } + } + + return ErrorEnum::eNone; + } + + RetWithError IsBlobUsed(const String& digest) override + { + std::lock_guard lock {mMutex}; + + return std::any_of(mItems.begin(), mItems.end(), [&](const Item& item) { + return std::any_of(item.mBlobs.begin(), item.mBlobs.end(), + [&](const ItemBlobInfo& blob) { return blob.mDigest == digest; }); + }); + } + + std::vector GetItems() + { + std::lock_guard lock {mMutex}; + + std::vector items; + + for (const auto& item : mItems) { + items.push_back(item.mInfo); + } + + return items; + } + +private: + struct Item { + DeployableItemInfo mInfo; + std::vector mBlobs; + }; + + std::vector::iterator Find(const String& itemID, const String& version) + { + return std::find_if(mItems.begin(), mItems.end(), + [&](const Item& item) { return item.mInfo.mItemID == itemID && item.mInfo.mVersion == version; }); + } + + std::mutex mMutex; + std::vector mItems; +}; + +} // namespace aos::common::imagemanager + +#endif diff --git a/src/core/common/tests/mocks/CMakeLists.txt b/src/core/common/tests/mocks/CMakeLists.txt index 1c2deba9e..f13d13502 100644 --- a/src/core/common/tests/mocks/CMakeLists.txt +++ b/src/core/common/tests/mocks/CMakeLists.txt @@ -16,6 +16,7 @@ set(TARGET_NAME mocks) set(HEADERS alertsmock.hpp + blobinfoprovidermock.hpp certprovidermock.hpp cloudconnectionmock.hpp cryptomock.hpp @@ -24,6 +25,7 @@ set(HEADERS fsmock.hpp identprovidermock.hpp instancestatusprovidermock.hpp + itemstatuslistenermock.hpp logprovidermock.hpp monitoringmock.hpp nodehandlermock.hpp diff --git a/src/core/common/tests/mocks/blobinfoprovidermock.hpp b/src/core/common/tests/mocks/blobinfoprovidermock.hpp new file mode 100644 index 000000000..290029e61 --- /dev/null +++ b/src/core/common/tests/mocks/blobinfoprovidermock.hpp @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_TESTS_MOCKS_BLOBINFOPROVIDERMOCK_HPP_ +#define AOS_CORE_COMMON_TESTS_MOCKS_BLOBINFOPROVIDERMOCK_HPP_ + +#include + +#include + +namespace aos::common::imagemanager { + +/** + * Download blob info provider mock. + */ +class DownloadBlobInfoProviderMock : public DownloadBlobInfoProviderItf { +public: + MOCK_METHOD(Error, GetBlobInfo, (const String& digest, BlobInfo& blobInfo), (override)); +}; + +/** + * Local blob info provider mock. + */ +class LocalBlobInfoProviderMock : public LocalBlobInfoProviderItf { +public: + MOCK_METHOD(Error, GetBlobPath, (const String& digest, String& path), (const, override)); +}; + +} // namespace aos::common::imagemanager + +#endif diff --git a/src/core/common/tests/mocks/itemstatuslistenermock.hpp b/src/core/common/tests/mocks/itemstatuslistenermock.hpp new file mode 100644 index 000000000..50e7a95ce --- /dev/null +++ b/src/core/common/tests/mocks/itemstatuslistenermock.hpp @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_TESTS_MOCKS_ITEMSTATUSLISTENERMOCK_HPP_ +#define AOS_CORE_COMMON_TESTS_MOCKS_ITEMSTATUSLISTENERMOCK_HPP_ + +#include + +#include + +namespace aos::common::imagemanager { + +/** + * Item status listener mock. + */ +class ItemStatusListenerMock : public ItemStatusListenerItf { +public: + MOCK_METHOD(void, OnItemStatusChanged, (const UpdateItemStatus& status), (override)); + MOCK_METHOD(void, OnItemRemovedFromCache, (const String& id, const String& version), (override)); +}; + +} // namespace aos::common::imagemanager + +#endif From d278ff63f69b4d2ede09c4751d63e11ccb725fa9 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 29 Oct 2026 17:55:46 +0200 Subject: [PATCH 04/54] common: tools: add eIntegrity error Add eIntegrity error to the error enum to report integrity issues. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/common/tools/error.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/common/tools/error.hpp b/src/core/common/tools/error.hpp index 3a0d919fc..ed8ead1b6 100644 --- a/src/core/common/tools/error.hpp +++ b/src/core/common/tools/error.hpp @@ -60,6 +60,7 @@ class Error { // NOSONAR cpp:S3624 - non-owning members, nothing to release eNotSupported, eEOF, eCanceled, + eIntegrity, eNumErrors }; @@ -328,6 +329,7 @@ class Error { // NOSONAR cpp:S3624 - non-owning members, nothing to release "not supported", "EOF", "canceled", + "integrity error", }; return Pair(sErrorTypeStrings, ArraySize(sErrorTypeStrings)); From 6fdeac965e69764e4cd534cf13be5731311a528c Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Fri, 30 Oct 2026 09:49:01 +0200 Subject: [PATCH 05/54] common: spaceallocator: protect outdated items with partition lock Signed-off-by: Mykola Solianko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../spaceallocator/itf/spaceallocator.hpp | 3 + .../common/spaceallocator/spaceallocator.hpp | 232 +++++++---------- .../spaceallocator/tests/spaceallocator.cpp | 233 ++++++++++++++++++ 3 files changed, 321 insertions(+), 147 deletions(-) diff --git a/src/core/common/spaceallocator/itf/spaceallocator.hpp b/src/core/common/spaceallocator/itf/spaceallocator.hpp index a9c8d7ce8..e0830585b 100644 --- a/src/core/common/spaceallocator/itf/spaceallocator.hpp +++ b/src/core/common/spaceallocator/itf/spaceallocator.hpp @@ -19,6 +19,9 @@ class ItemRemoverItf { /** * Removes item. * + * It is called with the space allocator lock held, so the implementation must not call the methods of any space + * allocator of the same partition and must not take a lock that is held by a caller of the space allocator methods. + * * @param id item id. * @param version item version. * @return RetWithError. diff --git a/src/core/common/spaceallocator/spaceallocator.hpp b/src/core/common/spaceallocator/spaceallocator.hpp index 3d7ebced2..aaadbb063 100644 --- a/src/core/common/spaceallocator/spaceallocator.hpp +++ b/src/core/common/spaceallocator/spaceallocator.hpp @@ -35,6 +35,10 @@ struct OutdatedItem { /** * Partition. + * + * Partition methods must be called with the partition mutex locked, except AddLimit and RemoveLimit which are + * protected by the partitions mutex. The partition mutex also protects the state of the space allocators sharing the + * partition. */ class Partition { public: @@ -51,8 +55,6 @@ class Partition { */ Error AddLimit(size_t limit) { - LockGuard lock {mMutex}; - if (mLimit + limit > cMaxLimit) { return Error(ErrorEnum::eNoMemory, "partition limit exceeded"); } @@ -70,8 +72,6 @@ class Partition { */ Error RemoveLimit(size_t limit) { - LockGuard lock {mMutex}; - if (mLimit < limit) { return Error(ErrorEnum::eNoMemory, "partition limit exceeded"); } @@ -89,8 +89,6 @@ class Partition { */ Error Allocate(size_t size) { - LockGuard lock {mMutex}; - if (mAllocationCount == 0) { auto [availableSize, err] = mPlatformFS->GetAvailableSize(mMountPoint); if (!err.IsNone()) { @@ -101,17 +99,10 @@ class Partition { } if (size > mAvailableSize) { - if (mOutdatedItems.Size() == 0) { - return Error(ErrorEnum::eNoMemory, "not enough space"); - } - - auto [freedSize, err] = RemoveOutdatedItems(size - mAvailableSize); - if (!err.IsNone()) { + if (auto err = RemoveOutdatedItems(size - mAvailableSize).mError; !err.IsNone()) { return err; } - mAvailableSize += freedSize; - if (size > mAvailableSize) { return Error(ErrorEnum::eNoMemory, "not enough space"); } @@ -129,16 +120,7 @@ class Partition { * @param size size to free. * @return void. */ - void Free(size_t size) - { - LockGuard lock {mMutex}; - - if (mAllocationCount == 0) { - return; - } - - mAvailableSize += size; - } + void Free(size_t size) { mAvailableSize += size; } /** * Allocate done. @@ -147,8 +129,6 @@ class Partition { */ Error Done() { - LockGuard lock {mMutex}; - if (mAllocationCount == 0) { return Error(ErrorEnum::eNotFound, "no allocation"); } @@ -167,8 +147,6 @@ class Partition { */ Error Resize(int64_t deltaSize) { - LockGuard lock {mMutex}; - if (deltaSize <= 0) { mAvailableSize += static_cast(-deltaSize); @@ -178,17 +156,10 @@ class Partition { const auto extra = static_cast(deltaSize); if (extra > mAvailableSize) { - if (mOutdatedItems.Size() == 0) { - return Error(ErrorEnum::eNoMemory, "not enough space"); - } - - auto [freedSize, err] = RemoveOutdatedItems(extra - mAvailableSize); - if (!err.IsNone()) { + if (auto err = RemoveOutdatedItems(extra - mAvailableSize).mError; !err.IsNone()) { return err; } - mAvailableSize += freedSize; - if (extra > mAvailableSize) { return Error(ErrorEnum::eNoMemory, "not enough space"); } @@ -207,8 +178,6 @@ class Partition { */ Error AddOutdatedItem(const OutdatedItem& item) { - LockGuard lock {mMutex}; - for (auto& outdatedItem : mOutdatedItems) { if (outdatedItem.mID == item.mID && outdatedItem.mVersion == item.mVersion) { outdatedItem = item; @@ -219,19 +188,14 @@ class Partition { auto err = mOutdatedItems.PushBack(item); if (!err.IsNone()) { - mOutdatedItems.Sort( - [](const OutdatedItem& a, const OutdatedItem& b) { return a.mTimestamp < b.mTimestamp; }); + SortOutdatedItems(); auto& oldestItem = mOutdatedItems[0]; - auto [size, removeErr] = oldestItem.mRemover->RemoveItem(oldestItem.mID, oldestItem.mVersion); - if (!removeErr.IsNone()) { + if (auto removeErr = RemoveItem(oldestItem).mError; !removeErr.IsNone()) { return removeErr; } - oldestItem.mFreeCallback(reinterpret_cast(size)); - mAvailableSize += size; - oldestItem = item; return ErrorEnum::eNone; @@ -249,8 +213,6 @@ class Partition { */ void RestoreOutdatedItem(const String& id, const String& version) { - LockGuard lock {mMutex}; - for (size_t i = 0; i < mOutdatedItems.Size(); i++) { if (mOutdatedItems[i].mID == id && mOutdatedItems[i].mVersion == version) { mOutdatedItems.Erase(mOutdatedItems.begin() + i); @@ -263,18 +225,15 @@ class Partition { /** * Removes the oldest outdated item of the space allocator. * - * The item is removed from the outdated items list whatever the result of the item remover. The freed size is - * accounted in the partition only: the allocator free callback is not called, the caller must account the freed - * size in the space allocator. + * The item is removed from the outdated items list whatever the result of the item remover. On success, the freed + * size is accounted in the partition and in the space allocator the item belongs to. * * @param spaceAllocator space allocator the item belongs to. * @return RetWithError: freed size, eNotFound if there is no outdated item to remove. */ RetWithError RemoveOutdatedItem(const SpaceAllocatorItf* spaceAllocator) { - LockGuard lock {mMutex}; - - mOutdatedItems.Sort([](const OutdatedItem& a, const OutdatedItem& b) { return a.mTimestamp < b.mTimestamp; }); + SortOutdatedItems(); auto oldestItem = mOutdatedItems.FindIf( [spaceAllocator](const OutdatedItem& item) { return item.mSpaceAllocator == spaceAllocator; }); @@ -282,7 +241,7 @@ class Partition { return {0, Error(ErrorEnum::eNotFound, "no outdated item")}; } - auto [size, err] = oldestItem->mRemover->RemoveItem(oldestItem->mID, oldestItem->mVersion); + auto [size, err] = RemoveItem(*oldestItem); mOutdatedItems.Erase(oldestItem); @@ -290,11 +249,47 @@ class Partition { return {0, err}; } - mAvailableSize += size; - return {size, ErrorEnum::eNone}; } + /** + * Removes outdated items starting from the oldest one until the requested size is freed. + * + * The freed size of each removed item is accounted in the partition and in the space allocator the item belongs + * to. On item remover error, the already removed items stay removed and the failed item stays in the list. + * + * @param size size to free. + * @param spaceAllocator if set, only items of this space allocator are removed. + * @return RetWithError: freed size. + */ + RetWithError RemoveOutdatedItems(size_t size, const SpaceAllocatorItf* spaceAllocator = nullptr) + { + SortOutdatedItems(); + + size_t freedSize = 0; + + for (size_t i = 0; freedSize < size && i < mOutdatedItems.Size();) { + auto& item = mOutdatedItems[i]; + + if (spaceAllocator != nullptr && item.mSpaceAllocator != spaceAllocator) { + i++; + + continue; + } + + auto [itemSize, err] = RemoveItem(item); + if (!err.IsNone()) { + return {freedSize, err}; + } + + freedSize += itemSize; + + mOutdatedItems.Erase(mOutdatedItems.begin() + i); + } + + return {freedSize, ErrorEnum::eNone}; + } + static constexpr size_t cMaxNumOutdatedItems = AOS_CONFIG_SPACEALLOCATOR_MAX_OUTDATED_ITEMS; StaticString mMountPoint; @@ -302,39 +297,31 @@ class Partition { size_t mAllocatorCount {}; fs::FSPlatformItf* mPlatformFS {}; StaticArray mOutdatedItems; + Mutex mMutex {}; private: static constexpr size_t cMaxLimit = 100; - RetWithError RemoveOutdatedItems(size_t size) + void SortOutdatedItems() { mOutdatedItems.Sort([](const OutdatedItem& a, const OutdatedItem& b) { return a.mTimestamp < b.mTimestamp; }); + } - size_t freedSize = 0; - size_t i = 0; - - for (; freedSize < size && i < mOutdatedItems.Size(); i++) { - auto& item = mOutdatedItems[i]; - - auto [itemSize, err] = item.mRemover->RemoveItem(item.mID, item.mVersion); - if (!err.IsNone()) { - return {freedSize, err}; - } - - item.mFreeCallback(reinterpret_cast(itemSize)); - freedSize += itemSize; + RetWithError RemoveItem(OutdatedItem& item) + { + auto [size, err] = item.mRemover->RemoveItem(item.mID, item.mVersion); + if (!err.IsNone()) { + return {0, err}; } - if (i > 0) { - mOutdatedItems.Erase(mOutdatedItems.begin(), mOutdatedItems.begin() + i); - } + item.mFreeCallback(reinterpret_cast(size)); + mAvailableSize += size; - return {freedSize, ErrorEnum::eNone}; + return {size, ErrorEnum::eNone}; } size_t mAllocationCount {}; size_t mAvailableSize {}; - Mutex mMutex {}; }; /** @@ -486,6 +473,8 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { */ RetWithError> AllocateSpace(size_t size) override { + LockGuard lock {mPartition->mMutex}; + auto err = Allocate(size); if (!err.IsNone()) { return {nullptr, err}; @@ -493,6 +482,7 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { if (err = mPartition->Allocate(size); !err.IsNone()) { Free(size); + Done(); return {nullptr, err}; } @@ -500,7 +490,9 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { auto space = MakeUnique(mAllocator, size, this); if (!space) { mPartition->Free(size); + mPartition->Done(); Free(size); + Done(); return {nullptr, ErrorEnum::eNoMemory}; } @@ -516,6 +508,8 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { */ void FreeSpace(size_t size) override { + LockGuard lock {mPartition->mMutex}; + Free(size); mPartition->Free(size); @@ -528,6 +522,8 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { */ Error AllocateDone() override { + LockGuard lock {mPartition->mMutex}; + if (auto err = Done(); !err.IsNone()) { return err; } @@ -561,6 +557,8 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { item.mFreeCallback = aos::StaticFunction<> {[this](void* sizePtr) { this->Free(reinterpret_cast(sizePtr)); }}; + LockGuard lock {mPartition->mMutex}; + return mPartition->AddOutdatedItem(item); } @@ -573,6 +571,8 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { */ Error RestoreOutdatedItem(const String& id, const String& version) override { + LockGuard lock {mPartition->mMutex}; + mPartition->RestoreOutdatedItem(id, version); return ErrorEnum::eNone; @@ -585,14 +585,9 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { */ RetWithError RemoveOutdatedItem() override { - auto [size, err] = mPartition->RemoveOutdatedItem(this); - if (!err.IsNone()) { - return {0, err}; - } + LockGuard lock {mPartition->mMutex}; - Free(size); - - return {size, ErrorEnum::eNone}; + return mPartition->RemoveOutdatedItem(this); } private: @@ -602,6 +597,8 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { return ErrorEnum::eNone; } + LockGuard lock {mPartition->mMutex}; + Free(oldSize); if (auto err = Allocate(newSize); !err.IsNone()) { @@ -624,7 +621,6 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { partition.mPlatformFS = mPlatformFS; } - // called with mMutex locked Error UpdateSizes() { auto [totalSize, totalSizeErr] = mPlatformFS->GetTotalSize(mPartition->mMountPoint); @@ -646,8 +642,6 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { Error Allocate(size_t size) { - LockGuard lock {mMutex}; - if (mLimitPercent == 0) { return ErrorEnum::eNone; } @@ -659,28 +653,11 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { } if (mAllocatedSize + size > mSizeLimit) { - size_t outdatedCount = 0; - for (const auto& item : mPartition->mOutdatedItems) { - if (item.mSpaceAllocator == this) { - outdatedCount++; - } - } - - if (outdatedCount == 0) { - return Error(ErrorEnum::eNoMemory, "allocator limit exceeded"); - } - - auto [freedSize, err] = RemoveOutdatedItems(mAllocatedSize + size - mSizeLimit); - if (!err.IsNone()) { + if (auto err = mPartition->RemoveOutdatedItems(mAllocatedSize + size - mSizeLimit, this).mError; + !err.IsNone()) { return err; } - if (freedSize > mAllocatedSize) { - mAllocatedSize = 0; - } else { - mAllocatedSize -= freedSize; - } - if (mAllocatedSize + size > mSizeLimit) { return Error(ErrorEnum::eNoMemory, "allocator limit exceeded"); } @@ -692,59 +669,21 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { return ErrorEnum::eNone; } - RetWithError RemoveOutdatedItems(size_t size) - { - mPartition->mOutdatedItems.Sort( - [](const OutdatedItem& a, const OutdatedItem& b) { return a.mTimestamp < b.mTimestamp; }); - - size_t freedSize = 0; - size_t i = 0; - - for (auto& item : mPartition->mOutdatedItems) { - if (item.mSpaceAllocator != this || freedSize >= size) { - mPartition->mOutdatedItems[i] = item; - i++; - - continue; - } - - auto [itemSize, err] = item.mRemover->RemoveItem(item.mID, item.mVersion); - if (!err.IsNone()) { - return {freedSize, err}; - } - - item.mPartition->Free(itemSize); - freedSize += itemSize; - } - - if (i < mPartition->mOutdatedItems.Size()) { - mPartition->mOutdatedItems.Erase(mPartition->mOutdatedItems.begin() + i, mPartition->mOutdatedItems.end()); - } - - return {freedSize, ErrorEnum::eNone}; - } - void Free(size_t size) { - LockGuard lock {mMutex}; - if (mLimitPercent == 0) { return; } - if (mAllocationCount > 0) { - if (size < mAllocatedSize) { - mAllocatedSize -= size; - } else { - mAllocatedSize = 0; - } + if (size < mAllocatedSize) { + mAllocatedSize -= size; + } else { + mAllocatedSize = 0; } } Error Done() { - LockGuard lock {mMutex}; - if (mLimitPercent == 0) { return ErrorEnum::eNone; } @@ -767,7 +706,6 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { ItemRemoverItf* mRemover {}; fs::FSPlatformItf* mPlatformFS {}; Partition* mPartition {}; - Mutex mMutex; }; } // namespace aos::spaceallocator diff --git a/src/core/common/spaceallocator/tests/spaceallocator.cpp b/src/core/common/spaceallocator/tests/spaceallocator.cpp index 62c97acff..82c7c5eb4 100644 --- a/src/core/common/spaceallocator/tests/spaceallocator.cpp +++ b/src/core/common/spaceallocator/tests/spaceallocator.cpp @@ -4,7 +4,10 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include +#include #include +#include #include @@ -12,11 +15,68 @@ #include #include #include +#include using namespace testing; namespace aos::spaceallocator { +namespace { + +/*********************************************************************************************************************** + * Consts + **********************************************************************************************************************/ + +constexpr auto cWaitTimeout = 5 * Time::cSeconds; +constexpr auto cBlockTimeout = std::chrono::milliseconds(100); + +/*********************************************************************************************************************** + * Helpers + **********************************************************************************************************************/ + +class BlockingRemover { +public: + RetWithError Remove(size_t size) + { + UniqueLock lock {mMutex}; + + mEntered = true; + mCondVar.NotifyAll(); + + if (auto err = mCondVar.Wait(lock, cWaitTimeout, [this] { return mReleased; }); !err.IsNone()) { + return {0, err}; + } + + return {size, ErrorEnum::eNone}; + } + + Error WaitEntered() + { + UniqueLock lock {mMutex}; + + return mCondVar.Wait(lock, cWaitTimeout, [this] { return mEntered; }); + } + + void Release() + { + { + LockGuard lock {mMutex}; + + mReleased = true; + } + + mCondVar.NotifyAll(); + } + +private: + Mutex mMutex; + ConditionalVariable mCondVar; + bool mEntered {}; + bool mReleased {}; +}; + +} // namespace + /*********************************************************************************************************************** * Suite **********************************************************************************************************************/ @@ -1032,4 +1092,177 @@ TEST_F(SpaceallocatorTest, RemoveOutdatedItemAfterEviction) ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); } +TEST_F(SpaceallocatorTest, RemoveOutdatedItemAccountingIsAtomic) +{ + constexpr size_t cTotalSize = 100 * cKilobyte; + constexpr size_t cItemSize = 15 * cKilobyte; + + SpaceAllocator<5> mSpaceAllocator; + BlockingRemover blockingRemover; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, 50, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetTotalSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetDirSize(mPath)) + .WillRepeatedly(Return(RetWithError(25 * cKilobyte, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("item1", "1.0.0", Time::Now()).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("item1"), String("1.0.0"))) + .WillOnce( + Invoke([&blockingRemover](const String&, const String&) { return blockingRemover.Remove(cItemSize); })); + + size_t removedSize = 0; + Error removeErr; + Thread<> removeThread; + + ASSERT_TRUE(removeThread + .Run([&](void*) { + auto [size, err] = mSpaceAllocator.RemoveOutdatedItem(); + + removedSize = size; + removeErr = err; + }) + .IsNone()); + + EXPECT_TRUE(blockingRemover.WaitEntered().IsNone()); + + std::atomic_bool allocated {false}; + UniquePtr space1; + Error allocateErr; + Thread<> allocateThread; + + EXPECT_TRUE(allocateThread + .Run([&](void*) { + auto [space, err] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + + space1 = Move(space); + allocateErr = err; + allocated = true; + }) + .IsNone()); + + std::this_thread::sleep_for(cBlockTimeout); + + EXPECT_FALSE(allocated.load()); + + blockingRemover.Release(); + + ASSERT_TRUE(removeThread.Join().IsNone()); + ASSERT_TRUE(allocateThread.Join().IsNone()); + + ASSERT_TRUE(removeErr.IsNone()); + EXPECT_EQ(removedSize, cItemSize); + + ASSERT_TRUE(allocateErr.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + + auto [space2, err2] = mSpaceAllocator.AllocateSpace(15 * cKilobyte + 1); + EXPECT_EQ(err2, ErrorEnum::eNoMemory); + EXPECT_EQ(space2.Get(), nullptr); + + auto [space3, err3] = mSpaceAllocator.AllocateSpace(15 * cKilobyte); + ASSERT_TRUE(err3.IsNone()); + ASSERT_NE(space3.Get(), nullptr); + + ASSERT_TRUE(space3->Accept().IsNone()); + ASSERT_TRUE(space1->Accept().IsNone()); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, OutdatedItemsListIsConsistentDuringEviction) +{ + constexpr size_t cTotalSize = 100 * cKilobyte; + constexpr size_t cItemSize = 15 * cKilobyte; + + SpaceAllocator<5> mSpaceAllocator; + BlockingRemover blockingRemover; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, 50, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetTotalSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetDirSize(mPath)) + .WillRepeatedly(Return(RetWithError(45 * cKilobyte, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + + const Time now = Time::Now(); + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("itemA", "1.0.0", now.Add(-2 * Time::cHours)).IsNone()); + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("itemB", "1.0.0", now.Add(-1 * Time::cHours)).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("itemA"), String("1.0.0"))) + .WillOnce( + Invoke([&blockingRemover](const String&, const String&) { return blockingRemover.Remove(cItemSize); })); + + UniquePtr space; + Error allocateErr; + Thread<> allocateThread; + + ASSERT_TRUE(allocateThread + .Run([&](void*) { + auto [allocatedSpace, err] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + + space = Move(allocatedSpace); + allocateErr = err; + }) + .IsNone()); + + EXPECT_TRUE(blockingRemover.WaitEntered().IsNone()); + + std::atomic_bool updated {false}; + Error restoreErr; + Error addErr; + Thread<> updateThread; + + EXPECT_TRUE(updateThread + .Run([&](void*) { + restoreErr = mSpaceAllocator.RestoreOutdatedItem("itemB", "1.0.0"); + addErr = mSpaceAllocator.AddOutdatedItem("itemC", "1.0.0", now); + updated = true; + }) + .IsNone()); + + std::this_thread::sleep_for(cBlockTimeout); + + EXPECT_FALSE(updated.load()); + + blockingRemover.Release(); + + ASSERT_TRUE(allocateThread.Join().IsNone()); + ASSERT_TRUE(updateThread.Join().IsNone()); + + ASSERT_TRUE(allocateErr.IsNone()); + ASSERT_NE(space.Get(), nullptr); + + ASSERT_TRUE(restoreErr.IsNone()); + ASSERT_TRUE(addErr.IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("itemC"), String("1.0.0"))) + .WillOnce(Return(RetWithError(64 * cKilobyte, ErrorEnum::eNone))); + + auto [size1, err1] = mSpaceAllocator.RemoveOutdatedItem(); + ASSERT_TRUE(err1.IsNone()); + EXPECT_EQ(size1, 64 * cKilobyte); + + auto [size2, err2] = mSpaceAllocator.RemoveOutdatedItem(); + EXPECT_EQ(err2, ErrorEnum::eNotFound); + EXPECT_EQ(size2, 0U); + + ASSERT_TRUE(space->Accept().IsNone()); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + } // namespace aos::spaceallocator From b920474d74cd59eb2b03af0a0dd3d500254e6cba Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Fri, 30 Oct 2026 13:08:08 +0200 Subject: [PATCH 06/54] common: spaceallocator: clear outdated items on close Signed-off-by: Mykola Solianko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../common/spaceallocator/spaceallocator.hpp | 18 +++++++++++ .../spaceallocator/tests/spaceallocator.cpp | 32 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/core/common/spaceallocator/spaceallocator.hpp b/src/core/common/spaceallocator/spaceallocator.hpp index aaadbb063..80ef7b648 100644 --- a/src/core/common/spaceallocator/spaceallocator.hpp +++ b/src/core/common/spaceallocator/spaceallocator.hpp @@ -290,6 +290,18 @@ class Partition { return {freedSize, ErrorEnum::eNone}; } + /** + * Removes the outdated items of the space allocator from the outdated items list without removing the items. + * + * @param spaceAllocator space allocator the items belong to. + * @return void. + */ + void ClearOutdatedItems(const SpaceAllocatorItf* spaceAllocator) + { + mOutdatedItems.RemoveIf( + [spaceAllocator](const OutdatedItem& item) { return item.mSpaceAllocator == spaceAllocator; }); + } + static constexpr size_t cMaxNumOutdatedItems = AOS_CONFIG_SPACEALLOCATOR_MAX_OUTDATED_ITEMS; StaticString mMountPoint; @@ -451,6 +463,12 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { err = errRemovePartLimit; } + { + LockGuard partitionLock {mPartition->mMutex}; + + mPartition->ClearOutdatedItems(this); + } + mPartition->mAllocatorCount--; if (mPartition->mAllocatorCount != 0) { diff --git a/src/core/common/spaceallocator/tests/spaceallocator.cpp b/src/core/common/spaceallocator/tests/spaceallocator.cpp index 82c7c5eb4..618e865cc 100644 --- a/src/core/common/spaceallocator/tests/spaceallocator.cpp +++ b/src/core/common/spaceallocator/tests/spaceallocator.cpp @@ -1092,6 +1092,38 @@ TEST_F(SpaceallocatorTest, RemoveOutdatedItemAfterEviction) ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); } +TEST_F(SpaceallocatorTest, CloseClearsOutdatedItems) +{ + SpaceAllocator<5> allocator1; + SpaceAllocator<5> allocator2; + + StrictMock remover2; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .Times(2) + .WillRepeatedly(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(allocator1.Init(mAllocator, mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + ASSERT_TRUE(allocator2.Init(mAllocator, mPath, mPlatformFS, mLimit, &remover2).IsNone()); + + ASSERT_TRUE(allocator1.AddOutdatedItem("item1", "1.0.0", Time::Now()).IsNone()); + + ASSERT_TRUE(allocator1.Close().IsNone()); + + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillOnce(Return(RetWithError(10 * cKilobyte, ErrorEnum::eNone))); + + auto [space, err] = allocator2.AllocateSpace(20 * cKilobyte); + EXPECT_EQ(err, ErrorEnum::eNoMemory); + EXPECT_EQ(space.Get(), nullptr); + + auto [size, removeErr] = allocator2.RemoveOutdatedItem(); + EXPECT_EQ(removeErr, ErrorEnum::eNotFound); + EXPECT_EQ(size, 0U); + + ASSERT_TRUE(allocator2.Close().IsNone()); +} + TEST_F(SpaceallocatorTest, RemoveOutdatedItemAccountingIsAtomic) { constexpr size_t cTotalSize = 100 * cKilobyte; From c54a0fd39033dcd8286d0328da60a240bbaa13b1 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Fri, 30 Oct 2026 13:11:43 +0200 Subject: [PATCH 07/54] common: spaceallocator: fix allocation count on resize Signed-off-by: Mykola Solianko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../common/spaceallocator/spaceallocator.hpp | 31 +++++-- .../spaceallocator/tests/spaceallocator.cpp | 85 +++++++++++++++++++ 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/src/core/common/spaceallocator/spaceallocator.hpp b/src/core/common/spaceallocator/spaceallocator.hpp index 80ef7b648..82d22d465 100644 --- a/src/core/common/spaceallocator/spaceallocator.hpp +++ b/src/core/common/spaceallocator/spaceallocator.hpp @@ -617,15 +617,20 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { LockGuard lock {mPartition->mMutex}; - Free(oldSize); + const auto deltaSize = static_cast(newSize) - static_cast(oldSize); - if (auto err = Allocate(newSize); !err.IsNone()) { + if (deltaSize < 0) { + Free(oldSize - newSize); + + return mPartition->Resize(deltaSize); + } + + if (auto err = Reserve(newSize - oldSize); !err.IsNone()) { return err; } - if (auto err = mPartition->Resize(static_cast(newSize) - static_cast(oldSize)); - !err.IsNone()) { - Free(newSize); + if (auto err = mPartition->Resize(deltaSize); !err.IsNone()) { + Free(newSize - oldSize); return err; } @@ -664,6 +669,21 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { return ErrorEnum::eNone; } + if (auto err = Reserve(size); !err.IsNone()) { + return err; + } + + mAllocationCount++; + + return ErrorEnum::eNone; + } + + Error Reserve(size_t size) + { + if (mLimitPercent == 0) { + return ErrorEnum::eNone; + } + if (mAllocationCount == 0) { if (auto err = UpdateSizes(); !err.IsNone()) { return err; @@ -682,7 +702,6 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { } mAllocatedSize += size; - mAllocationCount++; return ErrorEnum::eNone; } diff --git a/src/core/common/spaceallocator/tests/spaceallocator.cpp b/src/core/common/spaceallocator/tests/spaceallocator.cpp index 618e865cc..f81a21d65 100644 --- a/src/core/common/spaceallocator/tests/spaceallocator.cpp +++ b/src/core/common/spaceallocator/tests/spaceallocator.cpp @@ -563,6 +563,91 @@ TEST_F(SpaceallocatorTest, ResizeSpaceInsufficientSpace) ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); } +TEST_F(SpaceallocatorTest, ResizeSpaceKeepsAllocationCount) +{ + constexpr size_t cTotalSize = 100 * cKilobyte; + + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mAllocator, mPath, mPlatformFS, 50).IsNone()); + + EXPECT_CALL(mPlatformFS, GetTotalSize(mMountPoint)) + .Times(2) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetDirSize(mPath)) + .Times(2) + .WillRepeatedly(Return(RetWithError(10 * cKilobyte, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .Times(2) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + + auto [space1, err1] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + + ASSERT_TRUE(space1->Resize(20 * cKilobyte).IsNone()); + ASSERT_TRUE(space1->Accept().IsNone()); + + auto [space2, err2] = mSpaceAllocator.AllocateSpace(10 * cKilobyte); + ASSERT_TRUE(err2.IsNone()); + ASSERT_NE(space2.Get(), nullptr); + + ASSERT_TRUE(space2->Accept().IsNone()); + + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, ResizeSpaceRollbackOnPartitionError) +{ + constexpr size_t cTotalSize = 100 * cKilobyte; + + SpaceAllocator<5> allocator1; + SpaceAllocator<5> allocator2; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .Times(2) + .WillRepeatedly(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + ASSERT_TRUE(allocator1.Init(mAllocator, mPath, mPlatformFS, 50).IsNone()); + ASSERT_TRUE(allocator2.Init(mAllocator, mPath, mPlatformFS, mLimit).IsNone()); + + EXPECT_CALL(mPlatformFS, GetTotalSize(mMountPoint)) + .WillRepeatedly(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetDirSize(mPath)).WillRepeatedly(Return(RetWithError(0, ErrorEnum::eNone))); + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillOnce(Return(RetWithError(cTotalSize, ErrorEnum::eNone))); + + auto [space2, err2] = allocator2.AllocateSpace(60 * cKilobyte); + ASSERT_TRUE(err2.IsNone()); + ASSERT_NE(space2.Get(), nullptr); + + auto [space1, err1] = allocator1.AllocateSpace(10 * cKilobyte); + ASSERT_TRUE(err1.IsNone()); + ASSERT_NE(space1.Get(), nullptr); + + EXPECT_EQ(space1->Resize(45 * cKilobyte), ErrorEnum::eNoMemory); + EXPECT_EQ(space1->Size(), 10 * cKilobyte); + + ASSERT_TRUE(space2->Release().IsNone()); + + auto [space3, err3] = allocator1.AllocateSpace(45 * cKilobyte); + EXPECT_EQ(err3, ErrorEnum::eNoMemory); + EXPECT_EQ(space3.Get(), nullptr); + + auto [space4, err4] = allocator1.AllocateSpace(40 * cKilobyte); + ASSERT_TRUE(err4.IsNone()); + ASSERT_NE(space4.Get(), nullptr); + + ASSERT_TRUE(space4->Accept().IsNone()); + ASSERT_TRUE(space1->Accept().IsNone()); + + ASSERT_TRUE(allocator1.Close().IsNone()); + ASSERT_TRUE(allocator2.Close().IsNone()); +} + TEST_F(SpaceallocatorTest, OutdatedItemsMultipleVersions) { SpaceAllocator<5> mSpaceAllocator; From 4227e635146fefc2838eb50d2104ee39d1ed177c Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 29 Oct 2026 19:19:04 +0200 Subject: [PATCH 08/54] common: imagemanager: implement item status provider Implement SubscribeListener, UnsubscribeListener and GetItemsStatuses. GetItemsStatuses appends the stored state of each item version to the output array. Subscribe and unsubscribe must not be called from a listener notification callback. Add unit tests for listener subscription and items statuses. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko --- src/core/common/imagemanager/imagemanager.cpp | 39 +++- src/core/common/imagemanager/imagemanager.hpp | 18 +- .../imagemanager/tests/imagemanager.cpp | 168 +++++++++++++++++- 3 files changed, 213 insertions(+), 12 deletions(-) diff --git a/src/core/common/imagemanager/imagemanager.cpp b/src/core/common/imagemanager/imagemanager.cpp index 5a59fa8df..16379ad44 100644 --- a/src/core/common/imagemanager/imagemanager.cpp +++ b/src/core/common/imagemanager/imagemanager.cpp @@ -30,29 +30,54 @@ Error ImageManager::Stop() Error ImageManager::GetItemsStatuses(Array& statuses) { - (void)statuses; - LOG_DBG() << "Get items statuses"; - return ErrorEnum::eNotSupported; + auto items = MakeUnique>(mAllocator); + if (!items) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = mStorage->GetAllDeployableItems(*items); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + for (const auto& item : *items) { + if (auto err = statuses.PushBack(item); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + + return ErrorEnum::eNone; } Error ImageManager::SubscribeListener(ItemStatusListenerItf& listener) { - (void)listener; + LockGuard lock {mListenersMutex}; LOG_DBG() << "Subscribe listener"; - return ErrorEnum::eNotSupported; + if (mListeners.Contains(&listener)) { + return AOS_ERROR_WRAP(ErrorEnum::eAlreadyExist); + } + + if (auto err = mListeners.PushBack(&listener); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; } Error ImageManager::UnsubscribeListener(ItemStatusListenerItf& listener) { - (void)listener; + LockGuard lock {mListenersMutex}; LOG_DBG() << "Unsubscribe listener"; - return ErrorEnum::eNotSupported; + if (mListeners.Remove(&listener) == 0) { + return AOS_ERROR_WRAP(ErrorEnum::eNotFound); + } + + return ErrorEnum::eNone; } Error ImageManager::GetBlobPath(const String& digest, String& path) const diff --git a/src/core/common/imagemanager/imagemanager.hpp b/src/core/common/imagemanager/imagemanager.hpp index 65a614d15..26f5d0951 100644 --- a/src/core/common/imagemanager/imagemanager.hpp +++ b/src/core/common/imagemanager/imagemanager.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "itf/blobinfoprovider.hpp" #include "itf/itemstatusprovider.hpp" @@ -50,16 +51,21 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI virtual Error Stop(); /** - * Retrieves deployable items statuses. Reports one status per item version in the storage. + * Retrieves deployable items statuses. Reports one status per item version in the storage with its stored state. + * + * @note The statuses are appended to the output array, which is not cleared. * * @param[out] statuses list of deployable items statuses. - * @return Error. + * @return Error: eNoMemory if the statuses don't fit the output array, storage errors as is. */ Error GetItemsStatuses(Array& statuses) override; /** * Subscribes status notifications. * + * @note Must not be called from a listener notification callback: listeners are notified under the listeners + * lock, so the call deadlocks. + * * @param listener status listener. * @return Error: eAlreadyExist if the listener is already subscribed. */ @@ -68,6 +74,9 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI /** * Unsubscribes from status notifications. * + * @note Must not be called from a listener notification callback: listeners are notified under the listeners + * lock, so the call deadlocks. + * * @param listener status listener. * @return Error: eNotFound if the listener is not subscribed. */ @@ -104,6 +113,8 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI /** * Initializes image manager. Derived classes call it from their own Init. * + * @note The allocator must provide one StaticArray for GetItemsStatuses. + * * @param config image manager config. * @param downloadBlobInfoProvider download blob info provider. * @param storage image manager storage. @@ -261,6 +272,9 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI fs::FSPlatformItf* mFSPlatform {}; AllocatorItf* mAllocator {}; + Mutex mListenersMutex; + StaticArray mListeners; + ItemRemover mItemRemover {*this}; }; diff --git a/src/core/common/imagemanager/tests/imagemanager.cpp b/src/core/common/imagemanager/tests/imagemanager.cpp index d447e633e..e53d51ccc 100644 --- a/src/core/common/imagemanager/tests/imagemanager.cpp +++ b/src/core/common/imagemanager/tests/imagemanager.cpp @@ -31,6 +31,48 @@ namespace { **********************************************************************************************************************/ constexpr auto cTestImagePath = "/tmp/common_imagemanager_test/images"; +constexpr auto cTestDigest = "sha256:a3b2c1d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90"; + +/*********************************************************************************************************************** + * Static + **********************************************************************************************************************/ + +DeployableItemInfo CreateDeployableItem( + const String& itemID, const String& version, ItemStateEnum state, const Error& error = ErrorEnum::eNone) +{ + DeployableItemInfo item; + + item.mItemID = itemID; + item.mType = UpdateItemTypeEnum::eService; + item.mVersion = version; + item.mRootDigest = cTestDigest; + item.mState = state; + item.mTimestamp = Time::Now(); + item.mError = error; + + return item; +} + +UpdateItemStatus CreateItemStatus(const String& itemID, const String& version, ItemStateEnum state) +{ + UpdateItemStatus status; + + status.mItemID = itemID; + status.mType = UpdateItemTypeEnum::eService; + status.mVersion = version; + status.mState = state; + + return status; +} + +const UpdateItemStatus* FindStatus(const Array& statuses, const String& itemID, const String& version) +{ + auto it = statuses.FindIf([&itemID, &version](const UpdateItemStatus& status) { + return status.mItemID == itemID && status.mVersion == version; + }); + + return it != statuses.end() ? it : nullptr; +} /*********************************************************************************************************************** * Test image manager @@ -58,6 +100,21 @@ class CommonImageManagerTest : public Test { protected: static void SetUpTestSuite() { tests::utils::InitLog(); } + Error InitImageManager() + { + Config config {cTestImagePath, 0, 10 * Time::cSeconds, 5 * Time::cSeconds}; + + return mImageManager.Init( + config, mBlobInfoProvider, mStorage, mSpaceAllocator, mFileInfoProvider, mOCISpec, mFSPlatform, mAllocator); + } + + void AddStoredItem( + const String& itemID, const String& version, ItemStateEnum state, const Error& error = ErrorEnum::eNone) + { + auto err = mStorage.AddDeployableItem(CreateDeployableItem(itemID, version, state, error)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + } + HeapAllocator mAllocator; NiceMock mBlobInfoProvider; StorageStub mStorage; @@ -75,11 +132,116 @@ class CommonImageManagerTest : public Test { TEST_F(CommonImageManagerTest, Init) { - Config config {cTestImagePath, 0, 10 * Time::cSeconds, 5 * Time::cSeconds}; + auto err = InitImageManager(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} - auto err = mImageManager.Init( - config, mBlobInfoProvider, mStorage, mSpaceAllocator, mFileInfoProvider, mOCISpec, mFSPlatform, mAllocator); +TEST_F(CommonImageManagerTest, SubscribeListenerTwice) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + auto err = mImageManager.SubscribeListener(mListener); ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + err = mImageManager.SubscribeListener(mListener); + EXPECT_TRUE(err.Is(ErrorEnum::eAlreadyExist)) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerTest, UnsubscribeUnknownListener) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + auto err = mImageManager.UnsubscribeListener(mListener); + EXPECT_TRUE(err.Is(ErrorEnum::eNotFound)) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerTest, SubscribeUnsubscribeListener) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + auto err = mImageManager.SubscribeListener(mListener); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + err = mImageManager.UnsubscribeListener(mListener); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + err = mImageManager.UnsubscribeListener(mListener); + EXPECT_TRUE(err.Is(ErrorEnum::eNotFound)) << tests::utils::ErrorToStr(err); + + err = mImageManager.SubscribeListener(mListener); + EXPECT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerTest, GetItemsStatusesEmpty) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + StaticArray statuses; + + auto err = mImageManager.GetItemsStatuses(statuses); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + EXPECT_TRUE(statuses.IsEmpty()); +} + +TEST_F(CommonImageManagerTest, GetItemsStatusesStoredStates) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + AddStoredItem("item1", "1.0.0", ItemStateEnum::eInstalled); + AddStoredItem("item1", "2.0.0", ItemStateEnum::eRemoved); + AddStoredItem("item2", "1.0.0", ItemStateEnum::eFailed, Error(ErrorEnum::eFailed, "download failed")); + AddStoredItem("item3", "1.0.0", ItemStateEnum::eInstalling); + + StaticArray statuses; + + // The statuses are appended to the output array. + const auto preFilledStatus = CreateItemStatus("item0", "1.0.0", ItemStateEnum::eInstalled); + + ASSERT_TRUE(statuses.PushBack(preFilledStatus).IsNone()); + + auto err = mImageManager.GetItemsStatuses(statuses); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + ASSERT_EQ(statuses.Size(), 5); + EXPECT_EQ(statuses[0], preFilledStatus); + + for (const auto& status : statuses) { + EXPECT_EQ(status.mType, UpdateItemTypeEnum::eService); + EXPECT_FALSE(status.mPreinstalled); + } + + const auto* status = FindStatus(statuses, "item1", "1.0.0"); + ASSERT_NE(status, nullptr); + EXPECT_EQ(status->mState, ItemStateEnum::eInstalled); + EXPECT_TRUE(status->mError.IsNone()); + + status = FindStatus(statuses, "item1", "2.0.0"); + ASSERT_NE(status, nullptr); + EXPECT_EQ(status->mState, ItemStateEnum::eRemoved); + EXPECT_TRUE(status->mError.IsNone()); + + status = FindStatus(statuses, "item2", "1.0.0"); + ASSERT_NE(status, nullptr); + EXPECT_EQ(status->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(status->mError.Is(ErrorEnum::eFailed)); + EXPECT_STREQ(status->mError.Message(), "download failed"); + + status = FindStatus(statuses, "item3", "1.0.0"); + ASSERT_NE(status, nullptr); + EXPECT_EQ(status->mState, ItemStateEnum::eInstalling); + EXPECT_TRUE(status->mError.IsNone()); +} + +TEST_F(CommonImageManagerTest, GetItemsStatusesNoMemory) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + AddStoredItem("item1", "1.0.0", ItemStateEnum::eInstalled); + AddStoredItem("item1", "2.0.0", ItemStateEnum::eInstalled); + + StaticArray statuses; + + auto err = mImageManager.GetItemsStatuses(statuses); + EXPECT_TRUE(err.Is(ErrorEnum::eNoMemory)) << tests::utils::ErrorToStr(err); } } // namespace aos::common::imagemanager From 495c089ea8450ca0406fc4966b6fb913e16dd2cf Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 29 Oct 2026 22:01:26 +0200 Subject: [PATCH 09/54] common: imagemanager: implement InstallItem Implement installing of a single deployable item version in the common image manager. The item version is locked for the install, so InstallItem and RemoveItem of the same version are mutually exclusive. A new item is added to the storage in the eInstalling state, a stored item is set to eInstalling and, if it was removed, restored from the outdated items of the space allocator. The root blob is the image index or the image manifest. The item blobs are processed starting from it: the manifests, the item and image configs and the layers. Each blob is locked while it is processed and is added to the item blob references before it is validated. A missing or invalid blob is removed, downloaded with DownloadBlob using the download blob info and validated again. Its space is allocated before the download, accepted on success and released on failure. A layer is checked with ValidateLayer first: if it is valid, the layer blob is not checked. eNotSupported means the layer is kept as packed blob only. Otherwise the unpacked layer is removed with RemoveLayer, the layer blob is processed and unpacked with UnpackLayer. The image path is synced whatever the install result. On success the item is set to eInstalled, on failure to eFailed with the error. The eInstalling, eDownloading and the final states are notified to the listeners, and GetItemsStatuses reports the last notified state of the item being installed. Add OCI image index and manifest media types to imagespec.hpp. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko --- src/core/common/imagemanager/imagemanager.cpp | 793 +++++++- src/core/common/imagemanager/imagemanager.hpp | 82 +- .../imagemanager/tests/imagemanager.cpp | 1788 ++++++++++++++++- .../imagemanager/tests/stubs/storagestub.hpp | 26 + src/core/common/ocispec/itf/imagespec.hpp | 6 + 5 files changed, 2671 insertions(+), 24 deletions(-) diff --git a/src/core/common/imagemanager/imagemanager.cpp b/src/core/common/imagemanager/imagemanager.cpp index 16379ad44..bc7690445 100644 --- a/src/core/common/imagemanager/imagemanager.cpp +++ b/src/core/common/imagemanager/imagemanager.cpp @@ -10,6 +10,74 @@ namespace aos::common::imagemanager { +namespace { + +/*********************************************************************************************************************** + * Static + **********************************************************************************************************************/ + +Error ParseDigest(const String& digest, String& algorithm, String& hash) +{ + // The blob paths are limited by the whole digest length, not by the lengths of its parts. + if (digest.Size() > oci::cDigestLen) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "malformed digest")); + } + + auto [pos, err] = digest.FindSubstr(0, ":"); + if (!err.IsNone() || pos == 0 || pos + 1 >= digest.Size()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "malformed digest")); + } + + if (err = algorithm.Assign(String(digest.CStr(), pos)); !err.IsNone()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "malformed digest")); + } + + if (err = hash.Assign(String(digest.CStr() + pos + 1, digest.Size() - pos - 1)); !err.IsNone()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "malformed digest")); + } + + const auto isLowerAlnum = [](char ch) { return (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9'); }; + + if (!isLowerAlnum(algorithm[0])) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "malformed digest")); + } + + for (const auto ch : algorithm) { + if (!(isLowerAlnum(ch) || ch == '+' || ch == '.' || ch == '_' || ch == '-')) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "malformed digest")); + } + } + + for (const auto ch : hash) { + if (!isLowerAlnum(ch)) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "malformed digest")); + } + } + + return ErrorEnum::eNone; +} + +RetWithError GetSupportedHash(const String& hashName) +{ + StaticString upperHash; + + if (auto err = upperHash.Assign(hashName); !err.IsNone()) { + return {{}, AOS_ERROR_WRAP(Error(ErrorEnum::eNotSupported, "unsupported hashing algorithm"))}; + } + + (void)upperHash.ToUpper(); + + if (upperHash == "SHA256") { + return {crypto::HashEnum::eSHA256, ErrorEnum::eNone}; + } else if (upperHash == "SHA512") { + return {crypto::HashEnum::eSHA512, ErrorEnum::eNone}; + } else { + return {{}, AOS_ERROR_WRAP(Error(ErrorEnum::eNotSupported, "unsupported hashing algorithm"))}; + } +} + +} // namespace + /*********************************************************************************************************************** * Public **********************************************************************************************************************/ @@ -30,6 +98,8 @@ Error ImageManager::Stop() Error ImageManager::GetItemsStatuses(Array& statuses) { + LockGuard lock {mMutex}; + LOG_DBG() << "Get items statuses"; auto items = MakeUnique>(mAllocator); @@ -42,7 +112,21 @@ Error ImageManager::GetItemsStatuses(Array& statuses) } for (const auto& item : *items) { - if (auto err = statuses.PushBack(item); !err.IsNone()) { + UpdateItemStatus status = item; // NOSONAR cpp:S5912 - intentional copy of the base part only + + const auto* itemInProgress = FindInProgressItem(item.mItemID, item.mVersion); + + // The item is left in eInstalling state by InstallItem that failed to update the item state. + if (itemInProgress == nullptr && status.mState == ItemStateEnum::eInstalling) { + status.mState = ItemStateEnum::eFailed; + } + + if (itemInProgress != nullptr && itemInProgress->mState != ItemStateEnum::eUnknown) { + status.mState = itemInProgress->mState; + status.mError = itemInProgress->mError; + } + + if (auto err = statuses.PushBack(status); !err.IsNone()) { return AOS_ERROR_WRAP(err); } } @@ -82,11 +166,20 @@ Error ImageManager::UnsubscribeListener(ItemStatusListenerItf& listener) Error ImageManager::GetBlobPath(const String& digest, String& path) const { - (void)path; - LOG_DBG() << "Get blob path" << Log::Field("digest", digest); - return ErrorEnum::eNotSupported; + StaticString algorithm; + StaticString hash; + + if (auto err = ParseDigest(digest, algorithm, hash); !err.IsNone()) { + return err; + } + + if (auto err = path.Assign(fs::JoinPath(mConfig.mImagePath, cBlobsFolder, algorithm, hash)); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; } /*********************************************************************************************************************** @@ -112,15 +205,60 @@ Error ImageManager::Init(const Config& config, DownloadBlobInfoProviderItf& blob << Log::Field("itemCacheTTL", mConfig.mItemCacheTTL) << Log::Field("removeOutdatedPeriod", mConfig.mRemoveOutdatedPeriod); + // The longest blob path is /blobs//, / has the digest length. + if (mConfig.mImagePath.Size() + 1 + String(cBlobsFolder).Size() + 1 + oci::cDigestLen > cFilePathLen) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "image path is too long")); + } + + if (auto err = fs::MakeDirAll(fs::JoinPath(mConfig.mImagePath, cBlobsFolder)); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + return ErrorEnum::eNone; } Error ImageManager::InstallItem(const InstallItemInfo& itemInfo) { - LOG_DBG() << "Install item" << Log::Field("itemID", itemInfo.mItemID) << Log::Field("version", itemInfo.mVersion) - << Log::Field("rootDigest", itemInfo.mRootDigest); + LOG_INF() << "Install item" << Log::Field("itemID", itemInfo.mItemID) << Log::Field("version", itemInfo.mVersion) + << Log::Field("type", itemInfo.mType) << Log::Field("rootDigest", itemInfo.mRootDigest); - return ErrorEnum::eNotSupported; + if (auto err = LockItem(itemInfo.mItemID, itemInfo.mVersion); !err.IsNone()) { + return err; + } + + auto unlockItem = DeferRelease([this, &itemInfo]() { UnlockItem(itemInfo.mItemID, itemInfo.mVersion); }); + + DeployableItemInfo storedInfo; + + if (auto err = StartItemInstall(itemInfo, storedInfo); !err.IsNone()) { + return err; + } + + auto err = InstallItemBlobs(itemInfo); + + if (auto syncErr = mFSPlatform->Sync(mConfig.mImagePath); !syncErr.IsNone()) { + LOG_ERR() << "Failed to sync image path" << Log::Field(syncErr); + + if (err.IsNone()) { + err = AOS_ERROR_WRAP(syncErr); + } + } + + if (err.IsNone()) { + err = SetStoredItemState(storedInfo, ItemStateEnum::eInstalled); + } + + if (!err.IsNone()) { + if (auto setStateErr = SetStoredItemState(storedInfo, ItemStateEnum::eFailed, err); !setStateErr.IsNone()) { + LOG_ERR() << "Failed to set stored item state" << Log::Field(setStateErr); + + if (setStateErr = SetInProgressItemState(itemInfo, ItemStateEnum::eFailed, err); !setStateErr.IsNone()) { + LOG_ERR() << "Failed to set in-progress item state" << Log::Field(setStateErr); + } + } + } + + return err; } Error ImageManager::RemoveItem(const String& itemID, const String& version) @@ -141,6 +279,358 @@ Error ImageManager::Cancel() * Private **********************************************************************************************************************/ +void ImageManager::NotifyItemStatusChanged(const UpdateItemStatus& status) +{ + LockGuard lock {mListenersMutex}; + + LOG_DBG() << "Item status changed" << Log::Field("itemID", status.mItemID) << Log::Field("type", status.mType) + << Log::Field("version", status.mVersion) << Log::Field("state", status.mState) + << Log::Field(status.mError); + + for (auto* listener : mListeners) { + listener->OnItemStatusChanged(status); + } +} + +Error ImageManager::ValidateBlob(const String& path, const String& digest, size_t size) +{ + LOG_DBG() << "Validate blob" << Log::Field("digest", digest) << Log::Field("path", path); + + StaticString algStr; + StaticString hash; + + if (auto err = ParseDigest(digest, algStr, hash); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto [alg, err] = GetSupportedHash(algStr); + if (!err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto hashBytes = MakeUnique>(mAllocator); + if (!hashBytes) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (err = hash.HexToByteArray(*hashBytes); !err.IsNone()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "malformed digest hash")); + } + + bool exists = false; + + if (Tie(exists, err) = fs::FileExist(path); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (!exists) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "file not found")); + } + + fs::FileInfo fileInfo; + + if (err = mFileInfoProvider->GetFileInfo(path, fileInfo, alg); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (size && fileInfo.mSize != size) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidChecksum, "size mismatch")); + } + + if (fileInfo.mCheckSum != *hashBytes) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidChecksum, "checksum mismatch")); + } + + return ErrorEnum::eNone; +} + +Error ImageManager::GetBlobInfo(const String& digest, BlobInfo& info) +{ + while (true) { + if (auto err = mBlobInfoProvider->GetBlobInfo(digest, info); !err.IsNone()) { + UniqueLock lock(mMutex); + + LOG_ERR() << "Failed to get blob info" << Log::Field("digest", digest) << Log::Field(err); + + // The provider is available, but it doesn't have the blob: retrying doesn't help. + if (err.Is(ErrorEnum::eNotFound)) { + return AOS_ERROR_WRAP(err); + } + + if (err = mCondVar.Wait(lock, cRetryTimeout); !err.IsNone() && err != ErrorEnum::eTimeout) { + return AOS_ERROR_WRAP(err); + } + + LOG_DBG() << "Retrying get blob info" << Log::Field("digest", digest); + + continue; + } + + break; + } + + return ErrorEnum::eNone; +} + +Error ImageManager::DownloadItemBlob(const InstallItemInfo& itemInfo, const String& path, const String& digest) +{ + auto blobInfo = MakeUnique(mAllocator); + if (!blobInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = GetBlobInfo(digest, *blobInfo); !err.IsNone()) { + return err; + } + + if (auto err = fs::MakeDirAll(fs::Dir(path)); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto [space, err] = mSpaceAllocator->AllocateSpace(blobInfo->mSize); + if (!err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (err = SetInProgressItemState(itemInfo, ItemStateEnum::eDownloading); !err.IsNone()) { + LOG_ERR() << "Failed to set in-progress item state" << Log::Field(err); + } + + LOG_DBG() << "Download blob" << Log::Field("digest", digest) << Log::Field("size", blobInfo->mSize); + + err = DownloadBlob(path, *blobInfo); + + // Done on a failed download as well, as the partial file may stay if it can't be removed. + if (auto resizeErr = ResizeBlobSpace(*space, path); !resizeErr.IsNone() && err.IsNone()) { + err = resizeErr; + } + + if (!err.IsNone()) { + if (auto removeErr = RemoveBlob(path, digest).mError; !removeErr.IsNone()) { + LOG_ERR() << "Failed to remove blob" << Log::Field("digest", digest) << Log::Field(removeErr); + + // The file stays, so its space stays used until the blob is processed again. + if (auto acceptErr = space->Accept(); !acceptErr.IsNone()) { + LOG_ERR() << "Failed to accept space" << Log::Field("digest", digest) << Log::Field(acceptErr); + } + + return err; + } + + if (auto releaseErr = space->Release(); !releaseErr.IsNone()) { + LOG_ERR() << "Failed to release space" << Log::Field("digest", digest) << Log::Field(releaseErr); + } + + return err; + } + + if (err = space->Accept(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +Error ImageManager::ResizeBlobSpace(spaceallocator::SpaceItf& space, const String& path) +{ + // The blob removal frees the size of the file, so the accepted space has to match it. + auto [size, err] = fs::CalculateSize(*mAllocator, path); + if (!err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (err = space.Resize(size); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +Error ImageManager::InstallItemBlob( + const InstallItemInfo& itemInfo, const oci::ContentDescriptor& descriptor, String& path) +{ + LOG_DBG() << "Install blob" << Log::Field("digest", descriptor.mDigest) + << Log::Field("mediaType", descriptor.mMediaType) << Log::Field("size", descriptor.mSize); + + if (auto err = GetBlobPath(descriptor.mDigest, path); !err.IsNone()) { + return err; + } + + if (auto err = LockBlob(descriptor.mDigest); !err.IsNone()) { + return err; + } + + auto unlockBlob = DeferRelease([this, digest = &descriptor.mDigest]() { UnlockBlob(*digest); }); + + if (auto err = AddItemBlob(itemInfo.mItemID, itemInfo.mVersion, descriptor); !err.IsNone()) { + return err; + } + + if (auto err = ProcessBlob(itemInfo, path, descriptor); !err.IsNone()) { + return err; + } + + return ErrorEnum::eNone; +} + +Error ImageManager::InstallLayerBlob( + const InstallItemInfo& itemInfo, const oci::ContentDescriptor& descriptor, const String& diffID) +{ + LOG_DBG() << "Install layer blob" << Log::Field("digest", descriptor.mDigest) + << Log::Field("mediaType", descriptor.mMediaType) << Log::Field("size", descriptor.mSize) + << Log::Field("diffID", diffID); + + StaticString path; + + if (auto err = GetBlobPath(descriptor.mDigest, path); !err.IsNone()) { + return err; + } + + if (auto err = LockBlob(descriptor.mDigest); !err.IsNone()) { + return err; + } + + auto unlockBlob = DeferRelease([this, digest = &descriptor.mDigest]() { UnlockBlob(*digest); }); + + if (auto err = AddItemBlob(itemInfo.mItemID, itemInfo.mVersion, descriptor, true, diffID); !err.IsNone()) { + return err; + } + + if (auto err = ProcessLayer(itemInfo, path, descriptor, diffID); !err.IsNone()) { + return err; + } + + return ErrorEnum::eNone; +} + +Error ImageManager::ProcessBlob( + const InstallItemInfo& itemInfo, const String& path, const oci::ContentDescriptor& descriptor) +{ + auto err = ValidateBlob(path, descriptor.mDigest, descriptor.mSize); + if (err.IsNone()) { + return ErrorEnum::eNone; + } + + // Only a missing or corrupted blob is repaired, the other errors don't mean that the blob is invalid. + if (!err.Is(ErrorEnum::eNotFound) && !err.Is(ErrorEnum::eInvalidChecksum) && !err.Is(ErrorEnum::eIntegrity)) { + return err; + } + + if (!err.Is(ErrorEnum::eNotFound)) { + LOG_ERR() << "Blob validation failed" << Log::Field("digest", descriptor.mDigest) << Log::Field(err); + } + + // The unpacked layer data is removed by ProcessLayer. + if (err = RemoveItemBlob(path, descriptor); !err.IsNone()) { + return err; + } + + if (err = DownloadItemBlob(itemInfo, path, descriptor.mDigest); !err.IsNone()) { + return err; + } + + if (err = ValidateBlob(path, descriptor.mDigest, descriptor.mSize); !err.IsNone()) { + return err; + } + + return ErrorEnum::eNone; +} + +Error ImageManager::ProcessLayer( + const InstallItemInfo& itemInfo, const String& path, const oci::ContentDescriptor& descriptor, const String& diffID) +{ + auto err = ValidateLayer(path, descriptor, diffID); + if (err.IsNone()) { + return ErrorEnum::eNone; + } + + // eNotSupported means the layer is kept as packed blob only: there is no unpacked layer data to handle. + if (err.Is(ErrorEnum::eNotSupported)) { + if (err = ProcessBlob(itemInfo, path, descriptor); !err.IsNone()) { + return err; + } + + return ErrorEnum::eNone; + } + + if (!err.Is(ErrorEnum::eNotFound)) { + LOG_ERR() << "Layer validation failed" << Log::Field("digest", descriptor.mDigest) << Log::Field(err); + } + + size_t layerSize = 0; + + Tie(layerSize, err) = RemoveLayer(path, descriptor, diffID); + if (!err.IsNone() && !err.Is(ErrorEnum::eNotSupported)) { + return err; + } + + if (layerSize > 0) { + mSpaceAllocator->FreeSpace(layerSize); + } + + if (err = ProcessBlob(itemInfo, path, descriptor); !err.IsNone()) { + return err; + } + + if (err = UnpackLayer(path, descriptor, diffID); !err.IsNone() && !err.Is(ErrorEnum::eNotSupported)) { + return err; + } + + return ErrorEnum::eNone; +} + +Error ImageManager::RemoveItemBlob( + const String& path, const oci::ContentDescriptor& descriptor, bool isLayer, const String& diffID) +{ + Error err; + size_t layerSize = 0; + size_t blobSize = 0; + + if (isLayer) { + Tie(layerSize, err) = RemoveLayer(path, descriptor, diffID); + if (!err.IsNone() && !err.Is(ErrorEnum::eNotSupported)) { + return err; + } + } + + Tie(blobSize, err) = RemoveBlob(path, descriptor.mDigest); + if (!err.IsNone()) { + return err; + } + + if (layerSize + blobSize > 0) { + mSpaceAllocator->FreeSpace(layerSize + blobSize); + } + + return ErrorEnum::eNone; +} + +RetWithError ImageManager::RemoveBlob(const String& path, const String& digest) +{ + LOG_DBG() << "Remove blob" << Log::Field("digest", digest) << Log::Field("path", path); + + auto [exists, err] = fs::FileExist(path); + if (!err.IsNone()) { + return {0, AOS_ERROR_WRAP(err)}; + } + + if (!exists) { + return {0, ErrorEnum::eNone}; + } + + size_t size = 0; + + if (Tie(size, err) = fs::CalculateSize(*mAllocator, path); !err.IsNone()) { + return {0, AOS_ERROR_WRAP(err)}; + } + + if (err = fs::RemoveAll(path); !err.IsNone()) { + return {0, AOS_ERROR_WRAP(err)}; + } + + return {size, ErrorEnum::eNone}; +} + RetWithError ImageManager::RemoveOutdatedItem(const String& itemID, const String& version) { LOG_DBG() << "Remove outdated item" << Log::Field("itemID", itemID) << Log::Field("version", version); @@ -148,4 +638,293 @@ RetWithError ImageManager::RemoveOutdatedItem(const String& itemID, cons return {0, ErrorEnum::eNotSupported}; } +Error ImageManager::LockItem(const String& itemID, const String& version) +{ + UniqueLock lock {mMutex}; + + if (auto err = mCondVar.Wait( + lock, [&]() { return FindInProgressItem(itemID, version) == nullptr && !mItemsInProgress.IsFull(); }); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + UpdateItemStatus item; + + item.mItemID = itemID; + item.mVersion = version; + + if (auto err = mItemsInProgress.PushBack(item); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +void ImageManager::UnlockItem(const String& itemID, const String& version) +{ + LockGuard lock {mMutex}; + + (void)mItemsInProgress.RemoveIf( + [&](const UpdateItemStatus& item) { return item.mItemID == itemID && item.mVersion == version; }); + + if (auto err = mCondVar.NotifyAll(); !err.IsNone()) { + LOG_ERR() << "Can't notify condition variable" << Log::Field(err); + } +} + +UpdateItemStatus* ImageManager::FindInProgressItem(const String& itemID, const String& version) +{ + auto it = mItemsInProgress.FindIf( + [&](const UpdateItemStatus& item) { return item.mItemID == itemID && item.mVersion == version; }); + + return it != mItemsInProgress.end() ? &*it : nullptr; +} + +Error ImageManager::LockBlob(const String& digest) +{ + UniqueLock lock {mMutex}; + + if (auto err = mCondVar.Wait(lock, [&]() { return !IsBlobInProgress(digest) && !mBlobsInProgress.IsFull(); }); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (auto err = mBlobsInProgress.PushBack(digest); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +void ImageManager::UnlockBlob(const String& digest) +{ + LockGuard lock {mMutex}; + + (void)mBlobsInProgress.RemoveIf([&digest](const String& blob) { return blob == digest; }); + + if (auto err = mCondVar.NotifyAll(); !err.IsNone()) { + LOG_ERR() << "Can't notify condition variable" << Log::Field(err); + } +} + +Error ImageManager::StartItemInstall(const InstallItemInfo& itemInfo, DeployableItemInfo& storedInfo) +{ + auto err = mStorage->GetDeployableItem(itemInfo.mItemID, itemInfo.mVersion, storedInfo); + if (!err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { + return AOS_ERROR_WRAP(err); + } + + if (err.IsNone()) { + if (storedInfo.mRootDigest != itemInfo.mRootDigest) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "root digest mismatch")); + } + + const auto prevState = storedInfo.mState; + + if (err = SetStoredItemState(storedInfo, ItemStateEnum::eInstalling, ErrorEnum::eNone); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + // The item is restored after it leaves the eRemoved state: if it stays outdated, it is not removed from the + // cache as it is not in the eRemoved state anymore. + if (prevState == ItemStateEnum::eRemoved) { + if (err = mSpaceAllocator->RestoreOutdatedItem(itemInfo.mItemID, itemInfo.mVersion); + !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { + LOG_ERR() << "Failed to restore outdated item" << Log::Field(err); + } + } + + return ErrorEnum::eNone; + } + + storedInfo.mItemID = itemInfo.mItemID; + storedInfo.mType = itemInfo.mType; + storedInfo.mVersion = itemInfo.mVersion; + storedInfo.mRootDigest = itemInfo.mRootDigest; + storedInfo.mState = ItemStateEnum::eInstalling; + storedInfo.mTimestamp = Time::Now(); + storedInfo.mError = ErrorEnum::eNone; + + if (err = mStorage->AddDeployableItem(storedInfo); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (err = SetInProgressItemState(itemInfo, ItemStateEnum::eInstalling); !err.IsNone()) { + LOG_ERR() << "Failed to set in-progress item state" << Log::Field(err); + } + + return ErrorEnum::eNone; +} + +Error ImageManager::InstallItemBlobs(const InstallItemInfo& itemInfo) +{ + if (!itemInfo.mIsRootIndex) { + if (auto err = InstallManifestBlobs( + itemInfo, oci::ContentDescriptor {oci::cMediaTypeImageManifest, itemInfo.mRootDigest, 0}); + !err.IsNone()) { + return err; + } + + return ErrorEnum::eNone; + } + + StaticString path; + + if (auto err + = InstallItemBlob(itemInfo, oci::ContentDescriptor {oci::cMediaTypeImageIndex, itemInfo.mRootDigest, 0}, path); + !err.IsNone()) { + return err; + } + + auto index = MakeUnique(mAllocator); + if (!index) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = mOCISpec->LoadImageIndex(path, *index); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + for (const auto& manifest : index->mManifests) { + if (auto err = InstallManifestBlobs(itemInfo, manifest); !err.IsNone()) { + return err; + } + } + + return ErrorEnum::eNone; +} + +Error ImageManager::InstallManifestBlobs(const InstallItemInfo& itemInfo, const oci::ContentDescriptor& descriptor) +{ + LOG_DBG() << "Install manifest" << Log::Field("digest", descriptor.mDigest); + + StaticString path; + + if (auto err = InstallItemBlob(itemInfo, descriptor, path); !err.IsNone()) { + return err; + } + + auto manifest = MakeUnique(mAllocator); + if (!manifest) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = mOCISpec->LoadImageManifest(path, *manifest); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (manifest->mItemConfig.HasValue()) { + if (auto err = InstallItemBlob(itemInfo, *manifest->mItemConfig, path); !err.IsNone()) { + return err; + } + } + + UniquePtr imageConfig; + + if (!manifest->mConfig.mDigest.IsEmpty()) { + if (auto err = InstallItemBlob(itemInfo, manifest->mConfig, path); !err.IsNone()) { + return err; + } + + // The empty config, for example of a component, has no diff IDs. + if (manifest->mConfig.mMediaType != oci::cMediaTypeEmptyBlob) { + imageConfig = MakeUnique(mAllocator); + if (!imageConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = mOCISpec->LoadImageConfig(path, *imageConfig); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (imageConfig->mRootfs.mDiffIDs.Size() < manifest->mLayers.Size()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "diff ID's size mismatch")); + } + } + } + + for (size_t i = 0; i < manifest->mLayers.Size(); ++i) { + StaticString diffID; + + if (imageConfig) { + diffID = imageConfig->mRootfs.mDiffIDs[i]; + } + + if (auto err = InstallLayerBlob(itemInfo, manifest->mLayers[i], diffID); !err.IsNone()) { + return err; + } + } + + return ErrorEnum::eNone; +} + +Error ImageManager::AddItemBlob(const String& itemID, const String& version, const oci::ContentDescriptor& descriptor, + bool isLayer, const String& diffID) +{ + if (auto err = mStorage->AddItemBlob(itemID, version, ItemBlobInfo {descriptor, isLayer, diffID}); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +Error ImageManager::SetInProgressItemState(const InstallItemInfo& itemInfo, ItemStateEnum state, const Error& error) +{ + UpdateItemStatus status; + bool stateChanged = false; + + { + LockGuard lock {mMutex}; + + auto itemInProgress = FindInProgressItem(itemInfo.mItemID, itemInfo.mVersion); + if (!itemInProgress) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "item not in progress")); + } + + stateChanged = itemInProgress->mState != state; + + itemInProgress->mType = itemInfo.mType; + itemInProgress->mState = state; + itemInProgress->mError = error; + + status = *itemInProgress; + } + + if (stateChanged) { + NotifyItemStatusChanged(status); + } + + return ErrorEnum::eNone; +} + +Error ImageManager::SetStoredItemState(DeployableItemInfo& item, ItemStateEnum state, const Error& error) +{ + // An item left in eInstalling is reported as eFailed, so its next install has to be notified. + auto stateChanged = (item.mState != state) || (state == ItemStateEnum::eInstalling); + + item.mState = state; + item.mTimestamp = Time::Now(); + item.mError = error; + + if (auto err = mStorage->UpdateDeployableItem(item); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + { + LockGuard lock {mMutex}; + + if (auto itemInProgress = FindInProgressItem(item.mItemID, item.mVersion); itemInProgress) { + itemInProgress->mType = item.mType; + itemInProgress->mState = state; + itemInProgress->mError = error; + } + } + + if (stateChanged) { + NotifyItemStatusChanged(item); + } + + return ErrorEnum::eNone; +} + } // namespace aos::common::imagemanager diff --git a/src/core/common/imagemanager/imagemanager.hpp b/src/core/common/imagemanager/imagemanager.hpp index 26f5d0951..1485632dd 100644 --- a/src/core/common/imagemanager/imagemanager.hpp +++ b/src/core/common/imagemanager/imagemanager.hpp @@ -7,9 +7,11 @@ #ifndef AOS_CORE_COMMON_IMAGEMANAGER_IMAGEMANAGER_HPP_ #define AOS_CORE_COMMON_IMAGEMANAGER_IMAGEMANAGER_HPP_ +#include #include #include #include +#include #include #include #include @@ -51,7 +53,12 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI virtual Error Stop(); /** - * Retrieves deployable items statuses. Reports one status per item version in the storage with its stored state. + * Retrieves deployable items statuses. Reports one status per item version in the storage. An item version being + * installed, whose eInstalling state is already notified, is reported with its last notified state (eInstalling or + * eDownloading) until its final state is notified. An item version being removed, whose eRemoving state is already + * notified, is reported as eRemoving until its eRemoved state is set in the storage or the storage update fails. + * Otherwise the stored state is reported. mError is set only for eFailed items and is taken from the storage. + * mPreinstalled is always false. ePending is never reported. * * @note The statuses are appended to the output array, which is not cleared. * @@ -113,7 +120,9 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI /** * Initializes image manager. Derived classes call it from their own Init. * - * @note The allocator must provide one StaticArray for GetItemsStatuses. + * @note The allocator must provide, per concurrent InstallItem (up to cMaxNumConcurrentItems), one + * oci::ImageIndex, one oci::ImageManifest, one oci::ImageConfig and one blob hash buffer of cBlobHashSize at the + * same time, plus one StaticArray for GetItemsStatuses. * * @param config image manager config. * @param downloadBlobInfoProvider download blob info provider. @@ -133,8 +142,13 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI * Installs item. Waits while the same item version is being installed or removed, or while * cMaxNumConcurrentItems items are being processed. * + * @note The final eInstalled or eFailed status is notified while the item is still in progress, so a listener + * must not call InstallItem or RemoveItem synchronously from the callback. + * * @param itemInfo item info. - * @return Error. + * @return Error: eInvalidArgument if the root digest is malformed, the item version is stored with a different + * root digest or a diff ID is missing in the image config, eNoMemory if the storage is full and there is no + * outdated item to remove, other errors as is. */ Error InstallItem(const InstallItemInfo& itemInfo); @@ -188,9 +202,12 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI virtual Error CancelDownload(const String& digest) = 0; /** - * Validates installed layer. Called by the image manager after the layer blob is validated, derived classes can - * override it to validate unpacked layer data. Returns eNotSupported by default, which the image manager treats as - * nothing to do (skip). + * Validates installed layer. Called by the image manager before the layer blob is validated, derived classes can + * override it to validate unpacked layer data. If it succeeds, the layer blob is not validated. Any error other + * than eNotSupported means the unpacked layer data is missing or invalid: the image manager calls RemoveLayer and + * UnpackLayer for the layer. Returns eNotSupported by default, which the image manager treats as the layer is kept + * as packed blob only (skip): RemoveLayer and UnpackLayer are not called. Derived classes which override + * UnpackLayer must override ValidateLayer and RemoveLayer as well. * * @param path layer blob path. * @param descriptor layer descriptor. @@ -207,9 +224,9 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI } /** - * Unpacks installed layer. Called by the image manager after the layer blob is installed, derived classes can - * override it to unpack layer data. Returns eNotSupported by default, which the image manager treats as nothing to - * do (skip). + * Unpacks installed layer. Called by the image manager after the layer blob is installed if ValidateLayer fails, + * derived classes can override it to unpack layer data. Returns eNotSupported by default, which the image manager + * treats as nothing to do (skip). * * @param path layer blob path. * @param descriptor layer descriptor. @@ -226,9 +243,9 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI } /** - * Removes installed layer. Called by the image manager before the layer blob is removed, derived classes can - * override it to remove unpacked layer data. Returns eNotSupported by default, which the image manager treats as - * nothing to do (skip). + * Removes installed layer. Called by the image manager before the layer is unpacked if ValidateLayer fails, and + * before the layer blob is removed, derived classes can override it to remove unpacked layer data. Returns + * eNotSupported by default, which the image manager treats as nothing to do (skip). * * @param path layer blob path. * @param descriptor layer descriptor. @@ -263,6 +280,42 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI RetWithError RemoveOutdatedItem(const String& itemID, const String& version); + Error ValidateBlob(const String& path, const String& digest, size_t size = 0); + Error GetBlobInfo(const String& digest, BlobInfo& info); + Error DownloadItemBlob(const InstallItemInfo& itemInfo, const String& path, const String& digest); + Error ResizeBlobSpace(spaceallocator::SpaceItf& space, const String& path); + RetWithError RemoveBlob(const String& path, const String& digest); + Error RemoveItemBlob( + const String& path, const oci::ContentDescriptor& descriptor, bool isLayer = false, const String& diffID = ""); + Error InstallItemBlob(const InstallItemInfo& itemInfo, const oci::ContentDescriptor& descriptor, String& path); + Error InstallLayerBlob( + const InstallItemInfo& itemInfo, const oci::ContentDescriptor& descriptor, const String& diffID); + Error ProcessBlob(const InstallItemInfo& itemInfo, const String& path, const oci::ContentDescriptor& descriptor); + Error ProcessLayer(const InstallItemInfo& itemInfo, const String& path, const oci::ContentDescriptor& descriptor, + const String& diffID); + Error InstallItemBlobs(const InstallItemInfo& itemInfo); + Error InstallManifestBlobs(const InstallItemInfo& itemInfo, const oci::ContentDescriptor& descriptor); + Error StartItemInstall(const InstallItemInfo& itemInfo, DeployableItemInfo& storedInfo); + Error AddItemBlob(const String& itemID, const String& version, const oci::ContentDescriptor& descriptor, + bool isLayer = false, const String& diffID = ""); + + Error SetInProgressItemState( + const InstallItemInfo& itemInfo, ItemStateEnum state, const Error& error = ErrorEnum::eNone); + Error SetStoredItemState(DeployableItemInfo& item, ItemStateEnum state, const Error& error = ErrorEnum::eNone); + + void NotifyItemStatusChanged(const UpdateItemStatus& status); + + Error LockItem(const String& itemID, const String& version); + void UnlockItem(const String& itemID, const String& version); + UpdateItemStatus* FindInProgressItem(const String& itemID, const String& version); + Error LockBlob(const String& digest); + void UnlockBlob(const String& digest); + + bool IsBlobInProgress(const String& digest) const + { + return mBlobsInProgress.Find(digest) != mBlobsInProgress.end(); + } + Config mConfig; DownloadBlobInfoProviderItf* mBlobInfoProvider {}; StorageItf* mStorage {}; @@ -275,6 +328,11 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI Mutex mListenersMutex; StaticArray mListeners; + Mutex mMutex; + ConditionalVariable mCondVar; + StaticList mItemsInProgress; + StaticList, cMaxNumConcurrentItems> mBlobsInProgress; + ItemRemover mItemRemover {*this}; }; diff --git a/src/core/common/imagemanager/tests/imagemanager.cpp b/src/core/common/imagemanager/tests/imagemanager.cpp index e53d51ccc..41ed29d6b 100644 --- a/src/core/common/imagemanager/tests/imagemanager.cpp +++ b/src/core/common/imagemanager/tests/imagemanager.cpp @@ -4,6 +4,24 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include @@ -30,13 +48,52 @@ namespace { * Consts **********************************************************************************************************************/ +constexpr auto cTestDir = "/tmp/common_imagemanager_test"; constexpr auto cTestImagePath = "/tmp/common_imagemanager_test/images"; +constexpr auto cTestHash = "a3b2c1d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90"; constexpr auto cTestDigest = "sha256:a3b2c1d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90"; +constexpr auto cOtherHash = "0000000000000000000000000000000000000000000000000000000000000000"; +constexpr auto cOtherDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; +constexpr auto cTestBlobPath = "/tmp/common_imagemanager_test/images/blobs/sha256/" + "a3b2c1d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90"; +constexpr auto cTestLayerType = "application/vnd.oci.image.layer.v1.tar+gzip"; + +constexpr auto cMaxImagePathSize = cFilePathLen - (sizeof("/blobs/") - 1) - oci::cDigestLen; + +constexpr char cTestBlobContent[] = "test blob content"; +constexpr size_t cTestBlobSize = sizeof(cTestBlobContent) - 1; + +constexpr auto cTestItemID = "item1"; +constexpr auto cTestVersion = "1.0.0"; +constexpr auto cImageConfigType = "application/vnd.oci.image.config.v1+json"; +constexpr auto cItemConfigType = "application/vnd.aos.item.config.v1+json"; +constexpr auto cInstallBlobSize = 64; + +// Same as the image manager private constants. +constexpr auto cMaxNumStoredItems = cMaxNumUpdateItems * 2; +constexpr auto cMaxNumItemBlobs = 1 + oci::cMaxNumManifests * (3 + oci::cMaxNumLayers); /*********************************************************************************************************************** * Static **********************************************************************************************************************/ +void CreateFile(const String& path, const String& content) +{ + auto err = fs::MakeDirAll(fs::Dir(path)); + ASSERT_TRUE(err.IsNone()) << "Failed to create dir: " << tests::utils::ErrorToStr(err); + + err = fs::WriteStringToFile(path, content, 0600); + ASSERT_TRUE(err.IsNone()) << "Failed to create file: " << tests::utils::ErrorToStr(err); +} + +bool FileExists(const String& path) +{ + auto [exists, err] = fs::FileExist(path); + EXPECT_TRUE(err.IsNone()) << "Failed to check file: " << tests::utils::ErrorToStr(err); + + return exists; +} + DeployableItemInfo CreateDeployableItem( const String& itemID, const String& version, ItemStateEnum state, const Error& error = ErrorEnum::eNone) { @@ -53,6 +110,18 @@ DeployableItemInfo CreateDeployableItem( return item; } +std::string MakeImagePath(size_t size) +{ + auto path = std::string(cTestDir) + "/"; + + // Split the path into short folders to fit the file name limit. + while (path.size() < size) { + path += (path.size() % 100 == 0 && path.size() + 1 < size) ? '/' : 'a'; + } + + return path; +} + UpdateItemStatus CreateItemStatus(const String& itemID, const String& version, ItemStateEnum state) { UpdateItemStatus status; @@ -74,6 +143,78 @@ const UpdateItemStatus* FindStatus(const Array& statuses, cons return it != statuses.end() ? it : nullptr; } +/** + * Test blob: the blob content is its hash, so the checksum is calculated from the content. + */ +struct TestBlob { + std::string mHash; + std::string mDigest; + std::string mPath; +}; + +TestBlob MakeTestBlob(uint32_t id) +{ + char hash[65]; + + std::snprintf(hash, sizeof(hash), "%064x", id); + + TestBlob blob; + + blob.mHash = hash; + blob.mDigest = std::string("sha256:") + hash; + blob.mPath = std::string(cTestImagePath) + "/blobs/sha256/" + hash; + + return blob; +} + +oci::ContentDescriptor MakeDescriptor(const TestBlob& blob, const char* mediaType) +{ + return oci::ContentDescriptor {mediaType, blob.mDigest.c_str(), cInstallBlobSize}; +} + +std::string ReadFile(const std::string& path) +{ + std::ifstream file(path); + std::stringstream content; + + content << file.rdbuf(); + + return content.str(); +} + +Error GetFileInfoFromContent(const String& path, fs::FileInfo& info, const crypto::Hash&) +{ + auto content = ReadFile(path.CStr()); + + info.mCheckSum.Clear(); + info.mSize = content.size(); + + // Content which is not a hash gives an empty checksum, so the validation fails. + if (content.size() == crypto::cSHA256Size * 2) { + StaticString hash {content.c_str()}; + + if (!hash.HexToByteArray(info.mCheckSum).IsNone()) { + info.mCheckSum.Clear(); + } + } + + return ErrorEnum::eNone; +} + +Error GetTestBlobInfo(const String& digest, BlobInfo& blobInfo) +{ + blobInfo.mDigest = digest; + blobInfo.mSize = cInstallBlobSize; + + return ErrorEnum::eNone; +} + +Error DownloadTestBlob(const String& path, const BlobInfo& blobInfo) +{ + // Write the blob hash as the blob content. + return fs::WriteStringToFile(path, String(blobInfo.mDigest.CStr() + std::strlen("sha256:")), 0600); +} + /*********************************************************************************************************************** * Test image manager **********************************************************************************************************************/ @@ -83,11 +224,46 @@ class TestImageManager : public ImageManager { using ImageManager::Init; using ImageManager::InstallItemInfo; + struct LayerCall { + std::string mDigest; + std::string mDiffDigest; + + bool operator==(const LayerCall& rhs) const { return mDigest == rhs.mDigest && mDiffDigest == rhs.mDiffDigest; } + }; + Error Install(const InstallItemInfo& itemInfo) { return InstallItem(itemInfo); } Error Remove(const String& itemID, const String& version) { return RemoveItem(itemID, version); } + Error ValidateLayer(const String& path, const oci::ContentDescriptor& descriptor, const String& diffID) override + { + { + std::lock_guard lock {mLayerCallsMutex}; + + mValidateLayerCalls.push_back({descriptor.mDigest.CStr(), diffID.CStr()}); + } + + return CheckLayer(path, descriptor, diffID); + } + + std::vector GetValidateLayerCalls() + { + std::lock_guard lock {mLayerCallsMutex}; + + return mValidateLayerCalls; + } + MOCK_METHOD(Error, DownloadBlob, (const String& path, const BlobInfo& blobInfo), (override)); MOCK_METHOD(Error, CancelDownload, (const String& digest), (override)); + MOCK_METHOD( + Error, CheckLayer, (const String& path, const oci::ContentDescriptor& descriptor, const String& diffID)); + MOCK_METHOD(Error, UnpackLayer, + (const String& path, const oci::ContentDescriptor& descriptor, const String& diffID), (override)); + MOCK_METHOD((RetWithError), RemoveLayer, + (const String& path, const oci::ContentDescriptor& descriptor, const String& diffID), (override)); + +private: + std::mutex mLayerCallsMutex; + std::vector mValidateLayerCalls; }; } // namespace @@ -100,9 +276,27 @@ class CommonImageManagerTest : public Test { protected: static void SetUpTestSuite() { tests::utils::InitLog(); } - Error InitImageManager() + void SetUp() override { - Config config {cTestImagePath, 0, 10 * Time::cSeconds, 5 * Time::cSeconds}; + auto err = fs::RemoveAll(cTestDir); + ASSERT_TRUE(err.IsNone()) << "Failed to remove test dir: " << tests::utils::ErrorToStr(err); + + // By default, the layers are kept as packed blobs only. + ON_CALL(mImageManager, CheckLayer(_, _, _)).WillByDefault(Return(ErrorEnum::eNotSupported)); + ON_CALL(mImageManager, UnpackLayer(_, _, _)).WillByDefault(Return(ErrorEnum::eNotSupported)); + ON_CALL(mImageManager, RemoveLayer(_, _, _)) + .WillByDefault(Return(RetWithError(0, ErrorEnum::eNotSupported))); + } + + void TearDown() override + { + auto err = fs::RemoveAll(cTestDir); + EXPECT_TRUE(err.IsNone()) << "Failed to remove test dir: " << tests::utils::ErrorToStr(err); + } + + Error InitImageManager(const String& imagePath = cTestImagePath) + { + Config config {imagePath, 0, 10 * Time::cSeconds, 5 * Time::cSeconds}; return mImageManager.Init( config, mBlobInfoProvider, mStorage, mSpaceAllocator, mFileInfoProvider, mOCISpec, mFSPlatform, mAllocator); @@ -115,6 +309,8 @@ class CommonImageManagerTest : public Test { ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); } + // mAllocator must be declared before any member that allocates from it, so it is destroyed after them, since + // members are destroyed in reverse declaration order. HeapAllocator mAllocator; NiceMock mBlobInfoProvider; StorageStub mStorage; @@ -134,6 +330,84 @@ TEST_F(CommonImageManagerTest, Init) { auto err = InitImageManager(); ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + auto [exists, existErr] = fs::DirExist(fs::JoinPath(cTestImagePath, "blobs")); + ASSERT_TRUE(existErr.IsNone()) << tests::utils::ErrorToStr(existErr); + EXPECT_TRUE(exists); +} + +TEST_F(CommonImageManagerTest, InitFailsIfImageDirCantBeCreated) +{ + const auto filePath = fs::JoinPath(cTestDir, "file"); + + CreateFile(filePath, "content"); + + auto err = InitImageManager(fs::JoinPath(filePath, "images")); + EXPECT_FALSE(err.IsNone()); +} + +TEST_F(CommonImageManagerTest, InitFailsIfImagePathTooLong) +{ + const auto imagePath = MakeImagePath(cMaxImagePathSize + 1); + + auto err = InitImageManager(imagePath.c_str()); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidArgument)) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerTest, GetBlobPathLongestImagePathAndDigest) +{ + const auto imagePath = MakeImagePath(cMaxImagePathSize); + const auto hash = std::string(oci::cDigestLen - strlen("sha256:"), '0'); + + auto err = InitImageManager(imagePath.c_str()); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + StaticString path; + + err = mImageManager.GetBlobPath(("sha256:" + hash).c_str(), path); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + EXPECT_EQ(path.Size(), cFilePathLen); + EXPECT_STREQ(path.CStr(), (imagePath + "/blobs/sha256/" + hash).c_str()); +} + +TEST_F(CommonImageManagerTest, GetBlobPathDigestTooLong) +{ + const auto imagePath = MakeImagePath(cMaxImagePathSize); + const auto digest = "sha256:" + std::string(oci::cDigestLen, '0'); + + auto err = InitImageManager(imagePath.c_str()); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + StaticString path; + + err = mImageManager.GetBlobPath(digest.c_str(), path); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidArgument)) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerTest, GetBlobPath) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + StaticString path; + + auto err = mImageManager.GetBlobPath(cTestDigest, path); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + EXPECT_EQ(path, cTestBlobPath); + + EXPECT_FALSE(FileExists(path)); +} + +TEST_F(CommonImageManagerTest, GetBlobPathMalformedDigest) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + for (const auto& digest : {"", "sha256", "sha256:", ":abc", "sha256:ab:cd", "sha256:../abc", "../x:abc", + "sha256:ab/cd", "SHA256:abc", ".:abc", "sha256:ab-cd", "sha256:ab_cd", "sha256:abc="}) { + StaticString path; + + auto err = mImageManager.GetBlobPath(digest, path); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidArgument)) << digest << ": " << tests::utils::ErrorToStr(err); + } } TEST_F(CommonImageManagerTest, SubscribeListenerTwice) @@ -176,7 +450,7 @@ TEST_F(CommonImageManagerTest, GetItemsStatusesEmpty) { ASSERT_TRUE(InitImageManager().IsNone()); - StaticArray statuses; + StaticArray statuses; auto err = mImageManager.GetItemsStatuses(statuses); ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); @@ -192,7 +466,7 @@ TEST_F(CommonImageManagerTest, GetItemsStatusesStoredStates) AddStoredItem("item2", "1.0.0", ItemStateEnum::eFailed, Error(ErrorEnum::eFailed, "download failed")); AddStoredItem("item3", "1.0.0", ItemStateEnum::eInstalling); - StaticArray statuses; + StaticArray statuses; // The statuses are appended to the output array. const auto preFilledStatus = CreateItemStatus("item0", "1.0.0", ItemStateEnum::eInstalled); @@ -225,9 +499,10 @@ TEST_F(CommonImageManagerTest, GetItemsStatusesStoredStates) EXPECT_TRUE(status->mError.Is(ErrorEnum::eFailed)); EXPECT_STREQ(status->mError.Message(), "download failed"); + // The item left in eInstalling state and not being installed is reported as eFailed without an error. status = FindStatus(statuses, "item3", "1.0.0"); ASSERT_NE(status, nullptr); - EXPECT_EQ(status->mState, ItemStateEnum::eInstalling); + EXPECT_EQ(status->mState, ItemStateEnum::eFailed); EXPECT_TRUE(status->mError.IsNone()); } @@ -244,4 +519,1507 @@ TEST_F(CommonImageManagerTest, GetItemsStatusesNoMemory) EXPECT_TRUE(err.Is(ErrorEnum::eNoMemory)) << tests::utils::ErrorToStr(err); } +/*********************************************************************************************************************** + * Install item suite + **********************************************************************************************************************/ + +class CommonImageManagerInstallTest : public CommonImageManagerTest { +protected: + struct TestManifest { + TestBlob mManifest; + TestBlob mImageConfig; + TestBlob mItemConfig; + std::vector mLayers; + std::vector mDiffIDs; + }; + + void SetUp() override + { + CommonImageManagerTest::SetUp(); + + ON_CALL(mFileInfoProvider, GetFileInfo(_, _, _)).WillByDefault(Invoke(GetFileInfoFromContent)); + ON_CALL(mBlobInfoProvider, GetBlobInfo(_, _)).WillByDefault(Invoke(GetTestBlobInfo)); + ON_CALL(mImageManager, DownloadBlob(_, _)).WillByDefault(Invoke(DownloadTestBlob)); + + ON_CALL(mSpaceAllocator, AllocateSpace(_)) + .WillByDefault(Invoke([this](size_t) -> RetWithError> { + auto space = MakeUnique>(&mAllocator); + + ON_CALL(*space, Accept()).WillByDefault(Invoke([this]() { + mAcceptCount++; + + return ErrorEnum::eNone; + })); + ON_CALL(*space, Release()).WillByDefault(Invoke([this]() { + mReleaseCount++; + + return ErrorEnum::eNone; + })); + ON_CALL(*space, Resize(_)).WillByDefault(Invoke([this](size_t size) -> Error { + mLastResizeSize = size; + + return mResizeError; + })); + + return {Move(space), ErrorEnum::eNone}; + })); + + ON_CALL(mOCISpec, LoadImageIndex(_, _)) + .WillByDefault(Invoke([this](const String& path, oci::ImageIndex& index) -> Error { + std::lock_guard lock {mOCIMutex}; + + auto it = mIndexes.find(path.CStr()); + if (it == mIndexes.end()) { + return ErrorEnum::eNotFound; + } + + index = *it->second; + + return ErrorEnum::eNone; + })); + ON_CALL(mOCISpec, LoadImageManifest(_, _)) + .WillByDefault(Invoke([this](const String& path, oci::ImageManifest& manifest) -> Error { + std::lock_guard lock {mOCIMutex}; + + auto it = mManifests.find(path.CStr()); + if (it == mManifests.end()) { + return ErrorEnum::eNotFound; + } + + manifest = *it->second; + + return ErrorEnum::eNone; + })); + ON_CALL(mOCISpec, LoadImageConfig(_, _)) + .WillByDefault(Invoke([this](const String& path, oci::ImageConfig& config) -> Error { + std::lock_guard lock {mOCIMutex}; + + auto it = mImageConfigs.find(path.CStr()); + if (it == mImageConfigs.end()) { + return ErrorEnum::eNotFound; + } + + config = *it->second; + + return ErrorEnum::eNone; + })); + + ON_CALL(mListener, OnItemStatusChanged(_)).WillByDefault(Invoke([this](const UpdateItemStatus& status) { + std::lock_guard lock {mStatusesMutex}; + + mStatuses.push_back(status); + })); + + ASSERT_TRUE(InitImageManager().IsNone()); + ASSERT_TRUE(mImageManager.SubscribeListener(mListener).IsNone()); + } + + // Registers manifest with image config, optional item config and layers. Blob IDs start from base. + TestManifest AddManifest(uint32_t base, size_t numLayers, size_t numDiffIDs, bool withItemConfig = false, + const std::vector& sharedLayers = {}) + { + TestManifest testManifest; + + testManifest.mManifest = MakeTestBlob(base); + testManifest.mImageConfig = MakeTestBlob(base + 1); + testManifest.mItemConfig = MakeTestBlob(base + 2); + + auto manifest = std::make_shared(); + + manifest->mMediaType = oci::cMediaTypeImageManifest; + manifest->mConfig = MakeDescriptor(testManifest.mImageConfig, cImageConfigType); + + if (withItemConfig) { + manifest->mItemConfig.SetValue(MakeDescriptor(testManifest.mItemConfig, cItemConfigType)); + } + + testManifest.mLayers = sharedLayers; + + for (size_t i = 0; i < numLayers; ++i) { + testManifest.mLayers.push_back(MakeTestBlob(base + 10 + i)); + } + + for (const auto& layer : testManifest.mLayers) { + EXPECT_TRUE(manifest->mLayers.PushBack(MakeDescriptor(layer, cTestLayerType)).IsNone()); + } + + auto imageConfig = std::make_shared(); + + for (size_t i = 0; i < numDiffIDs; ++i) { + testManifest.mDiffIDs.push_back(MakeTestBlob(base + 100 + i)); + + EXPECT_TRUE( + imageConfig->mRootfs.mDiffIDs.EmplaceBack(testManifest.mDiffIDs.back().mDigest.c_str()).IsNone()); + } + + std::lock_guard lock {mOCIMutex}; + + mManifests[testManifest.mManifest.mPath] = manifest; + mImageConfigs[testManifest.mImageConfig.mPath] = imageConfig; + + return testManifest; + } + + TestBlob AddIndex(uint32_t id, const std::vector& manifests) + { + auto blob = MakeTestBlob(id); + auto index = std::make_shared(); + + index->mMediaType = oci::cMediaTypeImageIndex; + + for (const auto& manifest : manifests) { + oci::IndexContentDescriptor descriptor; + + static_cast(descriptor) + = MakeDescriptor(manifest.mManifest, oci::cMediaTypeImageManifest); + + EXPECT_TRUE(index->mManifests.PushBack(descriptor).IsNone()); + } + + std::lock_guard lock {mOCIMutex}; + + mIndexes[blob.mPath] = index; + + return blob; + } + + void CreateValidBlobs(const TestManifest& manifest) + { + for (const auto* blob : {&manifest.mManifest, &manifest.mImageConfig, &manifest.mItemConfig}) { + CreateFile(blob->mPath.c_str(), blob->mHash.c_str()); + } + + for (const auto& layer : manifest.mLayers) { + CreateFile(layer.mPath.c_str(), layer.mHash.c_str()); + } + } + + static TestImageManager::InstallItemInfo MakeInstallInfo(const TestBlob& root, bool isRootIndex = false, + UpdateItemTypeEnum type = UpdateItemTypeEnum::eService, const char* itemID = cTestItemID, + const char* version = cTestVersion) + { + TestImageManager::InstallItemInfo itemInfo; + + itemInfo.mItemID = itemID; + itemInfo.mType = type; + itemInfo.mVersion = version; + itemInfo.mRootDigest = root.mDigest.c_str(); + itemInfo.mIsRootIndex = isRootIndex; + + return itemInfo; + } + + void StoreItem(const String& itemID, const String& version, const String& rootDigest, ItemStateEnum state) + { + auto item = CreateDeployableItem(itemID, version, state); + + item.mRootDigest = rootDigest; + + auto err = mStorage.AddDeployableItem(item); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + } + + std::optional GetStoredItem(const String& itemID, const String& version) + { + DeployableItemInfo item; + + if (!mStorage.GetDeployableItem(itemID, version, item).IsNone()) { + return std::nullopt; + } + + return item; + } + + std::vector GetNotifiedStatuses() + { + std::lock_guard lock {mStatusesMutex}; + + return mStatuses; + } + + std::vector GetNotifiedStates() + { + std::vector states; + + for (const auto& status : GetNotifiedStatuses()) { + states.push_back(status.mState.GetValue()); + } + + return states; + } + + std::optional GetReportedState(const String& itemID, const String& version) + { + auto statuses = std::make_unique>(); + + auto err = mImageManager.GetItemsStatuses(*statuses); + EXPECT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + const auto* status = FindStatus(*statuses, itemID, version); + if (status == nullptr) { + return std::nullopt; + } + + return status->mState.GetValue(); + } + + static ItemBlobInfo MakeBlobInfo(const TestBlob& blob, const char* mediaType, bool isLayer = false, + const std::string& diffID = "", size_t size = cInstallBlobSize) + { + ItemBlobInfo info; + + info.mDigest = blob.mDigest.c_str(); + info.mSize = size; + info.mMediaType = mediaType; + info.mDiffID = diffID.c_str(); + info.mIsLayer = isLayer; + + return info; + } + + std::atomic mAcceptCount {0}; + std::atomic mReleaseCount {0}; + std::atomic mLastResizeSize {0}; + Error mResizeError; + + std::mutex mOCIMutex; + std::map> mIndexes; + std::map> mManifests; + std::map> mImageConfigs; + + std::mutex mStatusesMutex; + std::vector mStatuses; +}; + +TEST_F(CommonImageManagerInstallTest, InstallNotifiesSubscribedListeners) +{ + const auto manifest = AddManifest(1, 1, 1); + + NiceMock otherListener; + + ASSERT_TRUE(mImageManager.SubscribeListener(otherListener).IsNone()); + + const auto expectedStates = std::vector( + {ItemStateEnum::eInstalling, ItemStateEnum::eDownloading, ItemStateEnum::eInstalled}); + + std::vector otherStates; + + EXPECT_CALL(otherListener, OnItemStatusChanged(_)).WillRepeatedly(Invoke([&](const UpdateItemStatus& status) { + otherStates.push_back(status.mState.GetValue()); + })); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), expectedStates); + EXPECT_EQ(otherStates, expectedStates); + + ASSERT_TRUE(mImageManager.UnsubscribeListener(otherListener).IsNone()); + + const auto otherManifest = AddManifest(2, 1, 1); + + err = mImageManager.Install(MakeInstallInfo(otherManifest.mManifest, false, UpdateItemTypeEnum::eService, "item2")); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates().size(), 2 * expectedStates.size()); + EXPECT_EQ(otherStates, expectedStates); +} + +TEST_F(CommonImageManagerInstallTest, InstallManifestRoot) +{ + const auto manifest = AddManifest(1, 2, 2, true); + const auto itemInfo = MakeInstallInfo(manifest.mManifest); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(5); + EXPECT_CALL(mSpaceAllocator, AllocateSpace(cInstallBlobSize)).Times(5); + EXPECT_CALL(mFSPlatform, Sync(String(cTestImagePath), true)).Times(1); + + auto err = mImageManager.Install(itemInfo); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), + std::vector( + {ItemStateEnum::eInstalling, ItemStateEnum::eDownloading, ItemStateEnum::eInstalled})); + + for (const auto& status : GetNotifiedStatuses()) { + EXPECT_EQ(status.mItemID, cTestItemID); + EXPECT_EQ(status.mVersion, cTestVersion); + EXPECT_EQ(status.mType, UpdateItemTypeEnum::eService); + EXPECT_TRUE(status.mError.IsNone()); + } + + EXPECT_EQ(mStorage.GetBlobs(cTestItemID, cTestVersion), + std::vector({ + MakeBlobInfo(manifest.mManifest, oci::cMediaTypeImageManifest, false, "", 0), + MakeBlobInfo(manifest.mItemConfig, cItemConfigType), + MakeBlobInfo(manifest.mImageConfig, cImageConfigType), + MakeBlobInfo(manifest.mLayers[0], cTestLayerType, true, manifest.mDiffIDs[0].mDigest), + MakeBlobInfo(manifest.mLayers[1], cTestLayerType, true, manifest.mDiffIDs[1].mDigest), + })); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); + EXPECT_EQ(item->mRootDigest, manifest.mManifest.mDigest.c_str()); + EXPECT_EQ(item->mType, UpdateItemTypeEnum::eService); + EXPECT_TRUE(item->mError.IsNone()); + + EXPECT_EQ(mAcceptCount.load(), 5); + EXPECT_EQ(mReleaseCount.load(), 0); + + for (const auto& layer : manifest.mLayers) { + EXPECT_TRUE(FileExists(layer.mPath.c_str())); + } + + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eInstalled); +} + +TEST_F(CommonImageManagerInstallTest, InstallIndexRoot) +{ + const auto manifest = AddManifest(1, 1, 1); + const auto index = AddIndex(1000, {manifest}); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(4); + EXPECT_CALL(mSpaceAllocator, AllocateSpace(cInstallBlobSize)).Times(4); + + auto err = mImageManager.Install(MakeInstallInfo(index, true)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), + std::vector( + {ItemStateEnum::eInstalling, ItemStateEnum::eDownloading, ItemStateEnum::eInstalled})); + + EXPECT_EQ(mStorage.GetBlobs(cTestItemID, cTestVersion), + std::vector({ + MakeBlobInfo(index, oci::cMediaTypeImageIndex, false, "", 0), + MakeBlobInfo(manifest.mManifest, oci::cMediaTypeImageManifest), + MakeBlobInfo(manifest.mImageConfig, cImageConfigType), + MakeBlobInfo(manifest.mLayers[0], cTestLayerType, true, manifest.mDiffIDs[0].mDigest), + })); + + EXPECT_TRUE(FileExists(index.mPath.c_str())); + EXPECT_TRUE(FileExists(manifest.mManifest.mPath.c_str())); + EXPECT_TRUE(FileExists(manifest.mImageConfig.mPath.c_str())); + EXPECT_TRUE(FileExists(manifest.mLayers[0].mPath.c_str())); + EXPECT_EQ(mAcceptCount.load(), 4); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); +} + +TEST_F(CommonImageManagerInstallTest, InstallAllBlobsValid) +{ + const auto manifest = AddManifest(1, 2, 2, true); + + CreateValidBlobs(manifest); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mSpaceAllocator, AllocateSpace(_)).Times(0); + EXPECT_CALL(mSpaceAllocator, FreeSpace(_)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eInstalled})); + EXPECT_EQ(mStorage.GetBlobs(cTestItemID, cTestVersion).size(), 5); +} + +TEST_F(CommonImageManagerInstallTest, InstallInvalidLayer) +{ + const auto manifest = AddManifest(1, 2, 2); + + CreateValidBlobs(manifest); + CreateFile(manifest.mLayers[1].mPath.c_str(), cOtherHash); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mImageManager, DownloadBlob(String(manifest.mLayers[1].mPath.c_str()), _)).Times(1); + EXPECT_CALL(mSpaceAllocator, FreeSpace(cInstallBlobSize)).Times(1); + EXPECT_CALL(mSpaceAllocator, AllocateSpace(cInstallBlobSize)).Times(1); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(ReadFile(manifest.mLayers[1].mPath), manifest.mLayers[1].mHash); + EXPECT_EQ(mAcceptCount.load(), 1); + EXPECT_EQ(GetNotifiedStates(), + std::vector( + {ItemStateEnum::eInstalling, ItemStateEnum::eDownloading, ItemStateEnum::eInstalled})); +} + +TEST_F(CommonImageManagerInstallTest, InstallInvalidMetadataBlob) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + CreateFile(manifest.mImageConfig.mPath.c_str(), cOtherHash); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mImageManager, DownloadBlob(String(manifest.mImageConfig.mPath.c_str()), _)).Times(1); + EXPECT_CALL(mSpaceAllocator, FreeSpace(cInstallBlobSize)).Times(1); + EXPECT_CALL(mSpaceAllocator, AllocateSpace(cInstallBlobSize)).Times(1); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(ReadFile(manifest.mImageConfig.mPath), manifest.mImageConfig.mHash); + EXPECT_EQ(mAcceptCount.load(), 1); +} + +TEST_F(CommonImageManagerInstallTest, InstallServiceDiffIDs) +{ + const auto manifest = AddManifest(1, 2, 2); + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(manifest.mLayers[1].mPath.c_str()).IsNone()); + + EXPECT_CALL(mOCISpec, LoadImageConfig(String(manifest.mImageConfig.mPath.c_str()), _)).Times(1); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mImageManager, DownloadBlob(String(manifest.mLayers[1].mPath.c_str()), _)).Times(1); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + using LayerCall = TestImageManager::LayerCall; + + EXPECT_EQ(mImageManager.GetValidateLayerCalls(), + std::vector({{manifest.mLayers[0].mDigest, manifest.mDiffIDs[0].mDigest}, + {manifest.mLayers[1].mDigest, manifest.mDiffIDs[1].mDigest}})); +} + +TEST_F(CommonImageManagerInstallTest, InstallLayerValid) +{ + const auto manifest = AddManifest(1, 1, 1); + const auto& layer = manifest.mLayers[0]; + + // The layer blob is removed after the layer is unpacked. + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(layer.mPath.c_str()).IsNone()); + + EXPECT_CALL(mImageManager, CheckLayer(String(layer.mPath.c_str()), _, String(manifest.mDiffIDs[0].mDigest.c_str()))) + .WillOnce(Return(ErrorEnum::eNone)); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mImageManager, RemoveLayer(_, _, _)).Times(0); + EXPECT_CALL(mImageManager, UnpackLayer(_, _, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_FALSE(FileExists(layer.mPath.c_str())); +} + +TEST_F(CommonImageManagerInstallTest, InstallLayerNotSupported) +{ + const auto manifest = AddManifest(1, 2, 2); + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(manifest.mLayers[1].mPath.c_str()).IsNone()); + + EXPECT_CALL(mImageManager, CheckLayer(_, _, _)).Times(2); + EXPECT_CALL(mImageManager, DownloadBlob(String(manifest.mLayers[1].mPath.c_str()), _)).Times(1); + EXPECT_CALL(mImageManager, RemoveLayer(_, _, _)).Times(0); + EXPECT_CALL(mImageManager, UnpackLayer(_, _, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerInstallTest, InstallLayerInvalid) +{ + const auto manifest = AddManifest(1, 1, 1); + const auto& layer = manifest.mLayers[0]; + const auto diffID = String(manifest.mDiffIDs[0].mDigest.c_str()); + + constexpr size_t cUnpackedSize = 100; + + CreateValidBlobs(manifest); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + { + InSequence seq; + + EXPECT_CALL(mImageManager, CheckLayer(String(layer.mPath.c_str()), _, diffID)) + .WillOnce(Return(ErrorEnum::eInvalidChecksum)); + EXPECT_CALL(mImageManager, RemoveLayer(String(layer.mPath.c_str()), _, diffID)) + .WillOnce(Return(RetWithError(cUnpackedSize))); + EXPECT_CALL(mSpaceAllocator, FreeSpace(cUnpackedSize)); + EXPECT_CALL(mImageManager, UnpackLayer(String(layer.mPath.c_str()), _, diffID)) + .WillOnce(Return(ErrorEnum::eNone)); + } + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerInstallTest, InstallLayerMissing) +{ + const auto manifest = AddManifest(1, 1, 1); + const auto& layer = manifest.mLayers[0]; + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(layer.mPath.c_str()).IsNone()); + + { + InSequence seq; + + EXPECT_CALL(mImageManager, CheckLayer(String(layer.mPath.c_str()), _, _)) + .WillOnce(Return(ErrorEnum::eNotFound)); + EXPECT_CALL(mImageManager, RemoveLayer(String(layer.mPath.c_str()), _, _)); + EXPECT_CALL(mImageManager, DownloadBlob(String(layer.mPath.c_str()), _)); + EXPECT_CALL(mImageManager, UnpackLayer(String(layer.mPath.c_str()), _, _)).WillOnce(Return(ErrorEnum::eNone)); + } + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerInstallTest, InstallRemoveLayerFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + EXPECT_CALL(mImageManager, CheckLayer(_, _, _)).WillOnce(Return(ErrorEnum::eNotFound)); + EXPECT_CALL(mImageManager, RemoveLayer(_, _, _)) + .WillOnce(Return(RetWithError(0, Error(ErrorEnum::eFailed, "remove failed")))); + EXPECT_CALL(mImageManager, UnpackLayer(_, _, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_STREQ(item->mError.Message(), "remove failed"); +} + +TEST_F(CommonImageManagerInstallTest, InstallUnpackLayerFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + EXPECT_CALL(mImageManager, CheckLayer(_, _, _)).WillOnce(Return(ErrorEnum::eNotFound)); + EXPECT_CALL(mImageManager, UnpackLayer(_, _, _)).WillOnce(Return(Error(ErrorEnum::eFailed, "unpack failed"))); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_STREQ(item->mError.Message(), "unpack failed"); +} + +TEST_F(CommonImageManagerInstallTest, InstallWithoutImageConfig) +{ + const auto manifest = AddManifest(1, 2, 0); + + mManifests[manifest.mManifest.mPath]->mConfig = oci::ContentDescriptor {}; + + EXPECT_CALL(mOCISpec, LoadImageConfig(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_FALSE(FileExists(manifest.mImageConfig.mPath.c_str())); + + using LayerCall = TestImageManager::LayerCall; + + const std::vector expectedCalls {{manifest.mLayers[0].mDigest, ""}, {manifest.mLayers[1].mDigest, ""}}; + + EXPECT_EQ(mImageManager.GetValidateLayerCalls(), expectedCalls); + + EXPECT_EQ(mStorage.GetBlobs(cTestItemID, cTestVersion), + std::vector({ + MakeBlobInfo(manifest.mManifest, oci::cMediaTypeImageManifest, false, "", 0), + MakeBlobInfo(manifest.mLayers[0], cTestLayerType, true), + MakeBlobInfo(manifest.mLayers[1], cTestLayerType, true), + })); +} + +TEST_F(CommonImageManagerInstallTest, InstallEmptyConfig) +{ + const auto manifest = AddManifest(1, 1, 0); + + CreateValidBlobs(manifest); + + // A component has an empty config and layers without diff IDs. + mManifests[manifest.mManifest.mPath]->mConfig.mMediaType = oci::cMediaTypeEmptyBlob; + + EXPECT_CALL(mOCISpec, LoadImageConfig(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + const auto calls = mImageManager.GetValidateLayerCalls(); + ASSERT_EQ(calls.size(), 1); + EXPECT_TRUE(calls[0].mDiffDigest.empty()); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); +} + +TEST_F(CommonImageManagerInstallTest, InstallMissingDiffID) +{ + const auto manifest = AddManifest(1, 2, 1); + + CreateValidBlobs(manifest); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidArgument)) << tests::utils::ErrorToStr(err); + + EXPECT_TRUE(mImageManager.GetValidateLayerCalls().empty()); + + auto statuses = GetNotifiedStatuses(); + ASSERT_EQ(statuses.size(), 2); + EXPECT_EQ(statuses[0].mState, ItemStateEnum::eInstalling); + EXPECT_EQ(statuses[1].mState, ItemStateEnum::eFailed); + EXPECT_TRUE(statuses[1].mError.Is(ErrorEnum::eInvalidArgument)); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eInvalidArgument)); +} + +TEST_F(CommonImageManagerInstallTest, InstallRootDigestMismatch) +{ + const auto manifest = AddManifest(1, 1, 1); + + StoreItem(cTestItemID, cTestVersion, cOtherDigest, ItemStateEnum::eInstalled); + + const auto storedItems = mStorage.GetItems(); + + EXPECT_CALL(mListener, OnItemStatusChanged(_)).Times(0); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mFSPlatform, Sync(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidArgument)) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(mStorage.GetItems(), storedItems); + EXPECT_TRUE(mStorage.GetBlobs(cTestItemID, cTestVersion).empty()); +} + +TEST_F(CommonImageManagerInstallTest, InstallMalformedRootDigest) +{ + auto itemInfo = MakeInstallInfo(MakeTestBlob(1)); + + itemInfo.mRootDigest = "sha256:../malformed"; + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mSpaceAllocator, AllocateSpace(_)).Times(0); + + auto err = mImageManager.Install(itemInfo); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidArgument)) << tests::utils::ErrorToStr(err); + + auto statuses = GetNotifiedStatuses(); + ASSERT_EQ(statuses.size(), 2); + EXPECT_EQ(statuses[0].mState, ItemStateEnum::eInstalling); + EXPECT_EQ(statuses[1].mState, ItemStateEnum::eFailed); + EXPECT_TRUE(statuses[1].mError.Is(ErrorEnum::eInvalidArgument)); + + EXPECT_TRUE(mStorage.GetBlobs(cTestItemID, cTestVersion).empty()); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eInvalidArgument)); +} + +TEST_F(CommonImageManagerInstallTest, InstallMalformedLayerDigest) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + constexpr auto cMalformedDigest = "sha256:../malformed"; + + mManifests[manifest.mManifest.mPath]->mLayers[0].mDigest = cMalformedDigest; + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mSpaceAllocator, AllocateSpace(_)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidArgument)) << tests::utils::ErrorToStr(err); + + EXPECT_TRUE(mImageManager.GetValidateLayerCalls().empty()); + + auto statuses = GetNotifiedStatuses(); + ASSERT_EQ(statuses.size(), 2); + EXPECT_EQ(statuses[0].mState, ItemStateEnum::eInstalling); + EXPECT_EQ(statuses[1].mState, ItemStateEnum::eFailed); + EXPECT_TRUE(statuses[1].mError.Is(ErrorEnum::eInvalidArgument)); + + // No blob reference is stored for the layer with the malformed digest. + EXPECT_EQ(mStorage.GetBlobs(cTestItemID, cTestVersion), + std::vector({ + MakeBlobInfo(manifest.mManifest, oci::cMediaTypeImageManifest, false, "", 0), + MakeBlobInfo(manifest.mImageConfig, cImageConfigType), + })); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eInvalidArgument)); +} + +TEST_F(CommonImageManagerInstallTest, InstallUnsupportedDigestAlgorithm) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + mManifests[manifest.mManifest.mPath]->mLayers[0].mDigest = "md5:0123456789abcdef0123456789abcdef"; + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eNotSupported)) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eNotSupported)); +} + +TEST_F(CommonImageManagerInstallTest, InstallOversizedDigestAlgorithm) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + const auto digest = std::string(crypto::cAlgLen + 1, 'a') + ":" + std::string(64, '0'); + + mManifests[manifest.mManifest.mPath]->mLayers[0].mDigest = digest.c_str(); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eNotSupported)) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eNotSupported)); +} + +TEST_F(CommonImageManagerInstallTest, InstallBlobIntegrityError) +{ + const auto manifest = AddManifest(1, 1, 1); + const auto& layer = manifest.mLayers[0]; + + CreateValidBlobs(manifest); + + EXPECT_CALL(mFileInfoProvider, GetFileInfo(_, _, _)).WillRepeatedly(Invoke(GetFileInfoFromContent)); + EXPECT_CALL(mFileInfoProvider, GetFileInfo(String(layer.mPath.c_str()), _, _)) + .WillOnce(Return(Error(ErrorEnum::eIntegrity))) + .WillRepeatedly(Invoke(GetFileInfoFromContent)); + + // The corrupted blob is downloaded again. + EXPECT_CALL(mImageManager, DownloadBlob(String(layer.mPath.c_str()), _)).WillOnce(Invoke(DownloadTestBlob)); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerInstallTest, InstallBlobReadFails) +{ + const auto manifest = AddManifest(1, 1, 1); + const auto& layer = manifest.mLayers[0]; + + CreateValidBlobs(manifest); + + ON_CALL(mFileInfoProvider, GetFileInfo(String(layer.mPath.c_str()), _, _)) + .WillByDefault(Return(Error(ErrorEnum::eFailed))); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + // The read error doesn't mean the blob is corrupted, so it is kept. + EXPECT_TRUE(FileExists(layer.mPath.c_str())); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eFailed)); +} + +TEST_F(CommonImageManagerInstallTest, InstallMalformedDigestHash) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + // The digest format is valid, but the hash is not a hex string. + mManifests[manifest.mManifest.mPath]->mLayers[0].mDigest = "sha256:zz"; + + EXPECT_CALL(mBlobInfoProvider, GetBlobInfo(_, _)).Times(0); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidArgument)) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerInstallTest, InstallDownloadedBlobSizeMismatch) +{ + constexpr auto cContent = "short"; + + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(manifest.mLayers[0].mPath.c_str()).IsNone()); + + EXPECT_CALL(mImageManager, DownloadBlob(String(manifest.mLayers[0].mPath.c_str()), _)) + .WillOnce( + Invoke([](const String& path, const BlobInfo&) { return fs::WriteStringToFile(path, cContent, 0600); })); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidChecksum)) << tests::utils::ErrorToStr(err); + + // The invalid blob is kept, so the accepted space matches its size. + EXPECT_TRUE(FileExists(manifest.mLayers[0].mPath.c_str())); + EXPECT_EQ(mLastResizeSize.load(), strlen(cContent)); + EXPECT_EQ(mAcceptCount.load(), 1); + EXPECT_EQ(mReleaseCount.load(), 0); +} + +TEST_F(CommonImageManagerInstallTest, InstallResizeSpaceFails) +{ + const auto manifest = AddManifest(1, 1, 1); + const auto& layer = manifest.mLayers[0]; + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(layer.mPath.c_str()).IsNone()); + + mResizeError = ErrorEnum::eNoMemory; + + EXPECT_CALL(mImageManager, DownloadBlob(String(layer.mPath.c_str()), _)).WillOnce(Invoke(DownloadTestBlob)); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eNoMemory)) << tests::utils::ErrorToStr(err); + + // The sizing failure is handled as a failed download. + EXPECT_FALSE(FileExists(layer.mPath.c_str())); + EXPECT_EQ(mAcceptCount.load(), 0); + EXPECT_EQ(mReleaseCount.load(), 1); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eNoMemory)); +} + +TEST_F(CommonImageManagerInstallTest, InstallRemovePartialBlobFails) +{ + if (geteuid() == 0) { + GTEST_SKIP() << "Root ignores the folder permissions"; + } + + constexpr auto cContent = "partial"; + + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + const auto& path = manifest.mLayers[0].mPath; + const auto dir = fs::Dir(path.c_str()); + + ASSERT_TRUE(fs::RemoveAll(path.c_str()).IsNone()); + + // The partial file can't be removed from the read-only folder. + EXPECT_CALL(mImageManager, DownloadBlob(String(path.c_str()), _)) + .WillOnce(Invoke([&](const String& blobPath, const BlobInfo&) -> Error { + EXPECT_TRUE(fs::WriteStringToFile(blobPath, cContent, 0600).IsNone()); + EXPECT_EQ(chmod(dir.CStr(), 0500), 0); + + return ErrorEnum::eFailed; + })); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + + EXPECT_EQ(chmod(dir.CStr(), 0700), 0); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + // The partial file stays, so its space stays used and is freed once when the blob is processed again. + EXPECT_TRUE(FileExists(path.c_str())); + EXPECT_EQ(mLastResizeSize.load(), strlen(cContent)); + EXPECT_EQ(mAcceptCount.load(), 1); + EXPECT_EQ(mReleaseCount.load(), 0); +} + +TEST_F(CommonImageManagerInstallTest, InstallDownloadedBlobInvalid) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(manifest.mLayers[0].mPath.c_str()).IsNone()); + + EXPECT_CALL(mImageManager, DownloadBlob(String(manifest.mLayers[0].mPath.c_str()), _)) + .WillOnce( + Invoke([](const String& path, const BlobInfo&) { return fs::WriteStringToFile(path, cOtherHash, 0600); })); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eInvalidChecksum)) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eInvalidChecksum)); +} + +TEST_F(CommonImageManagerInstallTest, InstallCorruptManifest) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mOCISpec, LoadImageManifest(String(manifest.mManifest.mPath.c_str()), _)) + .WillOnce(Return(Error(ErrorEnum::eFailed, "corrupt manifest"))); + EXPECT_CALL(mFSPlatform, Sync(String(cTestImagePath), true)).Times(1); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + auto statuses = GetNotifiedStatuses(); + ASSERT_EQ(statuses.size(), 2); + EXPECT_EQ(statuses[0].mState, ItemStateEnum::eInstalling); + EXPECT_EQ(statuses[1].mState, ItemStateEnum::eFailed); + EXPECT_TRUE(statuses[1].mError.Is(ErrorEnum::eFailed)); + EXPECT_STREQ(statuses[1].mError.Message(), "corrupt manifest"); + + EXPECT_EQ(mStorage.GetBlobs(cTestItemID, cTestVersion), + std::vector({MakeBlobInfo(manifest.mManifest, oci::cMediaTypeImageManifest, false, "", 0)})); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eFailed)); + EXPECT_STREQ(item->mError.Message(), "corrupt manifest"); +} + +TEST_F(CommonImageManagerInstallTest, InstallAllocateSpaceFails) +{ + const auto manifest = AddManifest(1, 2, 2); + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(manifest.mLayers[1].mPath.c_str()).IsNone()); + + EXPECT_CALL(mSpaceAllocator, AllocateSpace(cInstallBlobSize)) + .WillOnce(Invoke([](size_t) -> RetWithError> { + return {nullptr, ErrorEnum::eNoMemory}; + })); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + EXPECT_CALL(mFSPlatform, Sync(String(cTestImagePath), true)).Times(1); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eNoMemory)) << tests::utils::ErrorToStr(err); + + auto statuses = GetNotifiedStatuses(); + ASSERT_EQ(statuses.size(), 2); + EXPECT_EQ(statuses[0].mState, ItemStateEnum::eInstalling); + EXPECT_EQ(statuses[1].mState, ItemStateEnum::eFailed); + EXPECT_TRUE(statuses[1].mError.Is(ErrorEnum::eNoMemory)); + + EXPECT_FALSE(FileExists(manifest.mLayers[1].mPath.c_str())); + + auto blobs = mStorage.GetBlobs(cTestItemID, cTestVersion); + ASSERT_EQ(blobs.size(), 4); + EXPECT_EQ(blobs.back(), MakeBlobInfo(manifest.mLayers[1], cTestLayerType, true, manifest.mDiffIDs[1].mDigest)); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eNoMemory)); +} + +TEST_F(CommonImageManagerInstallTest, InstallRemovedItemRestoresOutdatedItem) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + StoreItem(cTestItemID, cTestVersion, manifest.mManifest.mDigest.c_str(), ItemStateEnum::eRemoved); + + EXPECT_CALL(mSpaceAllocator, RestoreOutdatedItem(String(cTestItemID), String(cTestVersion))) + .WillOnce(Return(ErrorEnum::eNotFound)); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); +} + +TEST_F(CommonImageManagerInstallTest, InstallRemovedItemSetInstallingStateFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + StoreItem(cTestItemID, cTestVersion, manifest.mManifest.mDigest.c_str(), ItemStateEnum::eRemoved); + + mStorage.SetFailUpdateStates({ItemStateEnum::eInstalling}); + + // The item stays in the eRemoved state, so it stays outdated. + EXPECT_CALL(mSpaceAllocator, RestoreOutdatedItem(_, _)).Times(0); + EXPECT_CALL(mListener, OnItemStatusChanged(_)).Times(0); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); +} + +TEST_F(CommonImageManagerInstallTest, InstallRemovedItemRestoreFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + StoreItem(cTestItemID, cTestVersion, manifest.mManifest.mDigest.c_str(), ItemStateEnum::eRemoved); + + EXPECT_CALL(mSpaceAllocator, RestoreOutdatedItem(String(cTestItemID), String(cTestVersion))) + .WillOnce(Return(ErrorEnum::eFailed)); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eInstalled})); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); +} + +TEST_F(CommonImageManagerInstallTest, InstallStoredItemDoesNotRestoreOutdatedItem) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + StoreItem(cTestItemID, cTestVersion, manifest.mManifest.mDigest.c_str(), ItemStateEnum::eFailed); + + EXPECT_CALL(mSpaceAllocator, RestoreOutdatedItem(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); + EXPECT_TRUE(item->mError.IsNone()); +} + +TEST_F(CommonImageManagerInstallTest, InstallBlobInfoNotFound) +{ + const auto manifest = AddManifest(1, 1, 1); + + // The provider doesn't have the blob: it fails without retrying. + EXPECT_CALL(mBlobInfoProvider, GetBlobInfo(_, _)).WillOnce(Return(Error(ErrorEnum::eNotFound))); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eNotFound)) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eNotFound)); +} + +TEST_F(CommonImageManagerInstallTest, InstallBlobInfoRetried) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(manifest.mLayers[0].mPath.c_str()).IsNone()); + + // The other errors mean that the provider is not available for a while. + EXPECT_CALL(mBlobInfoProvider, GetBlobInfo(_, _)) + .WillOnce(Return(Error(ErrorEnum::eFailed))) + .WillOnce(Invoke(GetTestBlobInfo)); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerInstallTest, InstallDownloadFails) +{ + const auto manifest = AddManifest(1, 2, 2); + + CreateValidBlobs(manifest); + ASSERT_TRUE(fs::RemoveAll(manifest.mLayers[0].mPath.c_str()).IsNone()); + ASSERT_TRUE(fs::RemoveAll(manifest.mLayers[1].mPath.c_str()).IsNone()); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).WillRepeatedly(Invoke(DownloadTestBlob)); + EXPECT_CALL(mImageManager, DownloadBlob(String(manifest.mLayers[1].mPath.c_str()), _)) + .WillOnce(Invoke([](const String& path, const BlobInfo&) { + EXPECT_TRUE(fs::WriteStringToFile(path, "partial", 0600).IsNone()); + + return Error(ErrorEnum::eFailed, "download failed"); + })); + EXPECT_CALL(mFSPlatform, Sync(String(cTestImagePath), true)).Times(1); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + auto statuses = GetNotifiedStatuses(); + ASSERT_EQ(statuses.size(), 3); + EXPECT_EQ(statuses[0].mState, ItemStateEnum::eInstalling); + EXPECT_EQ(statuses[1].mState, ItemStateEnum::eDownloading); + EXPECT_EQ(statuses[2].mState, ItemStateEnum::eFailed); + EXPECT_TRUE(statuses[2].mError.Is(ErrorEnum::eFailed)); + EXPECT_STREQ(statuses[2].mError.Message(), "download failed"); + + EXPECT_TRUE(FileExists(manifest.mLayers[0].mPath.c_str())); + EXPECT_FALSE(FileExists(manifest.mLayers[1].mPath.c_str())); + EXPECT_EQ(mAcceptCount.load(), 1); + EXPECT_EQ(mReleaseCount.load(), 1); + + auto blobs = mStorage.GetBlobs(cTestItemID, cTestVersion); + ASSERT_EQ(blobs.size(), 4); + EXPECT_EQ(blobs.back(), MakeBlobInfo(manifest.mLayers[1], cTestLayerType, true, manifest.mDiffIDs[1].mDigest)); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eFailed)); + EXPECT_STREQ(item->mError.Message(), "download failed"); + + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eFailed); +} + +TEST_F(CommonImageManagerInstallTest, InstallMetadataDownloadFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + EXPECT_CALL(mImageManager, DownloadBlob(String(manifest.mManifest.mPath.c_str()), _)) + .WillOnce(Return(ErrorEnum::eFailed)); + EXPECT_CALL(mSpaceAllocator, AllocateSpace(cInstallBlobSize)).Times(1); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + EXPECT_FALSE(FileExists(manifest.mManifest.mPath.c_str())); + EXPECT_EQ(mReleaseCount.load(), 1); + EXPECT_EQ(mStorage.GetBlobs(cTestItemID, cTestVersion), + std::vector({MakeBlobInfo(manifest.mManifest, oci::cMediaTypeImageManifest, false, "", 0)})); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); +} + +TEST_F(CommonImageManagerInstallTest, InstallSyncFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + EXPECT_CALL(mFSPlatform, Sync(String(cTestImagePath), true)).WillOnce(Return(ErrorEnum::eFailed)); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eFailed})); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_TRUE(item->mError.Is(ErrorEnum::eFailed)); +} + +TEST_F(CommonImageManagerInstallTest, InstallSetInstalledStateFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + mStorage.SetFailUpdateStates({ItemStateEnum::eInstalled}); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eFailed})); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eFailed); + EXPECT_STREQ(item->mError.Message(), "update failed"); +} + +TEST_F(CommonImageManagerInstallTest, InstallSetFailedStateFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + // Both the eInstalled and the eFailed state updates fail. The blobs are downloaded, so eDownloading is notified. + mStorage.SetFailUpdateStates({ItemStateEnum::eInstalled, ItemStateEnum::eFailed}); + + std::optional reportedOnFailed; + + EXPECT_CALL(mListener, OnItemStatusChanged(_)).WillRepeatedly(Invoke([&](const UpdateItemStatus& status) { + { + std::lock_guard lock {mStatusesMutex}; + + mStatuses.push_back(status); + } + + if (status.mState == ItemStateEnum::eFailed) { + reportedOnFailed = GetReportedState(cTestItemID, cTestVersion); + } + })); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + auto statuses = GetNotifiedStatuses(); + ASSERT_EQ(statuses.size(), 3); + EXPECT_EQ(statuses[0].mState, ItemStateEnum::eInstalling); + EXPECT_EQ(statuses[1].mState, ItemStateEnum::eDownloading); + EXPECT_EQ(statuses[2].mState, ItemStateEnum::eFailed); + EXPECT_STREQ(statuses[2].mError.Message(), "update failed"); + + // While the final eFailed state is notified, the notified eFailed state is reported. + EXPECT_EQ(reportedOnFailed, ItemStateEnum::eFailed); + + // The item is left in eInstalling state, and it is reported as eFailed as it is not being installed. + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalling); + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eFailed); +} + +TEST_F(CommonImageManagerInstallTest, InstallAfterSetFailedStateFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + mStorage.SetFailUpdateStates({ItemStateEnum::eInstalled, ItemStateEnum::eFailed}); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + mStorage.SetFailUpdateStates({}); + + err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + // The stored state is already eInstalling, but the listeners were notified of eFailed, so the retry is notified. + EXPECT_EQ(GetNotifiedStates(), + std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eDownloading, ItemStateEnum::eFailed, + ItemStateEnum::eInstalling, ItemStateEnum::eInstalled})); +} + +TEST_F(CommonImageManagerInstallTest, GetItemsStatusesDuringInstall) +{ + const auto manifest = AddManifest(1, 1, 1); + + std::promise downloadStarted; + std::promise downloadReleased; + auto downloadReleasedFuture = downloadReleased.get_future(); + std::atomic installingChecked {false}; + + EXPECT_CALL(mListener, OnItemStatusChanged(_)).WillRepeatedly(Invoke([&](const UpdateItemStatus& status) { + if (status.mState == ItemStateEnum::eInstalling) { + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eInstalling); + + installingChecked = true; + } + })); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)) + .WillOnce(Invoke([&](const String& path, const BlobInfo& blobInfo) { + downloadStarted.set_value(); + downloadReleasedFuture.wait(); + + return DownloadTestBlob(path, blobInfo); + })) + .WillRepeatedly(Invoke(DownloadTestBlob)); + + Error installErr; + + std::thread installThread([&]() { installErr = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); }); + + downloadStarted.get_future().wait(); + + EXPECT_TRUE(installingChecked.load()); + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eDownloading); + + downloadReleased.set_value(); + installThread.join(); + + ASSERT_TRUE(installErr.IsNone()) << tests::utils::ErrorToStr(installErr); + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eInstalled); +} + +TEST_F(CommonImageManagerInstallTest, GetItemsStatusesDuringStoredItemInstall) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + + auto storedItem = CreateDeployableItem(cTestItemID, cTestVersion, ItemStateEnum::eFailed); + + storedItem.mType = UpdateItemTypeEnum::eComponent; + storedItem.mRootDigest = manifest.mManifest.mDigest.c_str(); + + ASSERT_TRUE(mStorage.AddDeployableItem(storedItem).IsNone()); + + std::optional reportedOnInstalling; + + EXPECT_CALL(mListener, OnItemStatusChanged(_)).WillRepeatedly(Invoke([&](const UpdateItemStatus& status) { + EXPECT_EQ(status.mType, UpdateItemTypeEnum::eComponent); + + if (status.mState != ItemStateEnum::eInstalling) { + return; + } + + auto statuses = std::make_unique>(); + + EXPECT_TRUE(mImageManager.GetItemsStatuses(*statuses).IsNone()); + + if (const auto* reported = FindStatus(*statuses, cTestItemID, cTestVersion); reported != nullptr) { + reportedOnInstalling = *reported; + } + })); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest, false, UpdateItemTypeEnum::eComponent)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + ASSERT_TRUE(reportedOnInstalling.has_value()); + EXPECT_EQ(reportedOnInstalling->mState, ItemStateEnum::eInstalling); + EXPECT_EQ(reportedOnInstalling->mType, UpdateItemTypeEnum::eComponent); +} + +TEST_F(CommonImageManagerInstallTest, InstallSameVersionConcurrently) +{ + const auto manifest = AddManifest(1, 1, 1); + + std::promise downloadStarted; + std::promise downloadReleased; + auto downloadReleasedFuture = downloadReleased.get_future(); + std::atomic downloadCount {0}; + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)) + .WillRepeatedly(Invoke([&](const String& path, const BlobInfo& blobInfo) { + if (downloadCount++ == 0) { + downloadStarted.set_value(); + downloadReleasedFuture.wait(); + } + + return DownloadTestBlob(path, blobInfo); + })); + + Error firstErr, secondErr; + + std::thread firstThread([&]() { firstErr = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); }); + + downloadStarted.get_future().wait(); + + std::thread secondThread([&]() { secondErr = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); }); + + // Gives the second install time to reach LockItem: without waiting there, it would notify eInstalling and fail + // the checks below. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + EXPECT_EQ( + GetNotifiedStates(), std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eDownloading})); + EXPECT_EQ(downloadCount.load(), 1); + + downloadReleased.set_value(); + + firstThread.join(); + secondThread.join(); + + ASSERT_TRUE(firstErr.IsNone()) << tests::utils::ErrorToStr(firstErr); + ASSERT_TRUE(secondErr.IsNone()) << tests::utils::ErrorToStr(secondErr); + + // The second install starts after the first one is finished, including its final notification, and finds all + // blobs valid. + EXPECT_EQ(GetNotifiedStates(), + std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eDownloading, ItemStateEnum::eInstalled, + ItemStateEnum::eInstalling, ItemStateEnum::eInstalled})); + EXPECT_EQ(downloadCount.load(), 3); +} + +TEST_F(CommonImageManagerInstallTest, InstallSharedBlobConcurrently) +{ + const auto sharedLayer = MakeTestBlob(500); + const auto manifestA = AddManifest(1, 0, 1, false, {sharedLayer}); + const auto manifestB = AddManifest(200, 0, 1, false, {sharedLayer}); + + CreateValidBlobs(manifestA); + CreateValidBlobs(manifestB); + ASSERT_TRUE(fs::RemoveAll(sharedLayer.mPath.c_str()).IsNone()); + + std::promise downloadStarted; + std::promise downloadReleased; + auto downloadReleasedFuture = downloadReleased.get_future(); + std::atomic sharedDownloadCount {0}; + + const auto countSharedLayerCalls = [&sharedLayer](const std::vector& calls) { + return std::count_if(calls.begin(), calls.end(), + [&sharedLayer](const TestImageManager::LayerCall& call) { return call.mDigest == sharedLayer.mDigest; }); + }; + + EXPECT_CALL(mImageManager, DownloadBlob(String(sharedLayer.mPath.c_str()), _)) + .WillOnce(Invoke([&](const String& path, const BlobInfo& blobInfo) { + sharedDownloadCount++; + downloadStarted.set_value(); + downloadReleasedFuture.wait(); + + return DownloadTestBlob(path, blobInfo); + })); + + Error errA, errB; + + std::thread threadA([&]() { + errA + = mImageManager.Install(MakeInstallInfo(manifestA.mManifest, false, UpdateItemTypeEnum::eService, "itemA")); + }); + + downloadStarted.get_future().wait(); + + std::thread threadB([&]() { + errB + = mImageManager.Install(MakeInstallInfo(manifestB.mManifest, false, UpdateItemTypeEnum::eService, "itemB")); + }); + + // Gives item B time to reach LockBlob of the shared layer: without waiting there, it would validate and download + // the shared layer a second time and fail the checks below. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + EXPECT_EQ(sharedDownloadCount.load(), 1); + + downloadReleased.set_value(); + + threadA.join(); + threadB.join(); + + ASSERT_TRUE(errA.IsNone()) << tests::utils::ErrorToStr(errA); + ASSERT_TRUE(errB.IsNone()) << tests::utils::ErrorToStr(errB); + + // Item B finds the shared layer downloaded by item A valid. + EXPECT_EQ(countSharedLayerCalls(mImageManager.GetValidateLayerCalls()), 2); + EXPECT_EQ(sharedDownloadCount.load(), 1); + + for (const auto& itemID : {"itemA", "itemB"}) { + auto blobs = mStorage.GetBlobs(itemID, cTestVersion); + ASSERT_EQ(blobs.size(), 3); + EXPECT_EQ(blobs.back().mDigest, sharedLayer.mDigest.c_str()); + + auto item = GetStoredItem(itemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); + } + + EXPECT_EQ(ReadFile(sharedLayer.mPath), sharedLayer.mHash); +} + } // namespace aos::common::imagemanager diff --git a/src/core/common/imagemanager/tests/stubs/storagestub.hpp b/src/core/common/imagemanager/tests/stubs/storagestub.hpp index e24932f77..61debabb9 100644 --- a/src/core/common/imagemanager/tests/stubs/storagestub.hpp +++ b/src/core/common/imagemanager/tests/stubs/storagestub.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -37,6 +38,10 @@ class StorageStub : public StorageItf { { std::lock_guard lock {mMutex}; + if (mFailUpdateStates.count(item.mState.GetValue()) != 0) { + return Error(ErrorEnum::eFailed, "update failed"); + } + auto it = Find(item.mItemID, item.mVersion); if (it == mItems.end()) { return ErrorEnum::eNotFound; @@ -153,6 +158,25 @@ class StorageStub : public StorageItf { return items; } + std::vector GetBlobs(const String& itemID, const String& version) + { + std::lock_guard lock {mMutex}; + + auto it = Find(itemID, version); + if (it == mItems.end()) { + return {}; + } + + return it->mBlobs; + } + + void SetFailUpdateStates(const std::set& states) + { + std::lock_guard lock {mMutex}; + + mFailUpdateStates = states; + } + private: struct Item { DeployableItemInfo mInfo; @@ -167,6 +191,8 @@ class StorageStub : public StorageItf { std::mutex mMutex; std::vector mItems; + // Item states which UpdateDeployableItem fails to set. + std::set mFailUpdateStates; }; } // namespace aos::common::imagemanager diff --git a/src/core/common/ocispec/itf/imagespec.hpp b/src/core/common/ocispec/itf/imagespec.hpp index 536652cf7..23c6b900b 100644 --- a/src/core/common/ocispec/itf/imagespec.hpp +++ b/src/core/common/ocispec/itf/imagespec.hpp @@ -52,6 +52,12 @@ constexpr auto cMaxNumLayers = AOS_CONFIG_OCISPEC_MAX_NUM_LAYERS; */ constexpr auto cRootfsTypeLen = AOS_CONFIG_OCISPEC_ROOTFS_TYPE_LEN; +/** + * OCI image media type constants. + */ +constexpr auto cMediaTypeImageIndex = "application/vnd.oci.image.index.v1+json"; +constexpr auto cMediaTypeImageManifest = "application/vnd.oci.image.manifest.v1+json"; + /** * OCI layer media type constants. */ From 84dd9c4694d170a4bddfaad2108e1b4394e9541a Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Fri, 30 Oct 2026 10:33:29 +0200 Subject: [PATCH 10/54] common: imagemanager: implement RemoveItem Implement removing of a single deployable item version in the common image manager. The item version is locked for the removal, so RemoveItem and InstallItem of the same version are mutually exclusive. RemoveItem fails with eNotFound if the item version is not stored and does nothing if the item is already in the eRemoved state. Items in any other state are removed the same way. The eRemoving state is notified and reported by GetItemsStatuses while the item is being removed. Then the item is set to eRemoved with the current timestamp in the storage, keeping its blobs and blob references, and eRemoved is notified. If the storage update fails, the previous item state is notified again with the error. Finally, the item is added as outdated to the space allocator without the image manager mutex locked. SetInProgressItemState takes the item ID, version and type instead of InstallItemInfo, so it is used by both install and remove. Add unit tests for RemoveItem. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko --- src/core/common/imagemanager/imagemanager.cpp | 74 +++++- src/core/common/imagemanager/imagemanager.hpp | 13 +- .../imagemanager/tests/imagemanager.cpp | 212 ++++++++++++++++++ 3 files changed, 287 insertions(+), 12 deletions(-) diff --git a/src/core/common/imagemanager/imagemanager.cpp b/src/core/common/imagemanager/imagemanager.cpp index bc7690445..744022f15 100644 --- a/src/core/common/imagemanager/imagemanager.cpp +++ b/src/core/common/imagemanager/imagemanager.cpp @@ -252,7 +252,9 @@ Error ImageManager::InstallItem(const InstallItemInfo& itemInfo) if (auto setStateErr = SetStoredItemState(storedInfo, ItemStateEnum::eFailed, err); !setStateErr.IsNone()) { LOG_ERR() << "Failed to set stored item state" << Log::Field(setStateErr); - if (setStateErr = SetInProgressItemState(itemInfo, ItemStateEnum::eFailed, err); !setStateErr.IsNone()) { + if (setStateErr = SetInProgressItemState( + itemInfo.mItemID, itemInfo.mVersion, itemInfo.mType, ItemStateEnum::eFailed, err); + !setStateErr.IsNone()) { LOG_ERR() << "Failed to set in-progress item state" << Log::Field(setStateErr); } } @@ -263,9 +265,43 @@ Error ImageManager::InstallItem(const InstallItemInfo& itemInfo) Error ImageManager::RemoveItem(const String& itemID, const String& version) { - LOG_DBG() << "Remove item" << Log::Field("itemID", itemID) << Log::Field("version", version); + LOG_INF() << "Remove item" << Log::Field("itemID", itemID) << Log::Field("version", version); - return ErrorEnum::eNotSupported; + if (auto err = LockItem(itemID, version); !err.IsNone()) { + return err; + } + + auto unlockItem = DeferRelease([this, &itemID, &version]() { UnlockItem(itemID, version); }); + + DeployableItemInfo storedInfo; + + if (auto err = mStorage->GetDeployableItem(itemID, version, storedInfo); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (storedInfo.mState == ItemStateEnum::eRemoved) { + LOG_DBG() << "Item already removed" << Log::Field("itemID", itemID) << Log::Field("version", version); + + return ErrorEnum::eNone; + } + + const UpdateItemStatus prevStatus = storedInfo; // NOSONAR cpp:S5912 - intentional copy of the base part only + + if (auto err = SetInProgressItemState(itemID, version, storedInfo.mType, ItemStateEnum::eRemoving); !err.IsNone()) { + LOG_ERR() << "Failed to set in-progress item state" << Log::Field(err); + } + + if (auto err = SetStoredItemState(storedInfo, ItemStateEnum::eRemoved); !err.IsNone()) { + RestoreRemovingItemState(prevStatus, err); + + return err; + } + + if (auto err = mSpaceAllocator->AddOutdatedItem(itemID, version, storedInfo.mTimestamp); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; } Error ImageManager::Cancel() @@ -392,7 +428,8 @@ Error ImageManager::DownloadItemBlob(const InstallItemInfo& itemInfo, const Stri return AOS_ERROR_WRAP(err); } - if (err = SetInProgressItemState(itemInfo, ItemStateEnum::eDownloading); !err.IsNone()) { + if (err = SetInProgressItemState(itemInfo.mItemID, itemInfo.mVersion, itemInfo.mType, ItemStateEnum::eDownloading); + !err.IsNone()) { LOG_ERR() << "Failed to set in-progress item state" << Log::Field(err); } @@ -749,7 +786,8 @@ Error ImageManager::StartItemInstall(const InstallItemInfo& itemInfo, Deployable return AOS_ERROR_WRAP(err); } - if (err = SetInProgressItemState(itemInfo, ItemStateEnum::eInstalling); !err.IsNone()) { + if (err = SetInProgressItemState(itemInfo.mItemID, itemInfo.mVersion, itemInfo.mType, ItemStateEnum::eInstalling); + !err.IsNone()) { LOG_ERR() << "Failed to set in-progress item state" << Log::Field(err); } @@ -868,7 +906,8 @@ Error ImageManager::AddItemBlob(const String& itemID, const String& version, con return ErrorEnum::eNone; } -Error ImageManager::SetInProgressItemState(const InstallItemInfo& itemInfo, ItemStateEnum state, const Error& error) +Error ImageManager::SetInProgressItemState( + const String& itemID, const String& version, const UpdateItemType& type, ItemStateEnum state, const Error& error) { UpdateItemStatus status; bool stateChanged = false; @@ -876,14 +915,14 @@ Error ImageManager::SetInProgressItemState(const InstallItemInfo& itemInfo, Item { LockGuard lock {mMutex}; - auto itemInProgress = FindInProgressItem(itemInfo.mItemID, itemInfo.mVersion); + auto itemInProgress = FindInProgressItem(itemID, version); if (!itemInProgress) { return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "item not in progress")); } stateChanged = itemInProgress->mState != state; - itemInProgress->mType = itemInfo.mType; + itemInProgress->mType = type; itemInProgress->mState = state; itemInProgress->mError = error; @@ -897,6 +936,25 @@ Error ImageManager::SetInProgressItemState(const InstallItemInfo& itemInfo, Item return ErrorEnum::eNone; } +void ImageManager::RestoreRemovingItemState(const UpdateItemStatus& prevStatus, const Error& error) +{ + { + LockGuard lock {mMutex}; + + // The stored state is reported again once the item is not being removed. + if (auto itemInProgress = FindInProgressItem(prevStatus.mItemID, prevStatus.mVersion); itemInProgress) { + itemInProgress->mState = ItemStateEnum::eUnknown; + itemInProgress->mError = ErrorEnum::eNone; + } + } + + auto status = prevStatus; + + status.mError = error; + + NotifyItemStatusChanged(status); +} + Error ImageManager::SetStoredItemState(DeployableItemInfo& item, ItemStateEnum state, const Error& error) { // An item left in eInstalling is reported as eFailed, so its next install has to be notified. diff --git a/src/core/common/imagemanager/imagemanager.hpp b/src/core/common/imagemanager/imagemanager.hpp index 1485632dd..e9e57294f 100644 --- a/src/core/common/imagemanager/imagemanager.hpp +++ b/src/core/common/imagemanager/imagemanager.hpp @@ -154,11 +154,15 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI /** * Removes item. Sets the item state to eRemoved in the storage, keeping the item blobs, and adds the item as - * outdated to the space allocator. + * outdated to the space allocator. Waits while the same item version is being installed or removed, or while + * cMaxNumConcurrentItems items are being processed. Does nothing if the item is already in the eRemoved state. + * + * @note The eRemoved status is notified while the item is still in progress, so a listener must not call + * InstallItem or RemoveItem synchronously from the callback. * * @param itemID item ID. * @param version item version. - * @return Error. + * @return Error: eNotFound if the item version is not stored, other errors as is. */ Error RemoveItem(const String& itemID, const String& version); @@ -299,8 +303,9 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI Error AddItemBlob(const String& itemID, const String& version, const oci::ContentDescriptor& descriptor, bool isLayer = false, const String& diffID = ""); - Error SetInProgressItemState( - const InstallItemInfo& itemInfo, ItemStateEnum state, const Error& error = ErrorEnum::eNone); + Error SetInProgressItemState(const String& itemID, const String& version, const UpdateItemType& type, + ItemStateEnum state, const Error& error = ErrorEnum::eNone); + void RestoreRemovingItemState(const UpdateItemStatus& prevStatus, const Error& error); Error SetStoredItemState(DeployableItemInfo& item, ItemStateEnum state, const Error& error = ErrorEnum::eNone); void NotifyItemStatusChanged(const UpdateItemStatus& status); diff --git a/src/core/common/imagemanager/tests/imagemanager.cpp b/src/core/common/imagemanager/tests/imagemanager.cpp index 41ed29d6b..0db6a9b3a 100644 --- a/src/core/common/imagemanager/tests/imagemanager.cpp +++ b/src/core/common/imagemanager/tests/imagemanager.cpp @@ -2022,4 +2022,216 @@ TEST_F(CommonImageManagerInstallTest, InstallSharedBlobConcurrently) EXPECT_EQ(ReadFile(sharedLayer.mPath), sharedLayer.mHash); } +/*********************************************************************************************************************** + * Remove item suite + **********************************************************************************************************************/ + +class CommonImageManagerRemoveTest : public CommonImageManagerInstallTest { }; + +TEST_F(CommonImageManagerRemoveTest, RemoveInstalledItem) +{ + const auto manifest = AddManifest(1, 1, 1); + + CreateValidBlobs(manifest); + StoreItem(cTestItemID, cTestVersion, manifest.mManifest.mDigest.c_str(), ItemStateEnum::eInstalled); + + const auto blobInfo = MakeBlobInfo(manifest.mManifest, oci::cMediaTypeImageManifest); + + ASSERT_TRUE(mStorage.AddItemBlob(cTestItemID, cTestVersion, blobInfo).IsNone()); + + Time outdatedTimestamp; + + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String(cTestItemID), String(cTestVersion), _)) + .WillOnce(Invoke([&](const String&, const String&, const Time& timestamp) { + outdatedTimestamp = timestamp; + + return ErrorEnum::eNone; + })); + EXPECT_CALL(mImageManager, RemoveLayer(_, _, _)).Times(0); + EXPECT_CALL(mSpaceAllocator, FreeSpace(_)).Times(0); + + auto err = mImageManager.Remove(cTestItemID, cTestVersion); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), std::vector({ItemStateEnum::eRemoving, ItemStateEnum::eRemoved})); + + for (const auto& status : GetNotifiedStatuses()) { + EXPECT_EQ(status.mType, UpdateItemTypeEnum::eService); + EXPECT_TRUE(status.mError.IsNone()); + } + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); + EXPECT_EQ(item->mTimestamp, outdatedTimestamp); + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eRemoved); + + // The item blobs and blob references are kept in the cache. + EXPECT_EQ(mStorage.GetBlobs(cTestItemID, cTestVersion), std::vector({blobInfo})); + EXPECT_TRUE(FileExists(manifest.mManifest.mPath.c_str())); +} + +TEST_F(CommonImageManagerRemoveTest, RemoveFailedAndInstallingItems) +{ + for (const auto state : {ItemStateEnum::eFailed, ItemStateEnum::eInstalling}) { + const auto version = std::string("1.0.") + std::to_string(static_cast(state)); + + AddStoredItem(cTestItemID, version.c_str(), state, Error(ErrorEnum::eFailed, "install failed")); + + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String(cTestItemID), String(version.c_str()), _)) + .WillOnce(Return(ErrorEnum::eNone)); + + auto err = mImageManager.Remove(cTestItemID, version.c_str()); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, version.c_str()); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); + EXPECT_TRUE(item->mError.IsNone()); + } + + EXPECT_EQ(GetNotifiedStates(), + std::vector( + {ItemStateEnum::eRemoving, ItemStateEnum::eRemoved, ItemStateEnum::eRemoving, ItemStateEnum::eRemoved})); +} + +TEST_F(CommonImageManagerRemoveTest, RemoveNotStoredItem) +{ + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(_, _, _)).Times(0); + EXPECT_CALL(mListener, OnItemStatusChanged(_)).Times(0); + + auto err = mImageManager.Remove(cTestItemID, cTestVersion); + EXPECT_TRUE(err.Is(ErrorEnum::eNotFound)) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerRemoveTest, RemoveRemovedItem) +{ + AddStoredItem(cTestItemID, cTestVersion, ItemStateEnum::eRemoved); + + const auto storedItem = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(storedItem.has_value()); + + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(_, _, _)).Times(0); + EXPECT_CALL(mListener, OnItemStatusChanged(_)).Times(0); + + auto err = mImageManager.Remove(cTestItemID, cTestVersion); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetStoredItem(cTestItemID, cTestVersion), storedItem); +} + +TEST_F(CommonImageManagerRemoveTest, RemoveSetRemovedStateFails) +{ + AddStoredItem(cTestItemID, cTestVersion, ItemStateEnum::eInstalled); + + mStorage.SetFailUpdateStates({ItemStateEnum::eRemoved}); + + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(_, _, _)).Times(0); + + auto err = mImageManager.Remove(cTestItemID, cTestVersion); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + const auto statuses = GetNotifiedStatuses(); + + ASSERT_EQ(statuses.size(), 2); + EXPECT_EQ(statuses[0].mState, ItemStateEnum::eRemoving); + EXPECT_TRUE(statuses[0].mError.IsNone()); + EXPECT_EQ(statuses[1].mState, ItemStateEnum::eInstalled); + EXPECT_EQ(statuses[1].mType, UpdateItemTypeEnum::eService); + EXPECT_TRUE(statuses[1].mError.Is(ErrorEnum::eFailed)); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eInstalled); +} + +TEST_F(CommonImageManagerRemoveTest, RemoveAddOutdatedItemFails) +{ + AddStoredItem(cTestItemID, cTestVersion, ItemStateEnum::eInstalled); + + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String(cTestItemID), String(cTestVersion), _)) + .WillOnce(Return(ErrorEnum::eFailed)); + + auto err = mImageManager.Remove(cTestItemID, cTestVersion); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(GetNotifiedStates(), std::vector({ItemStateEnum::eRemoving, ItemStateEnum::eRemoved})); + + // The item stays in the eRemoved state and is registered as outdated on the next start. + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); +} + +TEST_F(CommonImageManagerRemoveTest, GetItemsStatusesDuringRemove) +{ + AddStoredItem(cTestItemID, cTestVersion, ItemStateEnum::eInstalled); + + std::optional reportedOnRemoving; + + EXPECT_CALL(mListener, OnItemStatusChanged(_)).WillRepeatedly(Invoke([&](const UpdateItemStatus& status) { + if (status.mState == ItemStateEnum::eRemoving) { + reportedOnRemoving = GetReportedState(cTestItemID, cTestVersion); + } + })); + + auto err = mImageManager.Remove(cTestItemID, cTestVersion); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(reportedOnRemoving, ItemStateEnum::eRemoving); + EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eRemoved); +} + +TEST_F(CommonImageManagerRemoveTest, RemoveWaitsForInstall) +{ + const auto manifest = AddManifest(1, 1, 1); + + std::promise downloadStarted; + std::promise downloadReleased; + auto downloadReleasedFuture = downloadReleased.get_future(); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)) + .WillOnce(Invoke([&](const String& path, const BlobInfo& blobInfo) { + downloadStarted.set_value(); + downloadReleasedFuture.wait(); + + return DownloadTestBlob(path, blobInfo); + })) + .WillRepeatedly(Invoke(DownloadTestBlob)); + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String(cTestItemID), String(cTestVersion), _)) + .WillOnce(Return(ErrorEnum::eNone)); + + Error installErr, removeErr; + + std::thread installThread([&]() { installErr = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); }); + + downloadStarted.get_future().wait(); + + std::thread removeThread([&]() { removeErr = mImageManager.Remove(cTestItemID, cTestVersion); }); + + // Gives the remove time to reach LockItem: without waiting there, it would notify eRemoving and fail the check + // below. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + EXPECT_EQ( + GetNotifiedStates(), std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eDownloading})); + + downloadReleased.set_value(); + + installThread.join(); + removeThread.join(); + + ASSERT_TRUE(installErr.IsNone()) << tests::utils::ErrorToStr(installErr); + ASSERT_TRUE(removeErr.IsNone()) << tests::utils::ErrorToStr(removeErr); + + EXPECT_EQ(GetNotifiedStates(), + std::vector({ItemStateEnum::eInstalling, ItemStateEnum::eDownloading, ItemStateEnum::eInstalled, + ItemStateEnum::eRemoving, ItemStateEnum::eRemoved})); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); +} + } // namespace aos::common::imagemanager From 16c89cff30f1259b2e1866f935c5e6cbc4ab2683 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Fri, 30 Oct 2026 11:21:31 +0200 Subject: [PATCH 11/54] common: imagemanager: handle outdated items Register outdated items on start and implement the item remover used by the space allocator to remove them. On start, the items whose install was interrupted, i.e. left in the eInstalling state, are set to the eRemoved state, keeping their blobs. Then all the items in the eRemoved state are added as outdated to the space allocator. A failure to add an outdated item is only logged. No item status is notified on start. The item remover skips the items being installed or removed and the items which are not in the eRemoved state anymore. It is called with the space allocator lock held, so it never waits for an item or a blob: it tries to lock them and skips them if they can't be locked. One extra slot is reserved in the lists of the items and blobs in progress for it, so it can lock them while all the other slots are taken. A full list is logged as an error. The item and its blob references are removed from the storage first. Then each item blob which is not locked and not used by other items is removed, together with its unpacked layer data for the layers. The freed size is returned to the space allocator and OnItemRemovedFromCache is notified. A blob which fails to be removed is logged and left as orphan. Add unit tests for start and the item remover. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko --- src/core/common/imagemanager/imagemanager.cpp | 221 +++++++++++- src/core/common/imagemanager/imagemanager.hpp | 20 +- .../imagemanager/tests/imagemanager.cpp | 319 ++++++++++++++++++ .../imagemanager/tests/stubs/storagestub.hpp | 13 + 4 files changed, 563 insertions(+), 10 deletions(-) diff --git a/src/core/common/imagemanager/imagemanager.cpp b/src/core/common/imagemanager/imagemanager.cpp index 744022f15..009630c7b 100644 --- a/src/core/common/imagemanager/imagemanager.cpp +++ b/src/core/common/imagemanager/imagemanager.cpp @@ -86,7 +86,33 @@ Error ImageManager::Start() { LOG_DBG() << "Start image manager"; - return ErrorEnum::eNotSupported; + auto items = MakeUnique>(mAllocator); + if (!items) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = mStorage->GetAllDeployableItems(*items); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + for (auto& item : *items) { + if (item.mState == ItemStateEnum::eInstalling) { + if (auto err = RemoveInterruptedItem(item); !err.IsNone()) { + return err; + } + } + + if (item.mState != ItemStateEnum::eRemoved) { + continue; + } + + if (auto err = mSpaceAllocator->AddOutdatedItem(item.mItemID, item.mVersion, item.mTimestamp); !err.IsNone()) { + LOG_ERR() << "Failed to add outdated item" << Log::Field("itemID", item.mItemID) + << Log::Field("version", item.mVersion) << Log::Field(err); + } + } + + return ErrorEnum::eNone; } Error ImageManager::Stop() @@ -297,6 +323,9 @@ Error ImageManager::RemoveItem(const String& itemID, const String& version) return err; } + // The space allocator can remove the outdated item right away, which skips the locked item. + unlockItem.Reset(); + if (auto err = mSpaceAllocator->AddOutdatedItem(itemID, version, storedInfo.mTimestamp); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -328,6 +357,17 @@ void ImageManager::NotifyItemStatusChanged(const UpdateItemStatus& status) } } +void ImageManager::NotifyItemRemovedFromCache(const String& itemID, const String& version) +{ + LockGuard lock {mListenersMutex}; + + LOG_DBG() << "Item removed from cache" << Log::Field("itemID", itemID) << Log::Field("version", version); + + for (auto* listener : mListeners) { + listener->OnItemRemovedFromCache(itemID, version); + } +} + Error ImageManager::ValidateBlob(const String& path, const String& digest, size_t size) { LOG_DBG() << "Validate blob" << Log::Field("digest", digest) << Log::Field("path", path); @@ -672,15 +712,120 @@ RetWithError ImageManager::RemoveOutdatedItem(const String& itemID, cons { LOG_DBG() << "Remove outdated item" << Log::Field("itemID", itemID) << Log::Field("version", version); - return {0, ErrorEnum::eNotSupported}; + // The item remover is called with the space allocator lock held, so it never waits for the item or its blobs: the + // waiting thread may hold them. The item which can't be locked is skipped. + if (!TryLockItem(itemID, version)) { + LOG_DBG() << "Can't lock outdated item" << Log::Field("itemID", itemID) << Log::Field("version", version); + + return {0, ErrorEnum::eNone}; + } + + auto unlockItem = DeferRelease([this, &itemID, &version]() { UnlockItem(itemID, version); }); + + DeployableItemInfo item; + + auto err = mStorage->GetDeployableItem(itemID, version, item); + if (!err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { + return {0, AOS_ERROR_WRAP(err)}; + } + + // The item could be removed from the cache or installed again after it was added as outdated. + if (err.Is(ErrorEnum::eNotFound) || item.mState != ItemStateEnum::eRemoved) { + LOG_DBG() << "Outdated item is not removed" << Log::Field("itemID", itemID) << Log::Field("version", version); + + return {0, ErrorEnum::eNone}; + } + + auto blobs = MakeUnique>(mAllocator); + if (!blobs) { + return {0, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } + + if (err = mStorage->GetItemBlobs(itemID, version, *blobs); !err.IsNone()) { + return {0, AOS_ERROR_WRAP(err)}; + } + + if (err = mStorage->RemoveDeployableItem(itemID, version); !err.IsNone()) { + return {0, AOS_ERROR_WRAP(err)}; + } + + size_t freedSize = 0; + + for (const auto& blob : *blobs) { + size_t blobSize = 0; + + // The blob which is failed to remove becomes orphan and is removed by the orphan blobs cleanup. + if (Tie(blobSize, err) = RemoveUnusedItemBlob(blob); !err.IsNone()) { + LOG_ERR() << "Failed to remove item blob" << Log::Field("digest", blob.mDigest) << Log::Field(err); + } + + freedSize += blobSize; + } + + LOG_INF() << "Outdated item removed" << Log::Field("itemID", itemID) << Log::Field("version", version) + << Log::Field("freedSize", freedSize); + + NotifyItemRemovedFromCache(itemID, version); + + return {freedSize, ErrorEnum::eNone}; +} + +RetWithError ImageManager::RemoveUnusedItemBlob(const ItemBlobInfo& blob) +{ + // A locked blob is being processed by an item being installed, so it is in use. The blob which can't be locked is + // kept and becomes orphan if it is not used. + if (!TryLockBlob(blob.mDigest)) { + return {0, ErrorEnum::eNone}; + } + + auto unlockBlob = DeferRelease([this, &blob]() { UnlockBlob(blob.mDigest); }); + + auto [used, err] = mStorage->IsBlobUsed(blob.mDigest); + if (!err.IsNone()) { + return {0, AOS_ERROR_WRAP(err)}; + } + + if (used) { + return {0, ErrorEnum::eNone}; + } + + StaticString path; + + if (err = GetBlobPath(blob.mDigest, path); !err.IsNone()) { + return {0, err}; + } + + size_t layerSize = 0; + + if (blob.mIsLayer) { + if (Tie(layerSize, err) = RemoveLayer(path, blob, blob.mDiffID); + !err.IsNone() && !err.Is(ErrorEnum::eNotSupported)) { + LOG_ERR() << "Failed to remove layer" << Log::Field("digest", blob.mDigest) << Log::Field(err); + } + + if (!err.IsNone()) { + layerSize = 0; + } + } + + size_t blobSize = 0; + + if (Tie(blobSize, err) = RemoveBlob(path, blob.mDigest); !err.IsNone()) { + return {layerSize, err}; + } + + return {layerSize + blobSize, ErrorEnum::eNone}; } Error ImageManager::LockItem(const String& itemID, const String& version) { UniqueLock lock {mMutex}; - if (auto err = mCondVar.Wait( - lock, [&]() { return FindInProgressItem(itemID, version) == nullptr && !mItemsInProgress.IsFull(); }); + if (auto err = mCondVar.Wait(lock, + [&]() { + return FindInProgressItem(itemID, version) == nullptr + && mItemsInProgress.Size() < cMaxNumConcurrentItems; + }); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -697,6 +842,32 @@ Error ImageManager::LockItem(const String& itemID, const String& version) return ErrorEnum::eNone; } +bool ImageManager::TryLockItem(const String& itemID, const String& version) +{ + LockGuard lock {mMutex}; + + if (FindInProgressItem(itemID, version) != nullptr) { + return false; + } + + if (mItemsInProgress.IsFull()) { + LOG_ERR() << "Items in progress list is full" << Log::Field("itemID", itemID) << Log::Field("version", version); + + return false; + } + + UpdateItemStatus item; + + item.mItemID = itemID; + item.mVersion = version; + + if (auto err = mItemsInProgress.PushBack(item); !err.IsNone()) { + return false; + } + + return true; +} + void ImageManager::UnlockItem(const String& itemID, const String& version) { LockGuard lock {mMutex}; @@ -721,7 +892,8 @@ Error ImageManager::LockBlob(const String& digest) { UniqueLock lock {mMutex}; - if (auto err = mCondVar.Wait(lock, [&]() { return !IsBlobInProgress(digest) && !mBlobsInProgress.IsFull(); }); + if (auto err = mCondVar.Wait( + lock, [&]() { return !IsBlobInProgress(digest) && mBlobsInProgress.Size() < cMaxNumConcurrentItems; }); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -733,6 +905,27 @@ Error ImageManager::LockBlob(const String& digest) return ErrorEnum::eNone; } +bool ImageManager::TryLockBlob(const String& digest) +{ + LockGuard lock {mMutex}; + + if (IsBlobInProgress(digest)) { + return false; + } + + if (mBlobsInProgress.IsFull()) { + LOG_ERR() << "Blobs in progress list is full" << Log::Field("digest", digest); + + return false; + } + + if (auto err = mBlobsInProgress.PushBack(digest); !err.IsNone()) { + return false; + } + + return true; +} + void ImageManager::UnlockBlob(const String& digest) { LockGuard lock {mMutex}; @@ -936,6 +1129,24 @@ Error ImageManager::SetInProgressItemState( return ErrorEnum::eNone; } +Error ImageManager::RemoveInterruptedItem(DeployableItemInfo& item) +{ + LOG_WRN() << "Remove interrupted item" << Log::Field("itemID", item.mItemID) + << Log::Field("version", item.mVersion); + + // The install was interrupted rather than failed, so the item is kept in the cache with its blobs as outdated. + // The item status is not notified on start, it is reported by GetItemsStatuses. + item.mState = ItemStateEnum::eRemoved; + item.mTimestamp = Time::Now(); + item.mError = ErrorEnum::eNone; + + if (auto err = mStorage->UpdateDeployableItem(item); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + void ImageManager::RestoreRemovingItemState(const UpdateItemStatus& prevStatus, const Error& error) { { diff --git a/src/core/common/imagemanager/imagemanager.hpp b/src/core/common/imagemanager/imagemanager.hpp index e9e57294f..027acc6a9 100644 --- a/src/core/common/imagemanager/imagemanager.hpp +++ b/src/core/common/imagemanager/imagemanager.hpp @@ -38,9 +38,11 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI virtual ~ImageManager() = default; /** - * Starts image manager. + * Starts image manager. Sets the items whose install was interrupted (left in the eInstalling state) to the + * eRemoved state, keeping their blobs, and adds all the items in the eRemoved state as outdated to the space + * allocator. No item status is notified. * - * @return Error. + * @return Error: storage errors as is. Errors of adding an outdated item are only logged. */ virtual Error Start(); @@ -122,7 +124,8 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI * * @note The allocator must provide, per concurrent InstallItem (up to cMaxNumConcurrentItems), one * oci::ImageIndex, one oci::ImageManifest, one oci::ImageConfig and one blob hash buffer of cBlobHashSize at the - * same time, plus one StaticArray for GetItemsStatuses. + * same time, plus one StaticArray for Start and GetItemsStatuses and one + * StaticArray for the item remover. * * @param config image manager config. * @param downloadBlobInfoProvider download blob info provider. @@ -187,6 +190,8 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI static constexpr auto cMaxNumStoredItems = cMaxNumUpdateItems * 2; static constexpr auto cMaxNumItemBlobs = 1 + oci::cMaxNumManifests * (3 + oci::cMaxNumLayers); static constexpr auto cRetryTimeout = Time::cSeconds * 2; + // One extra slot for the item remover, which is called synchronously by the space allocator and must not wait. + static constexpr auto cMaxNumItemsInProgress = cMaxNumConcurrentItems + 1; /** * Downloads blob. Called by InstallItem, derived classes implement it but must not call it directly. @@ -283,6 +288,7 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI }; RetWithError RemoveOutdatedItem(const String& itemID, const String& version); + RetWithError RemoveUnusedItemBlob(const ItemBlobInfo& blob); Error ValidateBlob(const String& path, const String& digest, size_t size = 0); Error GetBlobInfo(const String& digest, BlobInfo& info); @@ -305,15 +311,19 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI Error SetInProgressItemState(const String& itemID, const String& version, const UpdateItemType& type, ItemStateEnum state, const Error& error = ErrorEnum::eNone); + Error RemoveInterruptedItem(DeployableItemInfo& item); void RestoreRemovingItemState(const UpdateItemStatus& prevStatus, const Error& error); Error SetStoredItemState(DeployableItemInfo& item, ItemStateEnum state, const Error& error = ErrorEnum::eNone); void NotifyItemStatusChanged(const UpdateItemStatus& status); + void NotifyItemRemovedFromCache(const String& itemID, const String& version); Error LockItem(const String& itemID, const String& version); + bool TryLockItem(const String& itemID, const String& version); void UnlockItem(const String& itemID, const String& version); UpdateItemStatus* FindInProgressItem(const String& itemID, const String& version); Error LockBlob(const String& digest); + bool TryLockBlob(const String& digest); void UnlockBlob(const String& digest); bool IsBlobInProgress(const String& digest) const @@ -335,8 +345,8 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI Mutex mMutex; ConditionalVariable mCondVar; - StaticList mItemsInProgress; - StaticList, cMaxNumConcurrentItems> mBlobsInProgress; + StaticList mItemsInProgress; + StaticList, cMaxNumItemsInProgress> mBlobsInProgress; ItemRemover mItemRemover {*this}; }; diff --git a/src/core/common/imagemanager/tests/imagemanager.cpp b/src/core/common/imagemanager/tests/imagemanager.cpp index 0db6a9b3a..d062dbc6b 100644 --- a/src/core/common/imagemanager/tests/imagemanager.cpp +++ b/src/core/common/imagemanager/tests/imagemanager.cpp @@ -446,6 +446,108 @@ TEST_F(CommonImageManagerTest, SubscribeUnsubscribeListener) EXPECT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); } +TEST_F(CommonImageManagerTest, StartRemovesInterruptedItemsAndAddsOutdatedItems) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + ASSERT_TRUE(mImageManager.SubscribeListener(mListener).IsNone()); + + AddStoredItem("item1", "1.0.0", ItemStateEnum::eInstalled); + AddStoredItem("item1", "2.0.0", ItemStateEnum::eInstalling); + AddStoredItem("item2", "1.0.0", ItemStateEnum::eRemoved); + + ASSERT_TRUE( + mStorage.AddItemBlob("item1", "2.0.0", ItemBlobInfo {{cTestLayerType, cTestDigest, 1}, true, ""}).IsNone()); + AddStoredItem("item3", "1.0.0", ItemStateEnum::eFailed, Error(ErrorEnum::eFailed, "download failed")); + AddStoredItem("item4", "1.0.0", ItemStateEnum::eRemoved); + + std::map storedItems; + + for (const auto& [itemID, version] : std::vector>( + {{"item1", "1.0.0"}, {"item2", "1.0.0"}, {"item3", "1.0.0"}, {"item4", "1.0.0"}})) { + DeployableItemInfo item; + + ASSERT_TRUE(mStorage.GetDeployableItem(itemID, version, item).IsNone()); + + storedItems[std::string(itemID) + ":" + version] = item; + } + + EXPECT_CALL(mListener, OnItemStatusChanged(_)).Times(0); + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(_, _, _)).Times(0); + EXPECT_CALL( + mSpaceAllocator, AddOutdatedItem(String("item2"), String("1.0.0"), storedItems["item2:1.0.0"].mTimestamp)) + .WillOnce(Return(ErrorEnum::eNone)); + EXPECT_CALL( + mSpaceAllocator, AddOutdatedItem(String("item4"), String("1.0.0"), storedItems["item4:1.0.0"].mTimestamp)) + .WillOnce(Return(ErrorEnum::eNone)); + + Time interruptedTimestamp; + + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String("item1"), String("2.0.0"), _)) + .WillOnce(Invoke([&](const String&, const String&, const Time& timestamp) { + interruptedTimestamp = timestamp; + + return ErrorEnum::eNone; + })); + + auto err = mImageManager.Start(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + DeployableItemInfo item; + + ASSERT_TRUE(mStorage.GetDeployableItem("item1", "2.0.0", item).IsNone()); + EXPECT_EQ(item.mState, ItemStateEnum::eRemoved); + EXPECT_TRUE(item.mError.IsNone()) << tests::utils::ErrorToStr(item.mError); + EXPECT_EQ(item.mTimestamp, interruptedTimestamp); + + // The blob references of the interrupted item are kept. + auto blobs = std::make_unique>(); + + ASSERT_TRUE(mStorage.GetItemBlobs("item1", "2.0.0", *blobs).IsNone()); + EXPECT_EQ(blobs->Size(), 1); + + // The other items are not changed. + for (const auto& [key, storedItem] : storedItems) { + ASSERT_TRUE(mStorage.GetDeployableItem(storedItem.mItemID, storedItem.mVersion, item).IsNone()); + EXPECT_EQ(item, storedItem) << key; + } +} + +TEST_F(CommonImageManagerTest, StartAddOutdatedItemFails) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + AddStoredItem("item1", "1.0.0", ItemStateEnum::eRemoved); + AddStoredItem("item2", "1.0.0", ItemStateEnum::eRemoved); + + // The failed item is only logged and the next items are still added. + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String("item1"), String("1.0.0"), _)) + .WillOnce(Return(ErrorEnum::eFailed)); + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String("item2"), String("1.0.0"), _)) + .WillOnce(Return(ErrorEnum::eNone)); + + auto err = mImageManager.Start(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + +TEST_F(CommonImageManagerTest, StartUpdateItemFails) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + AddStoredItem("item1", "1.0.0", ItemStateEnum::eInstalling); + + mStorage.SetFailUpdateStates({ItemStateEnum::eRemoved}); + + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(_, _, _)).Times(0); + + auto err = mImageManager.Start(); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + DeployableItemInfo item; + + ASSERT_TRUE(mStorage.GetDeployableItem("item1", "1.0.0", item).IsNone()); + EXPECT_EQ(item.mState, ItemStateEnum::eInstalling); +} + TEST_F(CommonImageManagerTest, GetItemsStatusesEmpty) { ASSERT_TRUE(InitImageManager().IsNone()); @@ -2146,6 +2248,28 @@ TEST_F(CommonImageManagerRemoveTest, RemoveSetRemovedStateFails) EXPECT_EQ(GetReportedState(cTestItemID, cTestVersion), ItemStateEnum::eInstalled); } +TEST_F(CommonImageManagerRemoveTest, RemoveItemUnlockedWhenAddedAsOutdated) +{ + AddStoredItem(cTestItemID, cTestVersion, ItemStateEnum::eInstalled); + + bool storedAfterRemover = true; + + // The space allocator can remove the oldest outdated item synchronously, so the item must not be locked here. + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String(cTestItemID), String(cTestVersion), _)) + .WillOnce(Invoke([&](const String&, const String&, const Time&) { + EXPECT_TRUE(mImageManager.GetItemRemover().RemoveItem(cTestItemID, cTestVersion).mError.IsNone()); + + storedAfterRemover = GetStoredItem(cTestItemID, cTestVersion).has_value(); + + return ErrorEnum::eNone; + })); + + auto err = mImageManager.Remove(cTestItemID, cTestVersion); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_FALSE(storedAfterRemover); +} + TEST_F(CommonImageManagerRemoveTest, RemoveAddOutdatedItemFails) { AddStoredItem(cTestItemID, cTestVersion, ItemStateEnum::eInstalled); @@ -2234,4 +2358,199 @@ TEST_F(CommonImageManagerRemoveTest, RemoveWaitsForInstall) EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); } +/*********************************************************************************************************************** + * Remove outdated item suite + **********************************************************************************************************************/ + +class CommonImageManagerRemoveOutdatedTest : public CommonImageManagerInstallTest { +protected: + void SetUp() override + { + CommonImageManagerInstallTest::SetUp(); + + ON_CALL(mSpaceAllocator, AddOutdatedItem(_, _, _)).WillByDefault(Return(ErrorEnum::eNone)); + } + + void InstallAndRemove(const TestManifest& manifest, const char* itemID = cTestItemID) + { + auto err = mImageManager.Install( + MakeInstallInfo(manifest.mManifest, false, UpdateItemTypeEnum::eService, itemID, cTestVersion)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + err = mImageManager.Remove(itemID, cTestVersion); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + } + + RetWithError RemoveOutdated(const char* itemID = cTestItemID) + { + return mImageManager.GetItemRemover().RemoveItem(itemID, cTestVersion); + } +}; + +TEST_F(CommonImageManagerRemoveOutdatedTest, RemoveUnusedBlobs) +{ + const auto sharedLayer = MakeTestBlob(500); + const auto manifest = AddManifest(1, 1, 2, false, {sharedLayer}); + const auto other = AddManifest(200, 0, 1, false, {sharedLayer}); + + InstallAndRemove(manifest); + InstallAndRemove(other, "item2"); + + const auto& ownLayer = manifest.mLayers[1]; + + EXPECT_CALL( + mImageManager, RemoveLayer(String(ownLayer.mPath.c_str()), _, String(manifest.mDiffIDs[1].mDigest.c_str()))) + .WillOnce(Return(RetWithError(10, ErrorEnum::eNone))); + EXPECT_CALL(mImageManager, RemoveLayer(String(sharedLayer.mPath.c_str()), _, _)).Times(0); + EXPECT_CALL(mListener, OnItemRemovedFromCache(String(cTestItemID), String(cTestVersion))); + EXPECT_CALL(mListener, OnItemStatusChanged(_)).Times(0); + EXPECT_CALL(mSpaceAllocator, FreeSpace(_)).Times(0); + + auto [size, err] = RemoveOutdated(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + // Manifest, image config and own layer blobs plus the unpacked layer data. + EXPECT_EQ(size, 3 * cInstallBlobSize + 10); + + EXPECT_FALSE(GetStoredItem(cTestItemID, cTestVersion).has_value()); + + for (const auto* blob : {&manifest.mManifest, &manifest.mImageConfig, &ownLayer}) { + EXPECT_FALSE(FileExists(blob->mPath.c_str())) << blob->mDigest; + } + + // The shared layer is used by item2. + EXPECT_TRUE(FileExists(sharedLayer.mPath.c_str())); +} + +TEST_F(CommonImageManagerRemoveOutdatedTest, SkipNotRemovedItem) +{ + AddStoredItem(cTestItemID, cTestVersion, ItemStateEnum::eInstalled); + + EXPECT_CALL(mListener, OnItemRemovedFromCache(_, _)).Times(0); + + auto [size, err] = RemoveOutdated(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + EXPECT_EQ(size, 0); + EXPECT_TRUE(GetStoredItem(cTestItemID, cTestVersion).has_value()); + + // Not stored item is already removed from the cache. + Tie(size, err) = RemoveOutdated("item2"); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + EXPECT_EQ(size, 0); +} + +TEST_F(CommonImageManagerRemoveOutdatedTest, SkipItemInProgress) +{ + AddStoredItem(cTestItemID, cTestVersion, ItemStateEnum::eInstalled); + + std::optional> removeResult; + + // The item is already in the eRemoved state in the storage, but it is still being removed by RemoveItem. + EXPECT_CALL(mListener, OnItemStatusChanged(_)).WillRepeatedly(Invoke([&](const UpdateItemStatus& status) { + if (status.mState == ItemStateEnum::eRemoved) { + removeResult = RemoveOutdated(); + } + })); + EXPECT_CALL(mListener, OnItemRemovedFromCache(_, _)).Times(0); + + auto err = mImageManager.Remove(cTestItemID, cTestVersion); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + ASSERT_TRUE(removeResult.has_value()); + EXPECT_TRUE(removeResult->mError.IsNone()) << tests::utils::ErrorToStr(removeResult->mError); + EXPECT_EQ(removeResult->mValue, 0); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); +} + +TEST_F(CommonImageManagerRemoveOutdatedTest, SkipLockedBlob) +{ + const auto sharedLayer = MakeTestBlob(500); + const auto manifest = AddManifest(1, 0, 1, false, {sharedLayer}); + const auto other = AddManifest(200, 0, 1, false, {sharedLayer}); + + InstallAndRemove(manifest); + + ASSERT_TRUE(fs::RemoveAll(sharedLayer.mPath.c_str()).IsNone()); + + std::promise downloadStarted; + std::promise downloadReleased; + auto downloadReleasedFuture = downloadReleased.get_future(); + + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).WillRepeatedly(Invoke(DownloadTestBlob)); + EXPECT_CALL(mImageManager, DownloadBlob(String(sharedLayer.mPath.c_str()), _)) + .WillOnce(Invoke([&](const String& path, const BlobInfo& blobInfo) { + downloadStarted.set_value(); + downloadReleasedFuture.wait(); + + return DownloadTestBlob(path, blobInfo); + })); + + Error installErr; + + std::thread installThread([&]() { + installErr + = mImageManager.Install(MakeInstallInfo(other.mManifest, false, UpdateItemTypeEnum::eService, "item2")); + }); + + downloadStarted.get_future().wait(); + + // The shared layer is locked by item2 being downloaded: the item remover doesn't wait for it. + auto [size, err] = RemoveOutdated(); + + downloadReleased.set_value(); + installThread.join(); + + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + EXPECT_EQ(size, 2 * cInstallBlobSize); + EXPECT_FALSE(GetStoredItem(cTestItemID, cTestVersion).has_value()); + + ASSERT_TRUE(installErr.IsNone()) << tests::utils::ErrorToStr(installErr); + EXPECT_EQ(ReadFile(sharedLayer.mPath), sharedLayer.mHash); +} + +TEST_F(CommonImageManagerRemoveOutdatedTest, RemoveLayerFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + InstallAndRemove(manifest); + + EXPECT_CALL(mImageManager, RemoveLayer(_, _, _)).WillOnce(Return(RetWithError(10, ErrorEnum::eFailed))); + EXPECT_CALL(mListener, OnItemRemovedFromCache(String(cTestItemID), String(cTestVersion))); + + auto [size, err] = RemoveOutdated(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + // The layer blob is still removed, the failed unpacked layer data is not counted. + EXPECT_EQ(size, 3 * cInstallBlobSize); + EXPECT_FALSE(FileExists(manifest.mLayers[0].mPath.c_str())); +} + +TEST_F(CommonImageManagerRemoveOutdatedTest, RemoveDeployableItemFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + InstallAndRemove(manifest); + + mStorage.SetFailRemoveItem(true); + + EXPECT_CALL(mImageManager, RemoveLayer(_, _, _)).Times(0); + EXPECT_CALL(mListener, OnItemRemovedFromCache(_, _)).Times(0); + + auto [size, err] = RemoveOutdated(); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + EXPECT_EQ(size, 0); + + // No blob is removed and the item stays in the eRemoved state. + for (const auto* blob : {&manifest.mManifest, &manifest.mImageConfig, &manifest.mLayers[0]}) { + EXPECT_TRUE(FileExists(blob->mPath.c_str())) << blob->mDigest; + } + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); +} + } // namespace aos::common::imagemanager diff --git a/src/core/common/imagemanager/tests/stubs/storagestub.hpp b/src/core/common/imagemanager/tests/stubs/storagestub.hpp index 61debabb9..84b1ecb19 100644 --- a/src/core/common/imagemanager/tests/stubs/storagestub.hpp +++ b/src/core/common/imagemanager/tests/stubs/storagestub.hpp @@ -56,6 +56,10 @@ class StorageStub : public StorageItf { { std::lock_guard lock {mMutex}; + if (mFailRemoveItem) { + return Error(ErrorEnum::eFailed, "remove failed"); + } + auto it = Find(itemID, version); if (it == mItems.end()) { return ErrorEnum::eNotFound; @@ -177,6 +181,13 @@ class StorageStub : public StorageItf { mFailUpdateStates = states; } + void SetFailRemoveItem(bool fail) + { + std::lock_guard lock {mMutex}; + + mFailRemoveItem = fail; + } + private: struct Item { DeployableItemInfo mInfo; @@ -193,6 +204,8 @@ class StorageStub : public StorageItf { std::vector mItems; // Item states which UpdateDeployableItem fails to set. std::set mFailUpdateStates; + // Makes RemoveDeployableItem fail. + bool mFailRemoveItem {}; }; } // namespace aos::common::imagemanager From 91856e80d1008ab27f97d61703d7b12e20f76d5a Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Fri, 30 Oct 2026 12:47:47 +0200 Subject: [PATCH 12/54] common: imagemanager: add item info provider Replace LocalBlobInfoProviderItf with ItemInfoProviderItf in a separate header. Besides GetBlobPath, it provides GetLayerPath to get the path of an unpacked layer by its diff ID. The common image manager doesn't keep the unpacked layers, so its GetLayerPath returns eNotSupported. The SM image manager overrides it. Rename DownloadBlobInfoProviderItf to BlobInfoProviderItf, as it is the only blob info provider of the common image manager now. Update the mocks accordingly and add a unit test for GetLayerPath. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko --- src/core/common/imagemanager/CMakeLists.txt | 4 +- src/core/common/imagemanager/imagemanager.cpp | 10 +++- src/core/common/imagemanager/imagemanager.hpp | 19 ++++++-- .../imagemanager/itf/blobinfoprovider.hpp | 24 +--------- .../imagemanager/itf/iteminfoprovider.hpp | 46 +++++++++++++++++++ .../imagemanager/tests/imagemanager.cpp | 12 ++++- src/core/common/tests/mocks/CMakeLists.txt | 1 + .../tests/mocks/blobinfoprovidermock.hpp | 12 +---- .../tests/mocks/iteminfoprovidermock.hpp | 27 +++++++++++ 9 files changed, 116 insertions(+), 39 deletions(-) create mode 100644 src/core/common/imagemanager/itf/iteminfoprovider.hpp create mode 100644 src/core/common/tests/mocks/iteminfoprovidermock.hpp diff --git a/src/core/common/imagemanager/CMakeLists.txt b/src/core/common/imagemanager/CMakeLists.txt index ed347a15b..1b3e3de9c 100644 --- a/src/core/common/imagemanager/CMakeLists.txt +++ b/src/core/common/imagemanager/CMakeLists.txt @@ -20,7 +20,9 @@ set(SOURCES imagemanager.cpp) # Headers # ###################################################################################################################### -set(HEADERS config.hpp imagemanager.hpp itf/blobinfoprovider.hpp itf/itemstatusprovider.hpp itf/storage.hpp) +set(HEADERS config.hpp imagemanager.hpp itf/blobinfoprovider.hpp itf/iteminfoprovider.hpp itf/itemstatusprovider.hpp + itf/storage.hpp +) # ###################################################################################################################### # Libraries diff --git a/src/core/common/imagemanager/imagemanager.cpp b/src/core/common/imagemanager/imagemanager.cpp index 009630c7b..0ed657f7f 100644 --- a/src/core/common/imagemanager/imagemanager.cpp +++ b/src/core/common/imagemanager/imagemanager.cpp @@ -208,11 +208,19 @@ Error ImageManager::GetBlobPath(const String& digest, String& path) const return ErrorEnum::eNone; } +Error ImageManager::GetLayerPath(const String& diffID, String& path) const +{ + (void)diffID; + (void)path; + + return ErrorEnum::eNotSupported; +} + /*********************************************************************************************************************** * Protected **********************************************************************************************************************/ -Error ImageManager::Init(const Config& config, DownloadBlobInfoProviderItf& blobInfoProvider, StorageItf& storage, +Error ImageManager::Init(const Config& config, BlobInfoProviderItf& blobInfoProvider, StorageItf& storage, spaceallocator::SpaceAllocatorItf& spaceAllocator, fs::FileInfoProviderItf& fileInfoProvider, oci::OCISpecItf& ociSpec, fs::FSPlatformItf& fsPlatform, AllocatorItf& allocator) { diff --git a/src/core/common/imagemanager/imagemanager.hpp b/src/core/common/imagemanager/imagemanager.hpp index 027acc6a9..85781613f 100644 --- a/src/core/common/imagemanager/imagemanager.hpp +++ b/src/core/common/imagemanager/imagemanager.hpp @@ -17,6 +17,7 @@ #include #include "itf/blobinfoprovider.hpp" +#include "itf/iteminfoprovider.hpp" #include "itf/itemstatusprovider.hpp" #include "itf/storage.hpp" @@ -30,7 +31,7 @@ namespace aos::common::imagemanager { * Implements the part of the image manager shared by CM and SM. CM and SM image managers are derived from this class * and specialize blob download and layer handling. */ -class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderItf, private NonCopyable { +class ImageManager : public ItemStatusProviderItf, public ItemInfoProviderItf, private NonCopyable { public: /** * Destructor. @@ -100,6 +101,16 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI */ Error GetBlobPath(const String& digest, String& path) const override; + /** + * Returns unpacked layer path. Returns eNotSupported by default, as the common image manager doesn't keep the + * unpacked layers. + * + * @param diffID layer diff ID. + * @param[out] path unpacked layer path. + * @return Error. + */ + Error GetLayerPath(const String& diffID, String& path) const override; + /** * Returns item remover which should be passed to the space allocator to remove outdated items. * @@ -128,7 +139,7 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI * StaticArray for the item remover. * * @param config image manager config. - * @param downloadBlobInfoProvider download blob info provider. + * @param blobInfoProvider blob info provider. * @param storage image manager storage. * @param spaceAllocator space allocator. * @param fileInfoProvider file info provider. @@ -137,7 +148,7 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI * @param allocator allocator for temporary objects. * @return Error. */ - Error Init(const Config& config, DownloadBlobInfoProviderItf& downloadBlobInfoProvider, StorageItf& storage, + Error Init(const Config& config, BlobInfoProviderItf& blobInfoProvider, StorageItf& storage, spaceallocator::SpaceAllocatorItf& spaceAllocator, fs::FileInfoProviderItf& fileInfoProvider, oci::OCISpecItf& ociSpec, fs::FSPlatformItf& fsPlatform, AllocatorItf& allocator); @@ -332,7 +343,7 @@ class ImageManager : public ItemStatusProviderItf, public LocalBlobInfoProviderI } Config mConfig; - DownloadBlobInfoProviderItf* mBlobInfoProvider {}; + BlobInfoProviderItf* mBlobInfoProvider {}; StorageItf* mStorage {}; spaceallocator::SpaceAllocatorItf* mSpaceAllocator {}; fs::FileInfoProviderItf* mFileInfoProvider {}; diff --git a/src/core/common/imagemanager/itf/blobinfoprovider.hpp b/src/core/common/imagemanager/itf/blobinfoprovider.hpp index d3428d7b7..e64811ea3 100644 --- a/src/core/common/imagemanager/itf/blobinfoprovider.hpp +++ b/src/core/common/imagemanager/itf/blobinfoprovider.hpp @@ -14,12 +14,12 @@ namespace aos::common::imagemanager { /** * Interface that provides download blob info. */ -class DownloadBlobInfoProviderItf { +class BlobInfoProviderItf { public: /** * Destructor. */ - virtual ~DownloadBlobInfoProviderItf() = default; + virtual ~BlobInfoProviderItf() = default; /** * Returns blob info. @@ -31,26 +31,6 @@ class DownloadBlobInfoProviderItf { virtual Error GetBlobInfo(const String& digest, BlobInfo& blobInfo) = 0; }; -/** - * Interface that provides local blob info. - */ -class LocalBlobInfoProviderItf { -public: - /** - * Destructor. - */ - virtual ~LocalBlobInfoProviderItf() = default; - - /** - * Returns local blob path. - * - * @param digest blob digest. - * @param[out] path local blob path. - * @return Error. - */ - virtual Error GetBlobPath(const String& digest, String& path) const = 0; -}; - } // namespace aos::common::imagemanager #endif diff --git a/src/core/common/imagemanager/itf/iteminfoprovider.hpp b/src/core/common/imagemanager/itf/iteminfoprovider.hpp new file mode 100644 index 000000000..c27a63b07 --- /dev/null +++ b/src/core/common/imagemanager/itf/iteminfoprovider.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_IMAGEMANAGER_ITF_ITEMINFOPROVIDER_HPP_ +#define AOS_CORE_COMMON_IMAGEMANAGER_ITF_ITEMINFOPROVIDER_HPP_ + +#include +#include + +namespace aos::common::imagemanager { + +/** + * Interface that provides local blob and unpacked layer info. + */ +class ItemInfoProviderItf { +public: + /** + * Destructor. + */ + virtual ~ItemInfoProviderItf() = default; + + /** + * Returns local blob path. + * + * @param digest blob digest. + * @param[out] path local blob path. + * @return Error. + */ + virtual Error GetBlobPath(const String& digest, String& path) const = 0; + + /** + * Returns unpacked layer path. + * + * @param diffID layer diff ID. + * @param[out] path unpacked layer path. + * @return Error: eNotSupported if the image manager doesn't keep the unpacked layers. + */ + virtual Error GetLayerPath(const String& diffID, String& path) const = 0; +}; + +} // namespace aos::common::imagemanager + +#endif diff --git a/src/core/common/imagemanager/tests/imagemanager.cpp b/src/core/common/imagemanager/tests/imagemanager.cpp index d062dbc6b..4f69633e1 100644 --- a/src/core/common/imagemanager/tests/imagemanager.cpp +++ b/src/core/common/imagemanager/tests/imagemanager.cpp @@ -312,7 +312,7 @@ class CommonImageManagerTest : public Test { // mAllocator must be declared before any member that allocates from it, so it is destroyed after them, since // members are destroyed in reverse declaration order. HeapAllocator mAllocator; - NiceMock mBlobInfoProvider; + NiceMock mBlobInfoProvider; StorageStub mStorage; NiceMock mSpaceAllocator; NiceMock mFileInfoProvider; @@ -397,6 +397,16 @@ TEST_F(CommonImageManagerTest, GetBlobPath) EXPECT_FALSE(FileExists(path)); } +TEST_F(CommonImageManagerTest, GetLayerPathNotSupported) +{ + ASSERT_TRUE(InitImageManager().IsNone()); + + StaticString path; + + auto err = mImageManager.GetLayerPath(cTestDigest, path); + EXPECT_TRUE(err.Is(ErrorEnum::eNotSupported)) << tests::utils::ErrorToStr(err); +} + TEST_F(CommonImageManagerTest, GetBlobPathMalformedDigest) { ASSERT_TRUE(InitImageManager().IsNone()); diff --git a/src/core/common/tests/mocks/CMakeLists.txt b/src/core/common/tests/mocks/CMakeLists.txt index f13d13502..346367c7b 100644 --- a/src/core/common/tests/mocks/CMakeLists.txt +++ b/src/core/common/tests/mocks/CMakeLists.txt @@ -25,6 +25,7 @@ set(HEADERS fsmock.hpp identprovidermock.hpp instancestatusprovidermock.hpp + iteminfoprovidermock.hpp itemstatuslistenermock.hpp logprovidermock.hpp monitoringmock.hpp diff --git a/src/core/common/tests/mocks/blobinfoprovidermock.hpp b/src/core/common/tests/mocks/blobinfoprovidermock.hpp index 290029e61..270d64f06 100644 --- a/src/core/common/tests/mocks/blobinfoprovidermock.hpp +++ b/src/core/common/tests/mocks/blobinfoprovidermock.hpp @@ -14,21 +14,13 @@ namespace aos::common::imagemanager { /** - * Download blob info provider mock. + * Blob info provider mock. */ -class DownloadBlobInfoProviderMock : public DownloadBlobInfoProviderItf { +class BlobInfoProviderMock : public BlobInfoProviderItf { public: MOCK_METHOD(Error, GetBlobInfo, (const String& digest, BlobInfo& blobInfo), (override)); }; -/** - * Local blob info provider mock. - */ -class LocalBlobInfoProviderMock : public LocalBlobInfoProviderItf { -public: - MOCK_METHOD(Error, GetBlobPath, (const String& digest, String& path), (const, override)); -}; - } // namespace aos::common::imagemanager #endif diff --git a/src/core/common/tests/mocks/iteminfoprovidermock.hpp b/src/core/common/tests/mocks/iteminfoprovidermock.hpp new file mode 100644 index 000000000..9c52b80b0 --- /dev/null +++ b/src/core/common/tests/mocks/iteminfoprovidermock.hpp @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_TESTS_MOCKS_ITEMINFOPROVIDERMOCK_HPP_ +#define AOS_CORE_COMMON_TESTS_MOCKS_ITEMINFOPROVIDERMOCK_HPP_ + +#include + +#include + +namespace aos::common::imagemanager { + +/** + * Item info provider mock. + */ +class ItemInfoProviderMock : public ItemInfoProviderItf { +public: + MOCK_METHOD(Error, GetBlobPath, (const String& digest, String& path), (const, override)); + MOCK_METHOD(Error, GetLayerPath, (const String& diffID, String& path), (const, override)); +}; + +} // namespace aos::common::imagemanager + +#endif From 2174ee4815dc35e84a34cb18d08d0728423be3ce Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Fri, 30 Oct 2026 13:29:54 +0200 Subject: [PATCH 13/54] common: imagemanager: remove outdated items when storage is full Before a new item version is added to the storage, InstallItem checks whether the storage is full, i.e. it keeps cMaxNumStoredItems items or more. While it is full, the oldest outdated item is removed with RemoveOutdatedItem of the space allocator. If the item remover skips the item, the next oldest outdated item is removed. If there is no outdated item to remove, InstallItem fails with eNoMemory before any blob is downloaded. Any other error of RemoveOutdatedItem is returned as is. The storage count check, the removal of the outdated items and adding the item are done under a separate storage mutex, so concurrent installs don't exceed the storage limit. The item remover never takes this mutex, so it can be held while the space allocator is called. Add unit tests for installing items when the storage is full. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko --- src/core/common/imagemanager/imagemanager.cpp | 40 ++++++- src/core/common/imagemanager/imagemanager.hpp | 3 + .../imagemanager/tests/imagemanager.cpp | 109 ++++++++++++++++++ 3 files changed, 150 insertions(+), 2 deletions(-) diff --git a/src/core/common/imagemanager/imagemanager.cpp b/src/core/common/imagemanager/imagemanager.cpp index 0ed657f7f..0a77d59ad 100644 --- a/src/core/common/imagemanager/imagemanager.cpp +++ b/src/core/common/imagemanager/imagemanager.cpp @@ -983,8 +983,8 @@ Error ImageManager::StartItemInstall(const InstallItemInfo& itemInfo, Deployable storedInfo.mTimestamp = Time::Now(); storedInfo.mError = ErrorEnum::eNone; - if (err = mStorage->AddDeployableItem(storedInfo); !err.IsNone()) { - return AOS_ERROR_WRAP(err); + if (err = AddStoredItem(storedInfo); !err.IsNone()) { + return err; } if (err = SetInProgressItemState(itemInfo.mItemID, itemInfo.mVersion, itemInfo.mType, ItemStateEnum::eInstalling); @@ -995,6 +995,42 @@ Error ImageManager::StartItemInstall(const InstallItemInfo& itemInfo, Deployable return ErrorEnum::eNone; } +Error ImageManager::AddStoredItem(const DeployableItemInfo& item) +{ + // Serializes the storage count check with adding the item, so concurrent installs don't exceed the storage limit. + // The item remover doesn't take this mutex, so it can be held while the space allocator is called. + LockGuard lock {mStorageMutex}; + + while (true) { + auto [count, err] = mStorage->GetDeployableItemsCount(); + if (!err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (count < cMaxNumStoredItems) { + break; + } + + LOG_DBG() << "Storage is full, remove outdated item" << Log::Field("count", count); + + // The space allocator drops the outdated item whatever the result, so the loop ends when no outdated item is + // left. + if (err = mSpaceAllocator->RemoveOutdatedItem().mError; !err.IsNone()) { + if (err.Is(ErrorEnum::eNotFound)) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eNoMemory, "storage is full")); + } + + return AOS_ERROR_WRAP(err); + } + } + + if (auto err = mStorage->AddDeployableItem(item); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + Error ImageManager::InstallItemBlobs(const InstallItemInfo& itemInfo) { if (!itemInfo.mIsRootIndex) { diff --git a/src/core/common/imagemanager/imagemanager.hpp b/src/core/common/imagemanager/imagemanager.hpp index 85781613f..d1fae3a03 100644 --- a/src/core/common/imagemanager/imagemanager.hpp +++ b/src/core/common/imagemanager/imagemanager.hpp @@ -317,6 +317,7 @@ class ImageManager : public ItemStatusProviderItf, public ItemInfoProviderItf, p Error InstallItemBlobs(const InstallItemInfo& itemInfo); Error InstallManifestBlobs(const InstallItemInfo& itemInfo, const oci::ContentDescriptor& descriptor); Error StartItemInstall(const InstallItemInfo& itemInfo, DeployableItemInfo& storedInfo); + Error AddStoredItem(const DeployableItemInfo& item); Error AddItemBlob(const String& itemID, const String& version, const oci::ContentDescriptor& descriptor, bool isLayer = false, const String& diffID = ""); @@ -354,6 +355,8 @@ class ImageManager : public ItemStatusProviderItf, public ItemInfoProviderItf, p Mutex mListenersMutex; StaticArray mListeners; + Mutex mStorageMutex; + Mutex mMutex; ConditionalVariable mCondVar; StaticList mItemsInProgress; diff --git a/src/core/common/imagemanager/tests/imagemanager.cpp b/src/core/common/imagemanager/tests/imagemanager.cpp index 4f69633e1..2c336b8dd 100644 --- a/src/core/common/imagemanager/tests/imagemanager.cpp +++ b/src/core/common/imagemanager/tests/imagemanager.cpp @@ -1729,6 +1729,115 @@ TEST_F(CommonImageManagerInstallTest, InstallStoredItemDoesNotRestoreOutdatedIte EXPECT_TRUE(item->mError.IsNone()); } +TEST_F(CommonImageManagerInstallTest, InstallStorageFullRemovesOutdatedItem) +{ + const auto manifest = AddManifest(1, 1, 1); + + for (size_t i = 0; i < cMaxNumStoredItems - 1; ++i) { + AddStoredItem(("item" + std::to_string(i + 100)).c_str(), cTestVersion, ItemStateEnum::eInstalled); + } + + AddStoredItem("outdated", cTestVersion, ItemStateEnum::eRemoved); + + EXPECT_CALL(mSpaceAllocator, RemoveOutdatedItem()).WillOnce(Invoke([this]() { + return mImageManager.GetItemRemover().RemoveItem("outdated", cTestVersion); + })); + EXPECT_CALL(mListener, OnItemRemovedFromCache(String("outdated"), String(cTestVersion))); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(mStorage.GetItems().size(), cMaxNumStoredItems); + EXPECT_FALSE(GetStoredItem("outdated", cTestVersion).has_value()); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); +} + +TEST_F(CommonImageManagerInstallTest, InstallStorageFullOutdatedItemSkipped) +{ + const auto manifest = AddManifest(1, 1, 1); + + for (size_t i = 0; i < cMaxNumStoredItems - 1; ++i) { + AddStoredItem(("item" + std::to_string(i + 100)).c_str(), cTestVersion, ItemStateEnum::eInstalled); + } + + AddStoredItem("outdated", cTestVersion, ItemStateEnum::eRemoved); + + // The first outdated item is skipped by the item remover, so the next one is removed. + EXPECT_CALL(mSpaceAllocator, RemoveOutdatedItem()) + .WillOnce(Return(RetWithError(0, ErrorEnum::eNone))) + .WillOnce(Invoke([this]() { return mImageManager.GetItemRemover().RemoveItem("outdated", cTestVersion); })); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(mStorage.GetItems().size(), cMaxNumStoredItems); + EXPECT_FALSE(GetStoredItem("outdated", cTestVersion).has_value()); +} + +TEST_F(CommonImageManagerInstallTest, InstallStorageFullNoOutdatedItem) +{ + const auto manifest = AddManifest(1, 1, 1); + + for (size_t i = 0; i < cMaxNumStoredItems; ++i) { + AddStoredItem(("item" + std::to_string(i + 100)).c_str(), cTestVersion, ItemStateEnum::eInstalled); + } + + const auto storedItems = mStorage.GetItems(); + + EXPECT_CALL(mSpaceAllocator, RemoveOutdatedItem()).WillOnce(Return(RetWithError(0, ErrorEnum::eNotFound))); + EXPECT_CALL(mListener, OnItemStatusChanged(_)).Times(0); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eNoMemory)) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(mStorage.GetItems(), storedItems); +} + +TEST_F(CommonImageManagerInstallTest, InstallStorageFullRemoveOutdatedItemFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + for (size_t i = 0; i < cMaxNumStoredItems; ++i) { + AddStoredItem(("item" + std::to_string(i + 100)).c_str(), cTestVersion, ItemStateEnum::eRemoved); + } + + const auto storedItems = mStorage.GetItems(); + + EXPECT_CALL(mSpaceAllocator, RemoveOutdatedItem()).WillOnce(Return(RetWithError(0, ErrorEnum::eFailed))); + EXPECT_CALL(mListener, OnItemStatusChanged(_)).Times(0); + EXPECT_CALL(mImageManager, DownloadBlob(_, _)).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + EXPECT_TRUE(err.Is(ErrorEnum::eFailed)) << tests::utils::ErrorToStr(err); + + EXPECT_EQ(mStorage.GetItems(), storedItems); +} + +TEST_F(CommonImageManagerInstallTest, InstallStoredItemStorageFull) +{ + const auto manifest = AddManifest(1, 1, 1); + + StoreItem(cTestItemID, cTestVersion, manifest.mManifest.mDigest.c_str(), ItemStateEnum::eFailed); + + for (size_t i = 0; i < cMaxNumStoredItems - 1; ++i) { + AddStoredItem(("item" + std::to_string(i + 100)).c_str(), cTestVersion, ItemStateEnum::eInstalled); + } + + // The stored item already has its storage record. + EXPECT_CALL(mSpaceAllocator, RemoveOutdatedItem()).Times(0); + + auto err = mImageManager.Install(MakeInstallInfo(manifest.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eInstalled); +} + TEST_F(CommonImageManagerInstallTest, InstallBlobInfoNotFound) { const auto manifest = AddManifest(1, 1, 1); From 14c4305d6d4011e2f3c822c66015d22460271e44 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Fri, 30 Oct 2026 13:59:16 +0200 Subject: [PATCH 14/54] common: imagemanager: remove expired items Remove the items whose cache TTL is expired, i.e. the items in the eRemoved state for longer than mItemCacheTTL. The removal is done by a separate thread on start, after the outdated items are registered, and then every mRemoveOutdatedPeriod triggered by a timer. Each expired item is locked without waiting: the items being installed or removed are skipped and checked again in the next period. The item is restored from the outdated items of the space allocator, then read again from the storage, as it could be removed from the cache, installed again or removed again with a new timestamp after the items are read. Only the item which is still expired is removed with its unused blobs, the freed size is released with FreeSpace and OnItemRemovedFromCache is notified. The item which stays in the eRemoved state is added as outdated again. The removal of the item and its blobs is shared with the item remover. The lists of the items and blobs in progress get one more extra slot for the expired items removal, so it doesn't take the slot of the item remover. Stop stops the timer and the thread. Add unit tests for the expired items removal. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko --- src/core/common/imagemanager/imagemanager.cpp | 198 ++++++++++- src/core/common/imagemanager/imagemanager.hpp | 36 +- .../imagemanager/tests/imagemanager.cpp | 309 +++++++++++++++++- .../imagemanager/tests/stubs/storagestub.hpp | 13 + 4 files changed, 535 insertions(+), 21 deletions(-) diff --git a/src/core/common/imagemanager/imagemanager.cpp b/src/core/common/imagemanager/imagemanager.cpp index 0a77d59ad..b43670c0f 100644 --- a/src/core/common/imagemanager/imagemanager.cpp +++ b/src/core/common/imagemanager/imagemanager.cpp @@ -112,6 +112,34 @@ Error ImageManager::Start() } } + { + LockGuard lock {mMutex}; + + mClose = false; + // The expired items are removed on start, after the outdated items are registered. + mRemoveExpiredItems = true; + } + + if (auto err = mTimer.Start( + mConfig.mRemoveOutdatedPeriod, + [this](void*) { + LockGuard lock {mMutex}; + + mRemoveExpiredItems = true; + + if (auto err = mCondVar.NotifyAll(); !err.IsNone()) { + LOG_ERR() << "Can't notify condition variable" << Log::Field(err); + } + }, + false); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (auto err = mThread.Run([this](void*) { ProcessExpiredItems(); }); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + return ErrorEnum::eNone; } @@ -119,7 +147,27 @@ Error ImageManager::Stop() { LOG_DBG() << "Stop image manager"; - return ErrorEnum::eNotSupported; + Error stopErr; + + if (auto err = mTimer.Stop(Timer::StopMode::WaitForCallbacks); !err.IsNone()) { + stopErr = AOS_ERROR_WRAP(err); + } + + { + LockGuard lock {mMutex}; + + mClose = true; + + if (auto err = mCondVar.NotifyAll(); !err.IsNone()) { + LOG_ERR() << "Can't notify condition variable" << Log::Field(err); + } + } + + if (auto err = mThread.Join(); !err.IsNone() && stopErr.IsNone()) { + stopErr = AOS_ERROR_WRAP(err); + } + + return stopErr; } Error ImageManager::GetItemsStatuses(Array& statuses) @@ -744,33 +792,37 @@ RetWithError ImageManager::RemoveOutdatedItem(const String& itemID, cons return {0, ErrorEnum::eNone}; } + return RemoveItemFromCache(itemID, version); +} + +RetWithError ImageManager::RemoveItemFromCache(const String& itemID, const String& version) +{ auto blobs = MakeUnique>(mAllocator); if (!blobs) { return {0, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; } - if (err = mStorage->GetItemBlobs(itemID, version, *blobs); !err.IsNone()) { + if (auto err = mStorage->GetItemBlobs(itemID, version, *blobs); !err.IsNone()) { return {0, AOS_ERROR_WRAP(err)}; } - if (err = mStorage->RemoveDeployableItem(itemID, version); !err.IsNone()) { + if (auto err = mStorage->RemoveDeployableItem(itemID, version); !err.IsNone()) { return {0, AOS_ERROR_WRAP(err)}; } size_t freedSize = 0; for (const auto& blob : *blobs) { - size_t blobSize = 0; - // The blob which is failed to remove becomes orphan and is removed by the orphan blobs cleanup. - if (Tie(blobSize, err) = RemoveUnusedItemBlob(blob); !err.IsNone()) { + auto [blobSize, err] = RemoveUnusedItemBlob(blob); + if (!err.IsNone()) { LOG_ERR() << "Failed to remove item blob" << Log::Field("digest", blob.mDigest) << Log::Field(err); } freedSize += blobSize; } - LOG_INF() << "Outdated item removed" << Log::Field("itemID", itemID) << Log::Field("version", version) + LOG_INF() << "Item removed from cache" << Log::Field("itemID", itemID) << Log::Field("version", version) << Log::Field("freedSize", freedSize); NotifyItemRemovedFromCache(itemID, version); @@ -778,6 +830,138 @@ RetWithError ImageManager::RemoveOutdatedItem(const String& itemID, cons return {freedSize, ErrorEnum::eNone}; } +void ImageManager::ProcessExpiredItems() +{ + while (true) { + { + UniqueLock lock {mMutex}; + + if (auto err = mCondVar.Wait(lock, [this]() { return mClose || mRemoveExpiredItems; }); !err.IsNone()) { + LOG_ERR() << "Wait failed" << Log::Field(err); + + continue; + } + + if (mClose) { + break; + } + + mRemoveExpiredItems = false; + } + + if (auto err = RemoveExpiredItems(); !err.IsNone()) { + LOG_ERR() << "Failed to remove expired items" << Log::Field(err); + } + } +} + +Error ImageManager::RemoveExpiredItems() +{ + LOG_DBG() << "Remove expired items"; + + auto items = MakeUnique>(mAllocator); + if (!items) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = mStorage->GetAllDeployableItems(*items); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + for (const auto& item : *items) { + if (!IsItemExpired(item)) { + continue; + } + + // The failed item stays in the eRemoved state and is checked again in the next period. + if (auto err = RemoveExpiredItem(item.mItemID, item.mVersion, item.mTimestamp); !err.IsNone()) { + LOG_ERR() << "Failed to remove expired item" << Log::Field("itemID", item.mItemID) + << Log::Field("version", item.mVersion) << Log::Field(err); + } + } + + return ErrorEnum::eNone; +} + +bool ImageManager::IsItemExpired(const DeployableItemInfo& item) const +{ + return item.mState == ItemStateEnum::eRemoved && Time::Now().Sub(item.mTimestamp) >= mConfig.mItemCacheTTL; +} + +Error ImageManager::RemoveExpiredItem(const String& itemID, const String& version, const Time& timestamp) +{ + LOG_DBG() << "Remove expired item" << Log::Field("itemID", itemID) << Log::Field("version", version); + + // The item being installed or removed is skipped: after it, the item is either not removed or not expired, and + // otherwise it is checked again in the next period. + if (!TryLockItem(itemID, version)) { + LOG_DBG() << "Can't lock expired item" << Log::Field("itemID", itemID) << Log::Field("version", version); + + return ErrorEnum::eNone; + } + + auto unlockItem = DeferRelease([this, &itemID, &version]() { UnlockItem(itemID, version); }); + + auto err = mSpaceAllocator->RestoreOutdatedItem(itemID, version); + if (!err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { + LOG_ERR() << "Failed to restore outdated item" << Log::Field(err); + } + + const auto restored = err.IsNone(); + + DeployableItemInfo item; + + // The item which stays in the eRemoved state is added as outdated again. + const auto addOutdatedItem = [this, &restored, &unlockItem, &itemID, &version, &item]() { + if (!restored) { + return; + } + + // The space allocator can remove the outdated item right away, which skips the locked item. + unlockItem.Reset(); + + if (auto addErr = mSpaceAllocator->AddOutdatedItem(itemID, version, item.mTimestamp); !addErr.IsNone()) { + LOG_ERR() << "Failed to add outdated item" << Log::Field(addErr); + } + }; + + // If the storage can't be read, the item is checked again in the next period. + if (err = mStorage->GetDeployableItem(itemID, version, item); !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { + item.mTimestamp = timestamp; + + addOutdatedItem(); + + return AOS_ERROR_WRAP(err); + } + + // The items are read before the item is locked: in the meantime, the space allocator could remove the item from + // the cache, or the item could be installed again. + if (err.Is(ErrorEnum::eNotFound) || item.mState != ItemStateEnum::eRemoved) { + return ErrorEnum::eNone; + } + + // The item could also be installed and removed again, so it is not expired anymore. + if (!IsItemExpired(item)) { + addOutdatedItem(); + + return ErrorEnum::eNone; + } + + size_t freedSize = 0; + + if (Tie(freedSize, err) = RemoveItemFromCache(itemID, version); !err.IsNone()) { + addOutdatedItem(); + + return err; + } + + if (freedSize > 0) { + mSpaceAllocator->FreeSpace(freedSize); + } + + return ErrorEnum::eNone; +} + RetWithError ImageManager::RemoveUnusedItemBlob(const ItemBlobInfo& blob) { // A locked blob is being processed by an item being installed, so it is in use. The blob which can't be locked is diff --git a/src/core/common/imagemanager/imagemanager.hpp b/src/core/common/imagemanager/imagemanager.hpp index d1fae3a03..7917849e3 100644 --- a/src/core/common/imagemanager/imagemanager.hpp +++ b/src/core/common/imagemanager/imagemanager.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include "itf/blobinfoprovider.hpp" #include "itf/iteminfoprovider.hpp" @@ -41,15 +42,16 @@ class ImageManager : public ItemStatusProviderItf, public ItemInfoProviderItf, p /** * Starts image manager. Sets the items whose install was interrupted (left in the eInstalling state) to the * eRemoved state, keeping their blobs, and adds all the items in the eRemoved state as outdated to the space - * allocator. No item status is notified. + * allocator. No item status is notified. Then starts removing the expired items, i.e. the items in the eRemoved + * state for longer than mItemCacheTTL, right away and every mRemoveOutdatedPeriod. The items being installed or + * removed are skipped. * * @return Error: storage errors as is. Errors of adding an outdated item are only logged. */ virtual Error Start(); /** - * Stops image manager. Cancels all items being installed and waits until all InstallItem and RemoveItem calls in - * progress return. + * Stops image manager. Stops removing the expired items and waits until the removal in progress is finished. * * @return Error. */ @@ -135,8 +137,9 @@ class ImageManager : public ItemStatusProviderItf, public ItemInfoProviderItf, p * * @note The allocator must provide, per concurrent InstallItem (up to cMaxNumConcurrentItems), one * oci::ImageIndex, one oci::ImageManifest, one oci::ImageConfig and one blob hash buffer of cBlobHashSize at the - * same time, plus one StaticArray for Start and GetItemsStatuses and one - * StaticArray for the item remover. + * same time, plus two StaticArray for Start or the expired items removal + * and GetItemsStatuses, and two StaticArray for the item remover and the expired + * items removal. * * @param config image manager config. * @param blobInfoProvider blob info provider. @@ -201,8 +204,9 @@ class ImageManager : public ItemStatusProviderItf, public ItemInfoProviderItf, p static constexpr auto cMaxNumStoredItems = cMaxNumUpdateItems * 2; static constexpr auto cMaxNumItemBlobs = 1 + oci::cMaxNumManifests * (3 + oci::cMaxNumLayers); static constexpr auto cRetryTimeout = Time::cSeconds * 2; - // One extra slot for the item remover, which is called synchronously by the space allocator and must not wait. - static constexpr auto cMaxNumItemsInProgress = cMaxNumConcurrentItems + 1; + // Extra slots for the item remover and the expired items removal, which don't wait for the items and blobs. The + // item remover is called synchronously by the space allocator, so it must not wait or take the slot of another one. + static constexpr auto cMaxNumInProgress = cMaxNumConcurrentItems + 2; /** * Downloads blob. Called by InstallItem, derived classes implement it but must not call it directly. @@ -299,6 +303,11 @@ class ImageManager : public ItemStatusProviderItf, public ItemInfoProviderItf, p }; RetWithError RemoveOutdatedItem(const String& itemID, const String& version); + RetWithError RemoveItemFromCache(const String& itemID, const String& version); + void ProcessExpiredItems(); + Error RemoveExpiredItems(); + Error RemoveExpiredItem(const String& itemID, const String& version, const Time& timestamp); + bool IsItemExpired(const DeployableItemInfo& item) const; RetWithError RemoveUnusedItemBlob(const ItemBlobInfo& blob); Error ValidateBlob(const String& path, const String& digest, size_t size = 0); @@ -357,10 +366,15 @@ class ImageManager : public ItemStatusProviderItf, public ItemInfoProviderItf, p Mutex mStorageMutex; - Mutex mMutex; - ConditionalVariable mCondVar; - StaticList mItemsInProgress; - StaticList, cMaxNumItemsInProgress> mBlobsInProgress; + Mutex mMutex; + ConditionalVariable mCondVar; + StaticList mItemsInProgress; + StaticList, cMaxNumInProgress> mBlobsInProgress; + + Timer mTimer; + Thread<> mThread; + bool mClose {}; + bool mRemoveExpiredItems {}; ItemRemover mItemRemover {*this}; }; diff --git a/src/core/common/imagemanager/tests/imagemanager.cpp b/src/core/common/imagemanager/tests/imagemanager.cpp index 2c336b8dd..8575a74eb 100644 --- a/src/core/common/imagemanager/tests/imagemanager.cpp +++ b/src/core/common/imagemanager/tests/imagemanager.cpp @@ -68,6 +68,8 @@ constexpr auto cTestVersion = "1.0.0"; constexpr auto cImageConfigType = "application/vnd.oci.image.config.v1+json"; constexpr auto cItemConfigType = "application/vnd.aos.item.config.v1+json"; constexpr auto cInstallBlobSize = 64; +constexpr auto cItemCacheTTL = 10 * Time::cSeconds; +constexpr auto cWaitTimeout = std::chrono::seconds(5); // Same as the image manager private constants. constexpr auto cMaxNumStoredItems = cMaxNumUpdateItems * 2; @@ -290,13 +292,16 @@ class CommonImageManagerTest : public Test { void TearDown() override { - auto err = fs::RemoveAll(cTestDir); + auto err = mImageManager.Stop(); + EXPECT_TRUE(err.IsNone()) << "Failed to stop image manager: " << tests::utils::ErrorToStr(err); + + err = fs::RemoveAll(cTestDir); EXPECT_TRUE(err.IsNone()) << "Failed to remove test dir: " << tests::utils::ErrorToStr(err); } - Error InitImageManager(const String& imagePath = cTestImagePath) + Error InitImageManager(const String& imagePath = cTestImagePath, Duration removeOutdatedPeriod = 5 * Time::cSeconds) { - Config config {imagePath, 0, 10 * Time::cSeconds, 5 * Time::cSeconds}; + Config config {imagePath, 0, cItemCacheTTL, removeOutdatedPeriod}; return mImageManager.Init( config, mBlobInfoProvider, mStorage, mSpaceAllocator, mFileInfoProvider, mOCISpec, mFSPlatform, mAllocator); @@ -2672,4 +2677,302 @@ TEST_F(CommonImageManagerRemoveOutdatedTest, RemoveDeployableItemFails) EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); } +/*********************************************************************************************************************** + * Expired items suite + **********************************************************************************************************************/ + +class CommonImageManagerExpiredTest : public CommonImageManagerRemoveOutdatedTest { +protected: + void SetUp() override + { + CommonImageManagerRemoveOutdatedTest::SetUp(); + + ON_CALL(mSpaceAllocator, RestoreOutdatedItem(_, _)).WillByDefault(Return(ErrorEnum::eNone)); + } + + // Installs and removes the item, then makes it expired. + void AddExpiredItem(const TestManifest& manifest, const char* itemID = cTestItemID) + { + InstallAndRemove(manifest, itemID); + + auto item = GetStoredItem(itemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + + item->mTimestamp = Time::Now().Add(-2 * cItemCacheTTL); + + ASSERT_TRUE(mStorage.UpdateDeployableItem(*item).IsNone()); + } + + // Returns future which is set when the item is removed from the cache. + std::future ExpectRemovedFromCache(const char* itemID = cTestItemID) + { + auto removed = std::make_shared>(); + + EXPECT_CALL(mListener, OnItemRemovedFromCache(String(itemID), String(cTestVersion))) + .WillOnce(Invoke([removed](const String&, const String&) { removed->set_value(); })); + + return removed->get_future(); + } +}; + +TEST_F(CommonImageManagerExpiredTest, RemoveExpiredItemsOnStart) +{ + const auto expired = AddManifest(1, 1, 1); + const auto notExpired = AddManifest(200, 1, 1); + const auto installed = AddManifest(400, 1, 1); + + AddExpiredItem(expired); + InstallAndRemove(notExpired, "item2"); + + auto err = mImageManager.Install( + MakeInstallInfo(installed.mManifest, false, UpdateItemTypeEnum::eService, "item3", cTestVersion)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_CALL(mSpaceAllocator, RestoreOutdatedItem(String(cTestItemID), String(cTestVersion))) + .WillOnce(Return(ErrorEnum::eNone)); + EXPECT_CALL(mSpaceAllocator, RestoreOutdatedItem(String("item2"), _)).Times(0); + EXPECT_CALL(mSpaceAllocator, RestoreOutdatedItem(String("item3"), _)).Times(0); + EXPECT_CALL(mSpaceAllocator, FreeSpace(3 * cInstallBlobSize)); + + auto removed = ExpectRemovedFromCache(); + + ASSERT_TRUE(mImageManager.Start().IsNone()); + ASSERT_EQ(removed.wait_for(cWaitTimeout), std::future_status::ready); + + // The item is removed from the cache before the listener is notified, the space is freed after it. + ASSERT_TRUE(mImageManager.Stop().IsNone()); + + EXPECT_FALSE(GetStoredItem(cTestItemID, cTestVersion).has_value()); + EXPECT_FALSE(FileExists(expired.mManifest.mPath.c_str())); + EXPECT_TRUE(GetStoredItem("item2", cTestVersion).has_value()); + EXPECT_TRUE(GetStoredItem("item3", cTestVersion).has_value()); +} + +TEST_F(CommonImageManagerExpiredTest, RemoveExpiredItemsPeriodically) +{ + ASSERT_TRUE(InitImageManager(cTestImagePath, 100 * Time::cMilliseconds).IsNone()); + ASSERT_TRUE(mImageManager.Start().IsNone()); + + const auto manifest = AddManifest(1, 1, 1); + + auto removed = ExpectRemovedFromCache(); + + AddExpiredItem(manifest); + + ASSERT_EQ(removed.wait_for(cWaitTimeout), std::future_status::ready); + ASSERT_TRUE(mImageManager.Stop().IsNone()); + + EXPECT_FALSE(GetStoredItem(cTestItemID, cTestVersion).has_value()); +} + +TEST_F(CommonImageManagerExpiredTest, SkipItemInProgress) +{ + const auto manifestA = AddManifest(1, 1, 1); + const auto manifestB = AddManifest(200, 1, 1); + + auto err = mImageManager.Install(MakeInstallInfo(manifestA.mManifest)); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + InstallAndRemove(manifestB, "item2"); + + ASSERT_TRUE(InitImageManager(cTestImagePath, 100 * Time::cMilliseconds).IsNone()); + ASSERT_TRUE(mImageManager.Start().IsNone()); + + auto removedB = ExpectRemovedFromCache("item2"); + + // Item A may be removed in a later period, after RemoveItem returns. + EXPECT_CALL(mListener, OnItemRemovedFromCache(String(cTestItemID), String(cTestVersion))).Times(AnyNumber()); + + bool storedAOnRemove = false; + + // Item A is in progress while eRemoved is notified: both items are made expired, so the expired items removal skips + // item A and removes item B. + EXPECT_CALL(mListener, OnItemStatusChanged(_)).WillRepeatedly(Invoke([&](const UpdateItemStatus& status) { + if (status.mState != ItemStateEnum::eRemoved) { + return; + } + + for (const auto* itemID : {cTestItemID, "item2"}) { + auto item = GetStoredItem(itemID, cTestVersion); + EXPECT_TRUE(item.has_value()); + + item->mTimestamp = Time::Now().Add(-2 * cItemCacheTTL); + + EXPECT_TRUE(mStorage.UpdateDeployableItem(*item).IsNone()); + } + + // The removal notification of item B waits for this callback, so check the storage. + const auto deadline = std::chrono::steady_clock::now() + cWaitTimeout; + + while (GetStoredItem("item2", cTestVersion).has_value() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + storedAOnRemove = GetStoredItem(cTestItemID, cTestVersion).has_value(); + })); + + err = mImageManager.Remove(cTestItemID, cTestVersion); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + ASSERT_TRUE(mImageManager.Stop().IsNone()); + + EXPECT_EQ(removedB.wait_for(cWaitTimeout), std::future_status::ready); + EXPECT_TRUE(storedAOnRemove); + EXPECT_FALSE(GetStoredItem("item2", cTestVersion).has_value()); +} + +TEST_F(CommonImageManagerExpiredTest, RemoveExpiredItemFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + AddExpiredItem(manifest); + + const auto storedItem = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(storedItem.has_value()); + + mStorage.SetFailRemoveItem(true); + + auto added = std::make_shared>(); + + // The item stays in the eRemoved state, so it is added as outdated again. + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String(cTestItemID), String(cTestVersion), storedItem->mTimestamp)) + .WillOnce(Return(ErrorEnum::eNone)) + .WillOnce(Invoke([added](const String&, const String&, const Time&) { + added->set_value(); + + return ErrorEnum::eNone; + })); + EXPECT_CALL(mListener, OnItemRemovedFromCache(_, _)).Times(0); + EXPECT_CALL(mSpaceAllocator, FreeSpace(_)).Times(0); + + ASSERT_TRUE(mImageManager.Start().IsNone()); + ASSERT_EQ(added->get_future().wait_for(cWaitTimeout), std::future_status::ready); + ASSERT_TRUE(mImageManager.Stop().IsNone()); + + auto item = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(item.has_value()); + EXPECT_EQ(item->mState, ItemStateEnum::eRemoved); + EXPECT_TRUE(FileExists(manifest.mManifest.mPath.c_str())); +} + +TEST_F(CommonImageManagerExpiredTest, ReadExpiredItemFails) +{ + const auto manifest = AddManifest(1, 1, 1); + + AddExpiredItem(manifest); + + const auto storedItem = GetStoredItem(cTestItemID, cTestVersion); + ASSERT_TRUE(storedItem.has_value()); + + mStorage.SetFailGetItem(true); + + auto added = std::make_shared>(); + + // The item is restored before it is read, so it is added as outdated again with its stored timestamp. + EXPECT_CALL(mSpaceAllocator, AddOutdatedItem(String(cTestItemID), String(cTestVersion), storedItem->mTimestamp)) + .WillOnce(Return(ErrorEnum::eNone)) + .WillOnce(Invoke([added](const String&, const String&, const Time&) { + added->set_value(); + + return ErrorEnum::eNone; + })); + EXPECT_CALL(mListener, OnItemRemovedFromCache(_, _)).Times(0); + + ASSERT_TRUE(mImageManager.Start().IsNone()); + ASSERT_EQ(added->get_future().wait_for(cWaitTimeout), std::future_status::ready); + ASSERT_TRUE(mImageManager.Stop().IsNone()); + + EXPECT_TRUE(FileExists(manifest.mManifest.mPath.c_str())); +} + +TEST_F(CommonImageManagerExpiredTest, SkipItemRemovedAgain) +{ + const auto manifest = AddManifest(1, 1, 1); + + AddExpiredItem(manifest); + + Time newTimestamp; + std::promise restored; + + // The item is installed and removed again after the expired items are read, before the item is locked. + EXPECT_CALL(mSpaceAllocator, RestoreOutdatedItem(String(cTestItemID), String(cTestVersion))) + .WillOnce(Invoke([&](const String& itemID, const String& version) -> Error { + restored.set_value(); + + auto item = GetStoredItem(itemID, version); + EXPECT_TRUE(item.has_value()); + + item->mTimestamp = newTimestamp = Time::Now(); + + EXPECT_TRUE(mStorage.UpdateDeployableItem(*item).IsNone()); + + return ErrorEnum::eNone; + })); + EXPECT_CALL(mListener, OnItemRemovedFromCache(_, _)).Times(0); + EXPECT_CALL(mSpaceAllocator, FreeSpace(_)).Times(0); + + std::vector