Feature imagemanager

config/cm.cfg.in

@@ -12,7 +12,7 @@
     "fileServerUrl": "localhost:8094",
     "imageManager": {
         "downloadPath": "/var/aos/workdirs/cm/imagemanager/downloads",
-        "updateItemTTL": "30d",
+        "itemCacheTtl": "30d",
         "removeOutdatedPeriod": "24h"
     },
     "migration": {

src/cm/app/aoscore.cpp

@@ -80,11 +80,12 @@
     err = mAlerts.Init(mAllocator, mConfig.mAlerts, mCommunication, mCommunication);
     AOS_ERROR_CHECK_AND_THROW(err, "can't initialize alerts");
 
-    err = mDownloadSpaceAllocator.Init(mAllocator, mConfig.mImageManager.mDownloadPath, mPlatformFS, 0, &mImageManager);
+    err = mDownloadSpaceAllocator.Init(
+        mAllocator, mConfig.mImageManager.mDownloadPath, mPlatformFS, 0, &mImageManager.GetDownloadItemRemover());
     AOS_ERROR_CHECK_AND_THROW(err, "can't initialize download space allocator");
 
-    err = mInstallSpaceAllocator.Init(
-        mAllocator, mConfig.mImageManager.mImagePath, mPlatformFS, mConfig.mImageManager.mPartLimit, &mImageManager);
+    err = mInstallSpaceAllocator.Init(mAllocator, mConfig.mImageManager.mImagePath, mPlatformFS,
+        mConfig.mImageManager.mPartLimit, &mImageManager.GetItemRemover());
     AOS_ERROR_CHECK_AND_THROW(err, "can't initialize install space allocator");
 
     err = mDownloader.Init(&mAlerts);
@@ -94,8 +95,9 @@
         mIAMClient, mCertLoader, mCryptoProvider);
     AOS_ERROR_CHECK_AND_THROW(err, "can't initialize file server");
 
-    err = mImageManager.Init(mAllocator, mConfig.mImageManager, mDatabase, mCommunication, mDownloadSpaceAllocator,
-        mInstallSpaceAllocator, mDownloader, mFileServer, mCryptoHelper, mFileInfoProvider, mOCISpec);
+    err = mImageManager.Init(mConfig.mImageManager, mCommunication, mDatabase, mInstallSpaceAllocator,
+        mDownloadSpaceAllocator, mDownloader, mCryptoHelper, mFileServer, mFileInfoProvider, mOCISpec, mPlatformFS,
+        mAllocator);
     AOS_ERROR_CHECK_AND_THROW(err, "can't initialize image manager");
 
     err = mNodeInfoProvider.Init(mAllocator, mConfig.mNodeInfoProvider, mIAMClient);
@@ -114,7 +116,7 @@
     AOS_ERROR_CHECK_AND_THROW(err, "can't initialize launcher");
 
     err = mUpdateManager.Init(mAllocator, {mConfig.mUnitStatusSendTimeout}, mIAMClient, mIAMClient, mUnitConfig,
-        mNodeInfoProvider, mImageManager, mLauncher, mCommunication, mCommunication, mDatabase);
+        mNodeInfoProvider, mImageManager, mImageManager, mLauncher, mCommunication, mCommunication, mDatabase);
     AOS_ERROR_CHECK_AND_THROW(err, "can't initialize update manager");
 
     err = mRootCerts.Init(mAllocator, mIAMClient, mNodeInfoProvider, mCommunication, mCommunication);
@@ -128,7 +130,7 @@
     InitSMController();
 }
 
 void AosCore::Start()
 {
     LOG_INF() << "Start CM";
 

src/cm/communication/communication.cpp

@@ -349,13 +349,13 @@
     return ErrorEnum::eNone;
 }
 
-Error Communication::GetBlobsInfos(const Array<StaticString<oci::cDigestLen>>& digests, Array<BlobInfo>& blobsInfo)
+Error Communication::GetBlobInfo(const String& digest, BlobInfo& blobInfo)
 {
-    LOG_DBG() << "Get blobs info" << Log::Field("count", digests.Size());
+    LOG_DBG() << "Get blob info" << Log::Field("digest", digest);
 
     auto request = std::make_unique<BlobURLsRequest>();
 
-    if (auto err = request->mDigests.Assign(digests); !err.IsNone()) {
+    if (auto err = request->mDigests.EmplaceBack(digest); !err.IsNone()) {
         return AOS_ERROR_WRAP(err);
     }
 
@@ -383,9 +383,16 @@
     }
 
     if (const auto blobURLsInfo = std::get_if<BlobURLsInfo>(response.get()); blobURLsInfo) {
-        LOG_DBG() << "Received blob URLs info response";
+        LOG_DBG() << "Received blob URLs info response" << Log::Field("count", blobURLsInfo->mItems.Size());
 
-        return blobsInfo.Assign(blobURLsInfo->mItems);
+        auto item = blobURLsInfo->mItems.FindIf([&digest](const BlobInfo& info) { return info.mDigest == digest; });
+        if (item == blobURLsInfo->mItems.end()) {
+            return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "blob info not found"));
+        }
+
+        blobInfo = *item;
+
+        return ErrorEnum::eNone;
     }
 
     return AOS_ERROR_WRAP(Error(ErrorEnum::eFailed, "invalid response type"));
@@ -728,7 +735,7 @@
         if (mClientSession) {
             mClientSession->reset();
         }
     } catch (const std::exception& e) {
         err = common::utils::ToAosError(e);
     }
 

src/cm/communication/communication.hpp

@@ -115,13 +115,13 @@ class Communication : public cm::communication::CommunicationItf, private NonCop
     Error SendOverrideEnvsStatuses(const OverrideEnvVarsStatuses& statuses) override;
 
     /**
-     * Returns blobs info.
+     * Returns blob info.
      *
-     * @param digests list of blob digests.
-     * @param[out] blobsInfo blobs infos.
-     * @return Error.
+     * @param digest blob digest.
+     * @param[out] blobInfo blob info.
+     * @return Error: eNotFound if the cloud response doesn't contain the requested blob.
      */
-    Error GetBlobsInfos(const Array<StaticString<oci::cDigestLen>>& digests, Array<BlobInfo>& blobsInfo) override;
+    Error GetBlobInfo(const String& digest, BlobInfo& blobInfo) override;
 
     /**
      * Sends monitoring.

src/cm/communication/tests/communication.cpp

@@ -771,21 +771,18 @@ TEST_F(CMCommunicationTest, HandleDeprovisioning)
     ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err);
 }
 
-TEST_F(CMCommunicationTest, GetBlobsInfosTimeout)
+TEST_F(CMCommunicationTest, GetBlobInfoTimeout)
 {
     mConfig.mCloudResponseWaitTimeout = Time::cMilliseconds;
 
     SubscribeAndWaitConnected();
 
     mUUIDProvider.SetUUID("fb6e8461-2601-4f9a-8957-7ab4e52f304c");
 
-    StaticArray<StaticString<oci::cDigestLen>, 1> digests;
-    digests.EmplaceBack("sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b");
+    auto blobInfo = std::make_unique<BlobInfo>();
 
-    // cppcheck-suppress templateRecursion
-    auto blobsInfo = std::make_unique<StaticArray<BlobInfo, 1>>();
-
-    auto err = mCommunication.GetBlobsInfos(digests, *blobsInfo);
+    auto err = mCommunication.GetBlobInfo(
+        "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", *blobInfo);
     EXPECT_TRUE(err.Is(ErrorEnum::eTimeout)) << tests::utils::ErrorToStr(err);
 
     // make sure get blobs info message is not sent one more time after timeout
@@ -799,7 +796,7 @@ TEST_F(CMCommunicationTest, GetBlobsInfosTimeout)
     ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err);
 }
 
-TEST_F(CMCommunicationTest, GetBlobsInfos)
+TEST_F(CMCommunicationTest, GetBlobInfo)
 {
     constexpr auto cRequestCorrelationID = "2a05b9cc-32fb-41b6-a099-0fca3bb39ce2";
     constexpr auto cRequestTxnID         = "fb6e8461-2601-4f9a-8957-7ab4e52f304c";
@@ -866,13 +863,10 @@ TEST_F(CMCommunicationTest, GetBlobsInfos)
     mUUIDProvider.SetUUID(cRequestCorrelationID);
     mUUIDProvider.SetUUID(cRequestTxnID);
 
-    StaticArray<StaticString<oci::cDigestLen>, 1> digests;
-    digests.EmplaceBack(cDigest);
-
-    auto blobsInfo = std::make_unique<StaticArray<BlobInfo, 1>>();
+    auto blobInfo = std::make_unique<BlobInfo>();
 
-    auto future = std::async(std::launch::async,
-        [this, &digests, &blobsInfo]() { return mCommunication.GetBlobsInfos(digests, *blobsInfo); });
+    auto future = std::async(
+        std::launch::async, [this, &blobInfo]() { return mCommunication.GetBlobInfo(cDigest, *blobInfo); });
 
     const auto cUrlsRequest = mCloudReceivedMessages.Pop().value_or("");
     EXPECT_TRUE(std::regex_match(cUrlsRequest, cExpectedMessage)) << "Received message: " << cUrlsRequest;
@@ -883,11 +877,9 @@ TEST_F(CMCommunicationTest, GetBlobsInfos)
     auto err = future.get();
     EXPECT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err);
 
-    EXPECT_EQ(blobsInfo->Size(), 1);
-    const auto& blobInfo = (*blobsInfo)[0];
-
-    EXPECT_STREQ(blobInfo.mDigest.CStr(), cDigest);
-    EXPECT_EQ(blobInfo.mURLs.Size(), 2);
+    EXPECT_STREQ(blobInfo->mDigest.CStr(), cDigest);
+    EXPECT_EQ(blobInfo->mURLs.Size(), 2);
+    EXPECT_EQ(blobInfo->mSize, 1000);
 
     const auto cAck = mCloudReceivedMessages.Pop().value_or("");
     EXPECT_TRUE(std::regex_match(cAck, cExpectedAckMsg)) << "Received message: " << cAck;
@@ -896,6 +888,67 @@ TEST_F(CMCommunicationTest, GetBlobsInfos)
     ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err);
 }
 
+TEST_F(CMCommunicationTest, GetBlobInfoNotFound)
+{
+    constexpr auto cRequestCorrelationID = "2a05b9cc-32fb-41b6-a099-0fca3bb39ce2";
+    constexpr auto cRequestTxnID         = "fb6e8461-2601-4f9a-8957-7ab4e52f304c";
+    constexpr auto cDigest               = "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b";
+    constexpr auto cCloudAck             = R"({
+        "header": {
+            "version": 7,
+            "systemId": "test_system_id",
+            "txn": "fb6e8461-2601-4f9a-8957-7ab4e52f304c"
+        },
+        "data": {
+            "messageType": "ack"
+        }
+    })";
+    constexpr auto cResponse             = R"({
+        "header": {
+            "version": 7,
+            "systemId": "test_system_id",
+            "txn": "f2df3016-de31-46e6-9ce8-9cde4ed7849f"
+        },
+        "data": {
+            "messageType": "blobUrls",
+            "correlationId": "2a05b9cc-32fb-41b6-a099-0fca3bb39ce2",
+            "items": [
+                {
+                    "digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
+                    "urls": [
+                        "http://example.com/other.bin"
+                    ],
+                    "sha256": "36f028580bb02cc8272a9a020f4200e346e276ae664e45ee80745574e2f5ab80",
+                    "size": 1000
+                }
+            ]
+        }
+    })";
+
+    SubscribeAndWaitConnected();
+
+    mUUIDProvider.SetUUID(cRequestCorrelationID);
+    mUUIDProvider.SetUUID(cRequestTxnID);
+
+    auto blobInfo = std::make_unique<BlobInfo>();
+
+    auto future = std::async(
+        std::launch::async, [this, &blobInfo]() { return mCommunication.GetBlobInfo(cDigest, *blobInfo); });
+
+    EXPECT_TRUE(mCloudReceivedMessages.Pop().has_value()) << "Blob URLs request expected, but not received";
+
+    mCloudSendMessageQueue.Push(cCloudAck);
+    mCloudSendMessageQueue.Push(cResponse);
+
+    auto err = future.get();
+    EXPECT_TRUE(err.Is(ErrorEnum::eNotFound)) << tests::utils::ErrorToStr(err);
+
+    EXPECT_TRUE(mCloudReceivedMessages.Pop().has_value()) << "Ack message expected, but not received";
+
+    err = mCommunication.Stop();
+    ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err);
+}
+
 TEST_F(CMCommunicationTest, SendMonitoring)
 {
     const auto cExpectedMessage

src/cm/config/config.cpp

@@ -30,7 +30,7 @@ constexpr auto cDefaultLauncherNodesConnectionTimeout     = "10m";
 constexpr auto cDefaultMonitoringSendPeriod               = "1m";
 constexpr auto cDefaultSMConnectionTimeout                = "1m";
 constexpr auto cDefaultUnitStatusSendTimeout              = "10s";
-constexpr auto cDefaultUpdateItemTTL                      = "30d";
+constexpr auto cDefaultItemCacheTTL                       = "30d";
 constexpr auto cDefaultRemoveOutdatedPeriod               = "24h";
 constexpr auto cDefaultMigrationPath                      = "/usr/share/aos/communicationmanager/migration";
 constexpr auto cDefaultCertStorage                        = "/var/aos/crypt/cm/";
@@ -83,9 +83,9 @@ void ParseImageManagerConfig(const common::utils::CaseInsensitiveObjectWrapper&
         object.GetValue<std::string>("downloadPath", std::filesystem::path(workingDir) / "download").c_str());
     AOS_ERROR_CHECK_AND_THROW(err, "error parsing downloadPath tag");
 
-    Tie(config.mUpdateItemTTL, err)
-        = common::utils::ParseDuration(object.GetValue<std::string>("updateItemTtl", cDefaultUpdateItemTTL));
-    AOS_ERROR_CHECK_AND_THROW(err, "error parsing updateItemTtl tag");
+    Tie(config.mItemCacheTTL, err)
+        = common::utils::ParseDuration(object.GetValue<std::string>("itemCacheTtl", cDefaultItemCacheTTL));
+    AOS_ERROR_CHECK_AND_THROW(err, "error parsing itemCacheTtl tag");
 
     Tie(config.mRemoveOutdatedPeriod, err) = common::utils::ParseDuration(
         object.GetValue<std::string>("removeOutdatedPeriod", cDefaultRemoveOutdatedPeriod));

src/cm/config/tests/config.cpp

@@ -48,7 +48,7 @@ constexpr auto cFullTestConfigJSON = R"({
         "imagePath": "/path/to/images",
         "imagesPartLimit": 50,
         "downloadPath": "/path/to/download",
-        "updateItemTtl": "30d",
+        "itemCacheTtl": "15d",
         "removeOutdatedPeriod": "1h"
     },
     "launcher": {
@@ -135,7 +135,7 @@ TEST_F(CMConfigTest, ParseFullConfig)
     EXPECT_STREQ(config.mImageManager.mImagePath.CStr(), "/path/to/images");
     EXPECT_EQ(config.mImageManager.mPartLimit, 50);
     EXPECT_STREQ(config.mImageManager.mDownloadPath.CStr(), "/path/to/download");
-    EXPECT_EQ(config.mImageManager.mUpdateItemTTL, aos::Time::cDay * 30);
+    EXPECT_EQ(config.mImageManager.mItemCacheTTL, 15 * aos::Time::cDay);
     EXPECT_EQ(config.mImageManager.mRemoveOutdatedPeriod, aos::Time::cHours * 1);
 
     EXPECT_EQ(config.mLauncher.mNodesConnectionTimeout, aos::Time::cMinutes * 1);
@@ -201,7 +201,7 @@ TEST_F(CMConfigTest, ParseMinimalConfigWithDefaults)
     EXPECT_STREQ(config.mImageManager.mImagePath.CStr(), (std::filesystem::path("workingDir") / "images").c_str());
     EXPECT_STREQ(config.mImageManager.mDownloadPath.CStr(), (std::filesystem::path("workingDir") / "download").c_str());
     EXPECT_EQ(config.mImageManager.mPartLimit, 0);
-    EXPECT_EQ(config.mImageManager.mUpdateItemTTL, aos::Time::cDay * 30);
+    EXPECT_EQ(config.mImageManager.mItemCacheTTL, aos::Time::cDay * 30);
 
     EXPECT_EQ(config.mMigration.mMigrationPath, "/usr/share/aos/communicationmanager/migration");
     EXPECT_EQ(config.mMigration.mMergedMigrationPath, (std::filesystem::path("workingDir") / "migration").string());

src/cm/database/CMakeLists.txt

@@ -16,7 +16,9 @@ set(SOURCES database.cpp)
 # Libraries
 # ######################################################################################################################
 
-set(LIBRARIES aos::common::cloudprotocol aos::common::utils aos::common::migration Poco::DataSQLite Poco::JSON)
+set(LIBRARIES aos::common::cloudprotocol aos::common::database aos::common::utils aos::common::migration
+              Poco::DataSQLite Poco::JSON
+)
 
 # ######################################################################################################################
 # Target