diff --git a/config/cm.cfg.in b/config/cm.cfg.in index 9cf2595c3..0f92ec1bf 100644 --- a/config/cm.cfg.in +++ b/config/cm.cfg.in @@ -12,7 +12,7 @@ "fileServerUrl": "localhost:8094", "imageManager": { "downloadPath": "/var/aos/workdirs/cm/imagemanager/downloads", - "updateItemTTL": "30d", + "itemCacheTtl": "30d", "removeOutdatedPeriod": "24h" }, "migration": { diff --git a/src/cm/app/aoscore.cpp b/src/cm/app/aoscore.cpp index c85948336..84c7c31e0 100644 --- a/src/cm/app/aoscore.cpp +++ b/src/cm/app/aoscore.cpp @@ -80,11 +80,12 @@ void AosCore::Init(const std::string& configFile) err = mAlerts.Init(mAllocator, mConfig.mAlerts, mCommunication, mCommunication); AOS_ERROR_CHECK_AND_THROW(err, "can't initialize alerts"); - err = mDownloadSpaceAllocator.Init(mAllocator, mConfig.mImageManager.mDownloadPath, mPlatformFS, 0, &mImageManager); + err = mDownloadSpaceAllocator.Init( + mAllocator, mConfig.mImageManager.mDownloadPath, mPlatformFS, 0, &mImageManager.GetDownloadItemRemover()); AOS_ERROR_CHECK_AND_THROW(err, "can't initialize download space allocator"); - err = mInstallSpaceAllocator.Init( - mAllocator, mConfig.mImageManager.mImagePath, mPlatformFS, mConfig.mImageManager.mPartLimit, &mImageManager); + err = mInstallSpaceAllocator.Init(mAllocator, mConfig.mImageManager.mImagePath, mPlatformFS, + mConfig.mImageManager.mPartLimit, &mImageManager.GetItemRemover()); AOS_ERROR_CHECK_AND_THROW(err, "can't initialize install space allocator"); err = mDownloader.Init(&mAlerts); @@ -94,8 +95,9 @@ void AosCore::Init(const std::string& configFile) mIAMClient, mCertLoader, mCryptoProvider); AOS_ERROR_CHECK_AND_THROW(err, "can't initialize file server"); - err = mImageManager.Init(mAllocator, mConfig.mImageManager, mDatabase, mCommunication, mDownloadSpaceAllocator, - mInstallSpaceAllocator, mDownloader, mFileServer, mCryptoHelper, mFileInfoProvider, mOCISpec); + err = mImageManager.Init(mConfig.mImageManager, mCommunication, mDatabase, mInstallSpaceAllocator, + mDownloadSpaceAllocator, mDownloader, mCryptoHelper, mFileServer, mFileInfoProvider, mOCISpec, mPlatformFS, + mAllocator); AOS_ERROR_CHECK_AND_THROW(err, "can't initialize image manager"); err = mNodeInfoProvider.Init(mAllocator, mConfig.mNodeInfoProvider, mIAMClient); @@ -114,7 +116,7 @@ void AosCore::Init(const std::string& configFile) AOS_ERROR_CHECK_AND_THROW(err, "can't initialize launcher"); err = mUpdateManager.Init(mAllocator, {mConfig.mUnitStatusSendTimeout}, mIAMClient, mIAMClient, mUnitConfig, - mNodeInfoProvider, mImageManager, mLauncher, mCommunication, mCommunication, mDatabase); + mNodeInfoProvider, mImageManager, mImageManager, mLauncher, mCommunication, mCommunication, mDatabase); AOS_ERROR_CHECK_AND_THROW(err, "can't initialize update manager"); err = mRootCerts.Init(mAllocator, mIAMClient, mNodeInfoProvider, mCommunication, mCommunication); diff --git a/src/cm/communication/communication.cpp b/src/cm/communication/communication.cpp index 1afe8adac..e3f905934 100644 --- a/src/cm/communication/communication.cpp +++ b/src/cm/communication/communication.cpp @@ -349,13 +349,13 @@ Error Communication::SendOverrideEnvsStatuses(const OverrideEnvVarsStatuses& sta return ErrorEnum::eNone; } -Error Communication::GetBlobsInfos(const Array>& digests, Array& blobsInfo) +Error Communication::GetBlobInfo(const String& digest, BlobInfo& blobInfo) { - LOG_DBG() << "Get blobs info" << Log::Field("count", digests.Size()); + LOG_DBG() << "Get blob info" << Log::Field("digest", digest); auto request = std::make_unique(); - if (auto err = request->mDigests.Assign(digests); !err.IsNone()) { + if (auto err = request->mDigests.EmplaceBack(digest); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -383,9 +383,16 @@ Error Communication::GetBlobsInfos(const Array>& d } if (const auto blobURLsInfo = std::get_if(response.get()); blobURLsInfo) { - LOG_DBG() << "Received blob URLs info response"; + LOG_DBG() << "Received blob URLs info response" << Log::Field("count", blobURLsInfo->mItems.Size()); - return blobsInfo.Assign(blobURLsInfo->mItems); + auto item = blobURLsInfo->mItems.FindIf([&digest](const BlobInfo& info) { return info.mDigest == digest; }); + if (item == blobURLsInfo->mItems.end()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "blob info not found")); + } + + blobInfo = *item; + + return ErrorEnum::eNone; } return AOS_ERROR_WRAP(Error(ErrorEnum::eFailed, "invalid response type")); diff --git a/src/cm/communication/communication.hpp b/src/cm/communication/communication.hpp index 077f5cda5..ac089508c 100644 --- a/src/cm/communication/communication.hpp +++ b/src/cm/communication/communication.hpp @@ -115,13 +115,13 @@ class Communication : public cm::communication::CommunicationItf, private NonCop Error SendOverrideEnvsStatuses(const OverrideEnvVarsStatuses& statuses) override; /** - * Returns blobs info. + * Returns blob info. * - * @param digests list of blob digests. - * @param[out] blobsInfo blobs infos. - * @return Error. + * @param digest blob digest. + * @param[out] blobInfo blob info. + * @return Error: eNotFound if the cloud response doesn't contain the requested blob. */ - Error GetBlobsInfos(const Array>& digests, Array& blobsInfo) override; + Error GetBlobInfo(const String& digest, BlobInfo& blobInfo) override; /** * Sends monitoring. diff --git a/src/cm/communication/tests/communication.cpp b/src/cm/communication/tests/communication.cpp index bc8e90f82..47672e0f6 100644 --- a/src/cm/communication/tests/communication.cpp +++ b/src/cm/communication/tests/communication.cpp @@ -771,7 +771,7 @@ TEST_F(CMCommunicationTest, HandleDeprovisioning) ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); } -TEST_F(CMCommunicationTest, GetBlobsInfosTimeout) +TEST_F(CMCommunicationTest, GetBlobInfoTimeout) { mConfig.mCloudResponseWaitTimeout = Time::cMilliseconds; @@ -779,13 +779,10 @@ TEST_F(CMCommunicationTest, GetBlobsInfosTimeout) mUUIDProvider.SetUUID("fb6e8461-2601-4f9a-8957-7ab4e52f304c"); - StaticArray, 1> digests; - digests.EmplaceBack("sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"); + auto blobInfo = std::make_unique(); - // cppcheck-suppress templateRecursion - auto blobsInfo = std::make_unique>(); - - auto err = mCommunication.GetBlobsInfos(digests, *blobsInfo); + auto err = mCommunication.GetBlobInfo( + "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", *blobInfo); EXPECT_TRUE(err.Is(ErrorEnum::eTimeout)) << tests::utils::ErrorToStr(err); // make sure get blobs info message is not sent one more time after timeout @@ -799,7 +796,7 @@ TEST_F(CMCommunicationTest, GetBlobsInfosTimeout) ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); } -TEST_F(CMCommunicationTest, GetBlobsInfos) +TEST_F(CMCommunicationTest, GetBlobInfo) { constexpr auto cRequestCorrelationID = "2a05b9cc-32fb-41b6-a099-0fca3bb39ce2"; constexpr auto cRequestTxnID = "fb6e8461-2601-4f9a-8957-7ab4e52f304c"; @@ -866,13 +863,10 @@ TEST_F(CMCommunicationTest, GetBlobsInfos) mUUIDProvider.SetUUID(cRequestCorrelationID); mUUIDProvider.SetUUID(cRequestTxnID); - StaticArray, 1> digests; - digests.EmplaceBack(cDigest); - - auto blobsInfo = std::make_unique>(); + auto blobInfo = std::make_unique(); - auto future = std::async(std::launch::async, - [this, &digests, &blobsInfo]() { return mCommunication.GetBlobsInfos(digests, *blobsInfo); }); + auto future = std::async( + std::launch::async, [this, &blobInfo]() { return mCommunication.GetBlobInfo(cDigest, *blobInfo); }); const auto cUrlsRequest = mCloudReceivedMessages.Pop().value_or(""); EXPECT_TRUE(std::regex_match(cUrlsRequest, cExpectedMessage)) << "Received message: " << cUrlsRequest; @@ -883,11 +877,9 @@ TEST_F(CMCommunicationTest, GetBlobsInfos) auto err = future.get(); EXPECT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); - EXPECT_EQ(blobsInfo->Size(), 1); - const auto& blobInfo = (*blobsInfo)[0]; - - EXPECT_STREQ(blobInfo.mDigest.CStr(), cDigest); - EXPECT_EQ(blobInfo.mURLs.Size(), 2); + EXPECT_STREQ(blobInfo->mDigest.CStr(), cDigest); + EXPECT_EQ(blobInfo->mURLs.Size(), 2); + EXPECT_EQ(blobInfo->mSize, 1000); const auto cAck = mCloudReceivedMessages.Pop().value_or(""); EXPECT_TRUE(std::regex_match(cAck, cExpectedAckMsg)) << "Received message: " << cAck; @@ -896,6 +888,67 @@ TEST_F(CMCommunicationTest, GetBlobsInfos) ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); } +TEST_F(CMCommunicationTest, GetBlobInfoNotFound) +{ + constexpr auto cRequestCorrelationID = "2a05b9cc-32fb-41b6-a099-0fca3bb39ce2"; + constexpr auto cRequestTxnID = "fb6e8461-2601-4f9a-8957-7ab4e52f304c"; + constexpr auto cDigest = "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"; + constexpr auto cCloudAck = R"({ + "header": { + "version": 7, + "systemId": "test_system_id", + "txn": "fb6e8461-2601-4f9a-8957-7ab4e52f304c" + }, + "data": { + "messageType": "ack" + } + })"; + constexpr auto cResponse = R"({ + "header": { + "version": 7, + "systemId": "test_system_id", + "txn": "f2df3016-de31-46e6-9ce8-9cde4ed7849f" + }, + "data": { + "messageType": "blobUrls", + "correlationId": "2a05b9cc-32fb-41b6-a099-0fca3bb39ce2", + "items": [ + { + "digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "urls": [ + "http://example.com/other.bin" + ], + "sha256": "36f028580bb02cc8272a9a020f4200e346e276ae664e45ee80745574e2f5ab80", + "size": 1000 + } + ] + } + })"; + + SubscribeAndWaitConnected(); + + mUUIDProvider.SetUUID(cRequestCorrelationID); + mUUIDProvider.SetUUID(cRequestTxnID); + + auto blobInfo = std::make_unique(); + + auto future = std::async( + std::launch::async, [this, &blobInfo]() { return mCommunication.GetBlobInfo(cDigest, *blobInfo); }); + + EXPECT_TRUE(mCloudReceivedMessages.Pop().has_value()) << "Blob URLs request expected, but not received"; + + mCloudSendMessageQueue.Push(cCloudAck); + mCloudSendMessageQueue.Push(cResponse); + + auto err = future.get(); + EXPECT_TRUE(err.Is(ErrorEnum::eNotFound)) << tests::utils::ErrorToStr(err); + + EXPECT_TRUE(mCloudReceivedMessages.Pop().has_value()) << "Ack message expected, but not received"; + + err = mCommunication.Stop(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + TEST_F(CMCommunicationTest, SendMonitoring) { const auto cExpectedMessage diff --git a/src/cm/config/config.cpp b/src/cm/config/config.cpp index 88245cc08..8e56c1b54 100644 --- a/src/cm/config/config.cpp +++ b/src/cm/config/config.cpp @@ -30,7 +30,7 @@ constexpr auto cDefaultLauncherNodesConnectionTimeout = "10m"; constexpr auto cDefaultMonitoringSendPeriod = "1m"; constexpr auto cDefaultSMConnectionTimeout = "1m"; constexpr auto cDefaultUnitStatusSendTimeout = "10s"; -constexpr auto cDefaultUpdateItemTTL = "30d"; +constexpr auto cDefaultItemCacheTTL = "30d"; constexpr auto cDefaultRemoveOutdatedPeriod = "24h"; constexpr auto cDefaultMigrationPath = "/usr/share/aos/communicationmanager/migration"; constexpr auto cDefaultCertStorage = "/var/aos/crypt/cm/"; @@ -83,9 +83,9 @@ void ParseImageManagerConfig(const common::utils::CaseInsensitiveObjectWrapper& object.GetValue("downloadPath", std::filesystem::path(workingDir) / "download").c_str()); AOS_ERROR_CHECK_AND_THROW(err, "error parsing downloadPath tag"); - Tie(config.mUpdateItemTTL, err) - = common::utils::ParseDuration(object.GetValue("updateItemTtl", cDefaultUpdateItemTTL)); - AOS_ERROR_CHECK_AND_THROW(err, "error parsing updateItemTtl tag"); + Tie(config.mItemCacheTTL, err) + = common::utils::ParseDuration(object.GetValue("itemCacheTtl", cDefaultItemCacheTTL)); + AOS_ERROR_CHECK_AND_THROW(err, "error parsing itemCacheTtl tag"); Tie(config.mRemoveOutdatedPeriod, err) = common::utils::ParseDuration( object.GetValue("removeOutdatedPeriod", cDefaultRemoveOutdatedPeriod)); diff --git a/src/cm/config/tests/config.cpp b/src/cm/config/tests/config.cpp index dd7609703..cd5aae7f1 100644 --- a/src/cm/config/tests/config.cpp +++ b/src/cm/config/tests/config.cpp @@ -48,7 +48,7 @@ constexpr auto cFullTestConfigJSON = R"({ "imagePath": "/path/to/images", "imagesPartLimit": 50, "downloadPath": "/path/to/download", - "updateItemTtl": "30d", + "itemCacheTtl": "15d", "removeOutdatedPeriod": "1h" }, "launcher": { @@ -135,7 +135,7 @@ TEST_F(CMConfigTest, ParseFullConfig) EXPECT_STREQ(config.mImageManager.mImagePath.CStr(), "/path/to/images"); EXPECT_EQ(config.mImageManager.mPartLimit, 50); EXPECT_STREQ(config.mImageManager.mDownloadPath.CStr(), "/path/to/download"); - EXPECT_EQ(config.mImageManager.mUpdateItemTTL, aos::Time::cDay * 30); + EXPECT_EQ(config.mImageManager.mItemCacheTTL, 15 * aos::Time::cDay); EXPECT_EQ(config.mImageManager.mRemoveOutdatedPeriod, aos::Time::cHours * 1); EXPECT_EQ(config.mLauncher.mNodesConnectionTimeout, aos::Time::cMinutes * 1); @@ -201,7 +201,7 @@ TEST_F(CMConfigTest, ParseMinimalConfigWithDefaults) EXPECT_STREQ(config.mImageManager.mImagePath.CStr(), (std::filesystem::path("workingDir") / "images").c_str()); EXPECT_STREQ(config.mImageManager.mDownloadPath.CStr(), (std::filesystem::path("workingDir") / "download").c_str()); EXPECT_EQ(config.mImageManager.mPartLimit, 0); - EXPECT_EQ(config.mImageManager.mUpdateItemTTL, aos::Time::cDay * 30); + EXPECT_EQ(config.mImageManager.mItemCacheTTL, aos::Time::cDay * 30); EXPECT_EQ(config.mMigration.mMigrationPath, "/usr/share/aos/communicationmanager/migration"); EXPECT_EQ(config.mMigration.mMergedMigrationPath, (std::filesystem::path("workingDir") / "migration").string()); diff --git a/src/cm/database/CMakeLists.txt b/src/cm/database/CMakeLists.txt index aca420fb0..ee7a254f8 100644 --- a/src/cm/database/CMakeLists.txt +++ b/src/cm/database/CMakeLists.txt @@ -16,7 +16,9 @@ set(SOURCES database.cpp) # Libraries # ###################################################################################################################### -set(LIBRARIES aos::common::cloudprotocol aos::common::utils aos::common::migration Poco::DataSQLite Poco::JSON) +set(LIBRARIES aos::common::cloudprotocol aos::common::database aos::common::utils aos::common::migration + Poco::DataSQLite Poco::JSON +) # ###################################################################################################################### # Target diff --git a/src/cm/database/database.cpp b/src/cm/database/database.cpp index 5013a23df..0da042038 100644 --- a/src/cm/database/database.cpp +++ b/src/cm/database/database.cpp @@ -737,9 +737,9 @@ Error Database::AddInstance(const launcher::InstanceInfo& info) FromAos(info, row); *mSession << "INSERT INTO launcher_instances (itemID, subjectID, instance, type, preinstalled, manifestDigest, " - "nodeID, prevNodeID, runtimeID, uid, gid, timestamp, state, isUnitSubject, version, ownerID, " - "subjectType, labels, priority, disableRebalancing) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", + "nodeID, prevNodeID, runtimeID, uid, gid, timestamp, state, isUnitSubject, version, indexDigest, " + "ownerID, subjectType, labels, priority, disableRebalancing) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", bind(row), now; } catch (const std::exception& e) { return AOS_ERROR_WRAP(common::utils::ToAosError(e)); @@ -757,15 +757,15 @@ Error Database::UpdateInstance(const launcher::InstanceInfo& info) statement << "UPDATE launcher_instances SET manifestDigest = ?, nodeID = ?, prevNodeID = ?, runtimeID = ?, uid = ?, " - "gid = ?, timestamp = ?, state = ?, isUnitSubject = ?, ownerID = ?, subjectType = ?, labels = ?, " - "priority = ?, disableRebalancing = ? " + "gid = ?, timestamp = ?, state = ?, isUnitSubject = ?, indexDigest = ?, ownerID = ?, subjectType = ?, " + "labels = ?, priority = ?, disableRebalancing = ? " "WHERE itemID = ? AND subjectID = ? AND instance = ? " "AND type = ? AND preinstalled = ? AND version = ?;", bind(info.mManifestDigest.CStr()), bind(info.mNodeID.CStr()), bind(info.mPrevNodeID.CStr()), bind(info.mRuntimeID.CStr()), bind(info.mUID), bind(info.mGID), bind(info.mTimestamp.UnixNano()), - bind(info.mState.ToString().CStr()), bind(info.mIsUnitSubject), bind(info.mOwnerID.CStr()), - bind(info.mSubjectType.ToString().CStr()), bind(SerializeLabels(info.mLabels)), bind(info.mPriority), - bind(info.mDisableRebalancing), bind(info.mInstanceIdent.mItemID.CStr()), + bind(info.mState.ToString().CStr()), bind(info.mIsUnitSubject), bind(info.mIndexDigest.CStr()), + bind(info.mOwnerID.CStr()), bind(info.mSubjectType.ToString().CStr()), bind(SerializeLabels(info.mLabels)), + bind(info.mPriority), bind(info.mDisableRebalancing), bind(info.mInstanceIdent.mItemID.CStr()), bind(info.mInstanceIdent.mSubjectID.CStr()), bind(info.mInstanceIdent.mInstance), bind(info.mInstanceIdent.mType.ToString().CStr()), bind(info.mInstanceIdent.mPreinstalled), bind(info.mVersion.CStr()); @@ -787,10 +787,11 @@ Error Database::LoadActiveInstances(Array& instances) co try { std::vector rows; - *mSession << "SELECT itemID, subjectID, instance, type, preinstalled, manifestDigest, nodeID, prevNodeID, " - "runtimeID, uid, gid, timestamp, state, isUnitSubject, version, ownerID, subjectType, labels, " - "priority, disableRebalancing " - "FROM launcher_instances;", + *mSession + << "SELECT itemID, subjectID, instance, type, preinstalled, manifestDigest, nodeID, prevNodeID, " + "runtimeID, uid, gid, timestamp, state, isUnitSubject, version, indexDigest, ownerID, subjectType, " + "labels, priority, disableRebalancing " + "FROM launcher_instances;", into(rows), now; auto instanceInfo = std::make_unique(); @@ -889,8 +890,8 @@ Error Database::LoadRunRequests(Array& requests) c try { std::vector rows; - *mSession << "SELECT itemID, type, version, ownerID, subjectID, subjectType, isUnitSubject, priority, " - "numInstances, labels FROM launcher_run_requests ORDER BY rowid;", + *mSession << "SELECT itemID, type, version, indexDigest, ownerID, subjectID, subjectType, isUnitSubject, " + "priority, numInstances, labels FROM launcher_run_requests ORDER BY rowid;", into(rows), now; requests.Clear(); @@ -921,8 +922,9 @@ Error Database::SaveRunRequests(const Array& reque LauncherRunRequestRow row; FromAos(request, row); - *mSession << "INSERT INTO launcher_run_requests (itemID, type, version, ownerID, subjectID, subjectType, " - "isUnitSubject, priority, numInstances, labels) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", + *mSession << "INSERT INTO launcher_run_requests (itemID, type, version, indexDigest, ownerID, subjectID, " + "subjectType, isUnitSubject, priority, numInstances, labels) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", bind(row), now; } transaction.commit(); @@ -934,118 +936,9 @@ Error Database::SaveRunRequests(const Array& reque } /*********************************************************************************************************************** - * imagemanager::StorageItf implementation + * updatemanager::StorageItf implementation **********************************************************************************************************************/ -Error Database::AddItem(const imagemanager::ItemInfo& item) -{ - std::lock_guard lock {mMutex}; - - try { - ImageManagerItemInfoRow row; - - FromAos(item, row); - *mSession << "INSERT INTO imagemanager (itemID, type, version, indexDigest, state, timestamp) VALUES " - "(?, ?, ?, ?, ?, ?);", - bind(row), now; - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - -Error Database::RemoveItem(const String& id, const String& version) -{ - std::lock_guard lock {mMutex}; - - try { - Poco::Data::Statement statement {*mSession}; - - statement << "DELETE FROM imagemanager WHERE itemID = ? AND version = ?;", bind(id.CStr()), - bind(version.CStr()); - - if (statement.execute() != 1) { - return ErrorEnum::eNotFound; - } - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - -Error Database::UpdateItemState(const String& id, const String& version, ItemState state, Time timestamp) -{ - std::lock_guard lock {mMutex}; - - try { - Poco::Data::Statement statement {*mSession}; - - statement << "UPDATE imagemanager SET state = ?, timestamp = ? WHERE itemID = ? AND version = ?;", - bind(state.ToString().CStr()), bind(timestamp.UnixNano()), bind(id.CStr()), bind(version.CStr()); - - if (statement.execute() != 1) { - return ErrorEnum::eNotFound; - } - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - -Error Database::GetAllItemsInfos(Array& items) -{ - std::lock_guard lock {mMutex}; - - try { - std::vector rows; - - *mSession << "SELECT itemID, type, version, indexDigest, state, timestamp FROM imagemanager;", into(rows), now; - - auto itemInfo = std::make_unique(); - - items.Clear(); - - for (const auto& row : rows) { - ToAos(row, *itemInfo); - auto err = items.PushBack(*itemInfo); - AOS_ERROR_CHECK_AND_THROW(AOS_ERROR_WRAP(err), "can't add item info"); - } - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - -Error Database::GetItemInfos(const String& id, Array& items) -{ - std::lock_guard lock {mMutex}; - - try { - std::vector rows; - - *mSession << "SELECT itemID, type, version, indexDigest, state, timestamp FROM imagemanager WHERE itemID = ?;", - bind(id.CStr()), into(rows), now; - - auto itemInfo = std::make_unique(); - - items.Clear(); - - for (const auto& row : rows) { - ToAos(row, *itemInfo); - auto err = items.PushBack(*itemInfo); - AOS_ERROR_CHECK_AND_THROW(AOS_ERROR_WRAP(err), "can't add item info"); - } - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - Error Database::StoreDesiredStatus(const DesiredStatus& desiredStatus) { std::lock_guard lock {mMutex}; @@ -1171,17 +1064,6 @@ void Database::CreateTables() ");", now; - *mSession << "CREATE TABLE IF NOT EXISTS imagemanager (" - "itemID TEXT," - "type TEXT," - "version TEXT," - "indexDigest TEXT," - "state TEXT," - "timestamp INTEGER," - "PRIMARY KEY(itemID,version)" - ");", - now; - *mSession << "CREATE TABLE IF NOT EXISTS networks (" "networkID TEXT," "subnet TEXT," @@ -1420,6 +1302,7 @@ void Database::FromAos(const launcher::InstanceInfo& src, LauncherInstanceInfoRo dst.set(src.mState.ToString().CStr()); dst.set(src.mIsUnitSubject); dst.set(src.mVersion.CStr()); + dst.set(src.mIndexDigest.CStr()); dst.set(src.mOwnerID.CStr()); dst.set(src.mSubjectType.ToString().CStr()); dst.set(SerializeLabels(src.mLabels)); @@ -1454,6 +1337,7 @@ void Database::ToAos(const LauncherInstanceInfoRow& src, launcher::InstanceInfo& dst.mIsUnitSubject = src.get(); dst.mVersion = src.get().c_str(); + dst.mIndexDigest = src.get().c_str(); dst.mOwnerID = src.get().c_str(); const auto& subjectTypeStr = src.get(); @@ -1466,29 +1350,6 @@ void Database::ToAos(const LauncherInstanceInfoRow& src, launcher::InstanceInfo& dst.mDisableRebalancing = src.get(); } -void Database::FromAos(const imagemanager::ItemInfo& src, ImageManagerItemInfoRow& dst) -{ - dst.set(src.mItemID.CStr()); - dst.set(src.mVersion.CStr()); - dst.set(src.mType.ToString().CStr()); - dst.set(src.mIndexDigest.CStr()); - dst.set(src.mState.ToString().CStr()); - dst.set(src.mTimestamp.UnixNano()); -} - -void Database::ToAos(const ImageManagerItemInfoRow& src, imagemanager::ItemInfo& dst) -{ - dst.mItemID = src.get().c_str(); - auto err = dst.mType.FromString(src.get().c_str()); - AOS_ERROR_CHECK_AND_THROW(err, "failed to parse item type"); - dst.mVersion = src.get().c_str(); - dst.mIndexDigest = src.get().c_str(); - err = dst.mState.FromString(src.get().c_str()); - AOS_ERROR_CHECK_AND_THROW(err, "failed to parse item state"); - auto timestamp = src.get(); - dst.mTimestamp = Time::Unix(timestamp / Time::cSeconds.Nanoseconds(), timestamp % Time::cSeconds.Nanoseconds()); -} - void Database::FromAos(const EnvVarsInstanceInfo& src, LauncherOverrideEnvVarsRow& dst) { std::string itemID; @@ -1551,6 +1412,7 @@ void Database::FromAos(const launcher::RunInstanceRequest& src, LauncherRunReque dst.set(src.mItemID.CStr()); dst.set(src.mUpdateItemType.ToString().CStr()); dst.set(src.mVersion.CStr()); + dst.set(src.mIndexDigest.CStr()); dst.set(src.mOwnerID.CStr()); dst.set(src.mSubjectInfo.mSubjectID.CStr()); dst.set(src.mSubjectInfo.mSubjectType.ToString().CStr()); @@ -1564,6 +1426,7 @@ void Database::ToAos(const LauncherRunRequestRow& src, launcher::RunInstanceRequ { dst.mItemID = src.get().c_str(); dst.mVersion = src.get().c_str(); + dst.mIndexDigest = src.get().c_str(); dst.mOwnerID = src.get().c_str(); dst.mPriority = src.get(); dst.mNumInstances = src.get(); diff --git a/src/cm/database/database.hpp b/src/cm/database/database.hpp index 26af9e845..70c37186c 100644 --- a/src/cm/database/database.hpp +++ b/src/cm/database/database.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -27,7 +28,7 @@ namespace aos::cm::database { /** * Database class. */ -class Database : public DatabaseItf, public networkmanager::StorageItf { +class Database : public DatabaseItf, public common::database::Database, public networkmanager::StorageItf { public: /** * Creates database instance. @@ -37,7 +38,7 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { /** * Destructor. */ - ~Database(); + ~Database() override; /** * Initializes database. @@ -296,54 +297,9 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { Error SaveRunRequests(const Array& requests) override; // - // imagemanager::StorageItf interface + // updatemanager::StorageItf interface // - /** - * Adds item. - * - * @param item Item info. - * @return Error. - */ - Error AddItem(const imagemanager::ItemInfo& item) override; - - /** - * Removes item. - * - * @param id ID. - * @param version Version. - * @return Error. - */ - Error RemoveItem(const String& id, const String& version) override; - - /** - * Updates item state. - * - * @param id ID. - * @param version Version. - * @param state Item state. - * @param timestamp Timestamp. - * @return Error. - */ - Error UpdateItemState(const String& id, const String& version, ItemState state, Time timestamp = {}) override; - - /** - * Gets all items info. - * - * @param items Items info. - * @return Error. - */ - Error GetAllItemsInfos(Array& items) override; - - /** - * Gets items info by ID. - * - * @param id Item ID. - * @param items Items info. - * @return Error. - */ - Error GetItemInfos(const String& id, Array& items) override; - /** * Stores desired status in storage. * @@ -352,10 +308,6 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { */ Error StoreDesiredStatus(const DesiredStatus& desiredStatus) override; - // - // updatemanager::StorageItf interface - // - /** * Stores update state in storage. * @@ -380,7 +332,7 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { RetWithError GetUpdateState() override; private: - static constexpr int cVersion = 1; + static constexpr int cVersion = 2; static constexpr auto cDBFileName = "cm.db"; enum class StorageStateInstanceInfoColumns : int { @@ -451,6 +403,7 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { eState, eIsUnitSubject, eVersion, + eIndexDigest, eOwnerID, eSubjectType, eLabels, @@ -459,18 +412,7 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { }; using LauncherInstanceInfoRow = Poco::Tuple; - - enum class ImageManagerItemInfoColumns : int { - eItemID = 0, - eType, - eVersion, - eIndexDigest, - eState, - eTimestamp, - }; - using ImageManagerItemInfoRow - = Poco::Tuple; + std::string, std::string, std::string, std::string, size_t, bool>; enum class LauncherOverrideEnvVarsColumns : int { eItemID = 0, eSubjectID, eInstance, eVariables }; using LauncherOverrideEnvVarsRow = Poco::Tuple; @@ -479,6 +421,7 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { eItemID = 0, eType, eVersion, + eIndexDigest, eOwnerID, eSubjectID, eSubjectType, @@ -488,7 +431,7 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { eLabels }; using LauncherRunRequestRow = Poco::Tuple; + std::string, std::string, bool, size_t, size_t, std::string>; // make virtual for unit tests virtual int GetVersion() const; @@ -512,18 +455,13 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { static void FromAos(const launcher::InstanceInfo& src, LauncherInstanceInfoRow& dst); static void ToAos(const LauncherInstanceInfoRow& src, launcher::InstanceInfo& dst); - static void FromAos(const imagemanager::ItemInfo& src, ImageManagerItemInfoRow& dst); - static void ToAos(const ImageManagerItemInfoRow& src, imagemanager::ItemInfo& dst); - static void FromAos(const EnvVarsInstanceInfo& src, LauncherOverrideEnvVarsRow& dst); static void ToAos(const LauncherOverrideEnvVarsRow& src, EnvVarsInstanceInfo& dst); static void FromAos(const launcher::RunInstanceRequest& src, LauncherRunRequestRow& dst); static void ToAos(const LauncherRunRequestRow& src, launcher::RunInstanceRequest& dst); - std::unique_ptr mSession; std::optional mDatabase; - mutable std::mutex mMutex; }; } // namespace aos::cm::database diff --git a/src/cm/database/database.md b/src/cm/database/database.md index 0ce8f0bee..fe7a1053c 100644 --- a/src/cm/database/database.md +++ b/src/cm/database/database.md @@ -8,7 +8,8 @@ storage interfaces using SQLite as the backend database. The `Database` class implements the following interfaces to persist data for different CM modules: - [aos::cm::storagestate::StorageItf][storagestate-storage-itf] - storage state interface; -- [aos::cm::imagemanager::StorageItf][iamgemanager-storage-itf] - image manager interface; +- [aos::common::imagemanager::StorageItf][imagemanager-storage-itf] - image manager interface, implemented by the + [common database](../../common/database/database.md) base class `aos::common::database::Database`; - [aos::cm::networkmanager::StorageItf](../networkmanager/itf/storage.hpp) - network manager interface; - [aos::cm::launcher::StorageItf][launcher-storage-itf] - launcher interface. @@ -17,10 +18,14 @@ The `Database` class implements the following interfaces to persist data for dif ```mermaid classDiagram class Database { - -Session mSession -Migration mDatabase +Init(config) Error } + + class CommonDatabase["aos::common::database::Database"] { + #Session mSession + #mutex mMutex + } class StorageStateItf { <> @@ -33,11 +38,15 @@ classDiagram class ImageManagerStorageItf { <> - +AddItem() - +SetItemState() - +RemoveItem() - +GetItemsInfo() - +GetItemVersionsByID() + +AddDeployableItem() + +UpdateDeployableItem() + +RemoveDeployableItem() + +GetDeployableItem() + +GetAllDeployableItems() + +GetDeployableItemsCount() + +AddItemBlob() + +GetItemBlobs() + +IsBlobUsed() } class NetworkManagerStorageItf { @@ -62,8 +71,9 @@ classDiagram +GetActiveInstances() } + Database --|> CommonDatabase + CommonDatabase ..|> ImageManagerStorageItf Database ..|> StorageStateItf - Database ..|> ImageManagerStorageItf Database ..|> NetworkManagerStorageItf Database ..|> LauncherStorageItf ``` @@ -87,25 +97,17 @@ Stores instance storage and state quota information. **Primary Key**: `(itemID, subjectID, instance)` -#### 2. `imagemanager` +#### 2. `imagemanager_items` -Stores image manager item information and metadata. +Stores image manager deployable items. See [common database](../../common/database/database.md) for the columns +description. -| Column | Type | Description | -|-------------|---------|----------------------------------- | -| id | TEXT | Item identifier | -| type | TEXT | Item type (service/layer/component)| -| version | TEXT | Item version | -| state | TEXT | Item state | -| path | TEXT | File system path | -| totalSize | INTEGER | Total size in bytes | -| gid | INTEGER | Group ID | -| timestamp | INTEGER | Timestamp in nanoseconds | -| images | TEXT | JSON array of image metadata | +#### 3. `imagemanager_blobs` -**Primary Key**: `(id, version)` +Stores blobs referenced by deployable items. See [common database](../../common/database/database.md) for the columns +description. -#### 3. `networks` +#### 4. `networks` Stores network definitions. @@ -117,7 +119,7 @@ Stores network definitions. **Primary Key**: `(networkID)` -#### 4. `hosts` +#### 5. `hosts` Stores host information within networks. @@ -129,7 +131,7 @@ Stores host information within networks. **Primary Key**: `(networkID, nodeID)` -#### 5. `networkmanager_instances` +#### 6. `networkmanager_instances` Stores network instance configurations. @@ -146,7 +148,7 @@ Stores network instance configurations. **Primary Key**: `(itemID, subjectID, instance)` -#### 6. `launcher_instances` +#### 7. `launcher_instances` Stores launcher instance information. @@ -163,9 +165,28 @@ Stores launcher instance information. | uid | INTEGER | User ID | | timestamp | INTEGER | Timestamp in nanoseconds | | cached | INTEGER | Cached indicator (0/1) | +| indexDigest | TEXT | Item index digest | **Primary Key**: `(itemID, subjectID, instance)` +#### 8. `launcher_run_requests` + +Stores launcher run instance requests. + +| Column | Type | Description | +|---------------|---------|------------------------------| +| itemID | TEXT | Service/component ID | +| type | TEXT | Update item type | +| version | TEXT | Item version | +| ownerID | TEXT | Owner identifier | +| subjectID | TEXT | Subject identifier | +| subjectType | TEXT | Subject type | +| isUnitSubject | INTEGER | Unit subject indicator (0/1) | +| priority | INTEGER | Request priority | +| numInstances | INTEGER | Number of instances | +| labels | TEXT | JSON array of labels | +| indexDigest | TEXT | Item index digest | + ## Data Conversion The module uses a consistent pattern for converting between AOS types and database rows: @@ -216,6 +237,15 @@ Helper functions in the anonymous namespace handle serialization: The database uses the common migration framework to support schema evolution across versions. Migration files should be placed in the configured migration path. +The current schema version is 2: + +- version 1 — adds `hostnames` column to `networkmanager_instances`; +- version 2 — drops the `imagemanager` table, creates `imagemanager_items`, `imagemanager_blobs` and the + `imagemanager_blobs_digest` index on `imagemanager_blobs(digest)`, adds `indexDigest` column to `launcher_instances` + and `launcher_run_requests`. The script records the schema version in its own transaction, so an interrupted + migration is either fully applied or not applied at all and the non-repeatable `ALTER TABLE` statements never run + twice. + ## Configuration Database configuration is provided through `Config`: @@ -229,5 +259,5 @@ struct Config { ``` [storagestate-storage-itf]: https://github.com/aosedge/aos_core_lib_cpp/blob/main/src/core/cm/storagestate/itf/storage.hpp -[iamgemanager-storage-itf]: https://github.com/aosedge/aos_core_lib_cpp/blob/main/src/core/cm/imagemanager/itf/storage.hpp +[imagemanager-storage-itf]: https://github.com/aosedge/aos_core_lib_cpp/blob/main/src/core/common/imagemanager/itf/storage.hpp [launcher-storage-itf]: https://github.com/aosedge/aos_core_lib_cpp/blob/main/src/core/cm/launcher/itf/storage.hpp diff --git a/src/cm/database/migration/2_update.down.sql b/src/cm/database/migration/2_update.down.sql new file mode 100644 index 000000000..54f154f17 --- /dev/null +++ b/src/cm/database/migration/2_update.down.sql @@ -0,0 +1,21 @@ +BEGIN TRANSACTION; + +ALTER TABLE launcher_run_requests DROP COLUMN indexDigest; +ALTER TABLE launcher_instances DROP COLUMN indexDigest; + +DROP TABLE IF EXISTS imagemanager_blobs; +DROP TABLE IF EXISTS imagemanager_items; + +CREATE TABLE IF NOT EXISTS imagemanager ( + itemID TEXT, + type TEXT, + version TEXT, + indexDigest TEXT, + state TEXT, + timestamp INTEGER, + PRIMARY KEY (itemID, version) +); + +UPDATE SchemaVersion SET version = 1; + +COMMIT; diff --git a/src/cm/database/migration/2_update.up.sql b/src/cm/database/migration/2_update.up.sql new file mode 100644 index 000000000..17d49f742 --- /dev/null +++ b/src/cm/database/migration/2_update.up.sql @@ -0,0 +1,37 @@ +BEGIN TRANSACTION; + +DROP TABLE IF EXISTS imagemanager; + +CREATE TABLE IF NOT EXISTS imagemanager_items ( + itemID TEXT, + type TEXT, + version TEXT, + preinstalled INTEGER, + state TEXT, + errorAosCode INTEGER, + errorSysCode INTEGER, + errorMessage TEXT, + rootDigest TEXT, + timestamp INTEGER, + PRIMARY KEY (itemID, version) +); + +CREATE TABLE IF NOT EXISTS imagemanager_blobs ( + itemID TEXT, + version TEXT, + digest TEXT, + mediaType TEXT, + size INTEGER, + isLayer INTEGER, + diffID TEXT, + PRIMARY KEY (itemID, version, digest) +); + +CREATE INDEX IF NOT EXISTS imagemanager_blobs_digest ON imagemanager_blobs(digest); + +ALTER TABLE launcher_instances ADD COLUMN indexDigest TEXT NOT NULL DEFAULT ''; +ALTER TABLE launcher_run_requests ADD COLUMN indexDigest TEXT NOT NULL DEFAULT ''; + +UPDATE SchemaVersion SET version = 2; + +COMMIT; diff --git a/src/cm/database/tests/database.cpp b/src/cm/database/tests/database.cpp index b8502547a..fc90c4b86 100644 --- a/src/cm/database/tests/database.cpp +++ b/src/cm/database/tests/database.cpp @@ -4,6 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + #include #include @@ -28,6 +30,11 @@ std::vector ToVector(const Array& src) return std::vector(src.begin(), src.end()); } +std::string CreateIndexDigest(const char* itemID, const char* version) +{ + return std::string("sha256:index-") + itemID + "-" + version; +} + InstanceIdent CreateInstanceIdent(const char* itemID, const char* subjectID, uint64_t instance, UpdateItemType itemType = UpdateItemTypeEnum::eService, bool preinstalled = true) { @@ -150,6 +157,7 @@ launcher::InstanceInfo CreateLauncherInstanceInfo(const char* itemID, const char info.mState = state; info.mIsUnitSubject = isUnitSubject; info.mVersion = version; + info.mIndexDigest = CreateIndexDigest(itemID, version).c_str(); info.mOwnerID = ownerID; info.mSubjectType = subjectType; info.mLabels.Clear(); @@ -162,19 +170,70 @@ launcher::InstanceInfo CreateLauncherInstanceInfo(const char* itemID, const char return info; } -imagemanager::ItemInfo CreateImageManagerItemInfo( - const char* itemID, const UpdateItemType& type, const char* version, const char* indexDigest, ItemState state) +common::imagemanager::DeployableItemInfo CreateDeployableItem(const char* itemID, const char* version, + ItemState state = ItemStateEnum::eInstalled, UpdateItemType type = UpdateItemTypeEnum::eService, + const char* rootDigest = "sha256:root", bool preinstalled = false, const Error& error = ErrorEnum::eNone) { - imagemanager::ItemInfo info; + common::imagemanager::DeployableItemInfo item; - info.mItemID = itemID; - info.mType = type; - info.mVersion = version; - info.mIndexDigest = indexDigest; - info.mState = state; - info.mTimestamp = Time::Now(); + item.mItemID = itemID; + item.mType = type; + item.mVersion = version; + item.mPreinstalled = preinstalled; + item.mState = state; + item.mError = error; + item.mRootDigest = rootDigest; + item.mTimestamp = Time::Now(); - return info; + return item; +} + +common::imagemanager::ItemBlobInfo CreateItemBlob( + const char* digest, bool isLayer = false, const char* diffID = "", uint64_t size = 1024) +{ + common::imagemanager::ItemBlobInfo blob; + + blob.mMediaType + = isLayer ? "application/vnd.oci.image.layer.v1.tar+gzip" : "application/vnd.oci.image.manifest.v1+json"; + blob.mDigest = digest; + blob.mSize = size; + blob.mIsLayer = isLayer; + blob.mDiffID = diffID; + + return blob; +} + +size_t CountTables(const std::string& dbPath, const std::string& tableName) +{ + Poco::Data::Session session("SQLite", dbPath); + size_t count = 0; + + session << "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?;", + Poco::Data::Keywords::bind(tableName), Poco::Data::Keywords::into(count), Poco::Data::Keywords::now; + + return count; +} + +bool HasColumn(const std::string& dbPath, const std::string& tableName, const std::string& columnName) +{ + Poco::Data::Session session("SQLite", dbPath); + size_t count = 0; + + session << "SELECT COUNT(*) FROM pragma_table_info(?) WHERE name = ?;", Poco::Data::Keywords::bind(tableName), + Poco::Data::Keywords::bind(columnName), Poco::Data::Keywords::into(count), Poco::Data::Keywords::now; + + return count != 0; +} + +int GetSchemaVersion(const std::string& dbPath) +{ + Poco::Data::Session session("SQLite", dbPath); + int version = 0; + + session << "SELECT version FROM SchemaVersion LIMIT 1;", Poco::Data::Keywords::into(version), + Poco::Data::Keywords::now; + + return version; } EnvVarInfo CreateEnvVarInfo(const char* name, const char* value, const Optional