From 87f80399534a538d759f76c74f8e7c7341780842 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 4 Nov 2026 10:47:21 +0200 Subject: [PATCH 01/18] common: downloader: download from several URLs with cancel context Signed-off-by: Mykola Solianko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/common/downloader/downloader.cpp | 195 +++++++---- src/common/downloader/downloader.hpp | 52 ++- src/common/downloader/tests/downloader.cpp | 327 +++++++++++++++--- .../downloader/tests/stubs/httpserverstub.hpp | 50 +++ 4 files changed, 465 insertions(+), 159 deletions(-) diff --git a/src/common/downloader/downloader.cpp b/src/common/downloader/downloader.cpp index 3e6f0f15f..355c2b05b 100644 --- a/src/common/downloader/downloader.cpp +++ b/src/common/downloader/downloader.cpp @@ -5,10 +5,12 @@ */ #include -#include #include +#include + #include +#include #include #include @@ -44,51 +46,50 @@ Error Downloader::Init(std::string_view certStorage, aos::iamclient::CertProvide return Init(sender, progressInterval); } -Downloader::~Downloader() +Error Downloader::Download( + CancelContext& ctx, const String& digest, const Array>& urls, const String& path) { - std::lock_guard lock {mMutex}; - - mShutdown = true; - mCondVar.notify_all(); -} + LOG_DBG() << "Start download" << Log::Field("digest", digest) << Log::Field("path", path) + << Log::Field("urls", urls.Size()); -Error Downloader::Download(const String& digest, const String& url, const String& path) -{ - if (mCertProvider) { - try { - if (Poco::URI(url.CStr()).getScheme() != "https") { - return Error(ErrorEnum::eInvalidArgument, "mutual TLS download requires HTTPS"); - } - } catch (const std::exception& e) { - return utils::ToAosError(e); - } + if (urls.IsEmpty()) { + return Error(ErrorEnum::eInvalidArgument, "no download URLs"); } - LOG_DBG() << "Start download" << Log::Field("url", url) << Log::Field("path", path) << Log::Field("digest", digest); + if (ctx.IsCanceled()) { + return Error(ErrorEnum::eCanceled, "download canceled"); + } ProgressContext context; - context.mDownloader = this; - context.mDigest = digest.CStr(); - context.mURL = url.CStr(); + context.mDownloader = this; + context.mDigest = digest.CStr(); + context.mCancelContext = &ctx; { std::lock_guard lock {mMutex}; - auto it = mCancelFlags.find(digest.CStr()); - if (it != mCancelFlags.end()) { + if (!mDigestsInProgress.insert(context.mDigest).second) { return Error(ErrorEnum::eAlreadyExist, "download already in progress"); } - - mCancelFlags[context.mDigest].store(false); - context.mCancelFlag = &mCancelFlags[context.mDigest]; } - auto err = RetryDownload(url, path, &context); - - { + auto releaseDigest = DeferRelease(&context.mDigest, [this](const std::string* digest) { std::lock_guard lock {mMutex}; - mCancelFlags.erase(context.mDigest); + + mDigestsInProgress.erase(*digest); + }); + + Error err; + + for (const auto& url : urls) { + context.mURL = url.CStr(); + + if (err = RetryDownload(ctx, url, path, &context); err.IsNone() || err.Is(ErrorEnum::eCanceled)) { + break; + } + + LOG_ERR() << "Failed to download from URL" << Log::Field("url", url) << Log::Field(err); } return err; @@ -168,15 +169,15 @@ CURLcode Downloader::SSLContextCallback(CURL*, void* sslContext, void* userData) return CURLE_OK; } -bool Downloader::SupportsRangeRequests(const String& url) +RetWithError Downloader::SupportsRangeRequests(const String& url, ProgressContext* context) { std::unique_ptr curl(curl_easy_init(), curl_easy_cleanup); if (!curl) { - return false; + return {false, Error(ErrorEnum::eFailed, "failed to init curl")}; } if (auto err = ConfigureTLS(curl.get()); !err.IsNone()) { - return false; + return {false, err}; } bool supportsRange = false; @@ -185,8 +186,13 @@ bool Downloader::SupportsRangeRequests(const String& url) auto* supports = static_cast(userdata); std::string header(buffer, size * nitems); - if (header.find("Accept-Ranges: bytes") != std::string::npos) { - *supports = true; + if (auto pos = header.find(':'); pos != std::string::npos) { + auto name = Poco::trim(header.substr(0, pos)); + auto value = Poco::trim(header.substr(pos + 1)); + + if (Poco::icompare(name, "Accept-Ranges") == 0 && Poco::icompare(value, "bytes") == 0) { + *supports = true; + } } return size * nitems; @@ -197,17 +203,37 @@ bool Downloader::SupportsRangeRequests(const String& url) curl_easy_setopt(curl.get(), CURLOPT_HEADERFUNCTION, static_cast(headerCallback)); curl_easy_setopt(curl.get(), CURLOPT_HEADERDATA, &supportsRange); curl_easy_setopt(curl.get(), CURLOPT_CONNECTTIMEOUT, cTimeoutSec); + curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(curl.get(), CURLOPT_XFERINFOFUNCTION, &Downloader::CancelXferInfoCallback); + curl_easy_setopt(curl.get(), CURLOPT_XFERINFODATA, context); - if (curl_easy_perform(curl.get()) != CURLE_OK) { - return false; + if (auto res = curl_easy_perform(curl.get()); res != CURLE_OK) { + return {false, Error(ErrorEnum::eFailed, curl_easy_strerror(res))}; } - return supportsRange; + long code = 0; + + curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &code); + + if (code >= 400) { + LOG_ERR() << "HEAD request failed" << Log::Field("HTTP_CODE", code); + + return {false, Error(code, "HEAD request failed")}; + } + + return {supportsRange, ErrorEnum::eNone}; } Error Downloader::DownloadImage(const String& url, const String& path, ProgressContext* context) { - Poco::URI uri(url.CStr()); + Poco::URI uri; + + try { + uri = Poco::URI(url.CStr()); + } catch (const std::exception& e) { + return AOS_ERROR_WRAP(utils::ToAosError(e)); + } + if (uri.getScheme() == "file") { return CopyFile(uri, path); } @@ -223,14 +249,28 @@ Error Downloader::DownloadImage(const String& url, const String& path, ProgressC curl_off_t existingSize = 0; - if (std::filesystem::exists(path.CStr())) { - existingSize = std::filesystem::file_size(path.CStr()); + try { + if (std::filesystem::exists(path.CStr())) { + existingSize = std::filesystem::file_size(path.CStr()); + } + } catch (const std::exception& e) { + return AOS_ERROR_WRAP(utils::ToAosError(e)); } bool canResume = false; if (existingSize > 0) { - canResume = SupportsRangeRequests(url); + Error err; + + Tie(canResume, err) = SupportsRangeRequests(url, context); + + if (IsCanceled(context)) { + return Error(ErrorEnum::eCanceled, "download canceled"); + } + + if (!err.IsNone()) { + return err; + } if (!canResume) { LOG_DBG() << "Server does not support range requests, starting from beginning"; @@ -307,15 +347,12 @@ Error Downloader::CopyFile(const Poco::URI& uri, const String& outfilename) path = uri.getHost(); } - if (!std::filesystem::exists(path)) { - return Error(ErrorEnum::eFailed, "file not found"); - } - try { - std::ifstream ifs(path, std::ios::binary); - std::ofstream ofs(outfilename.CStr(), std::ios::binary | std::ios::trunc); + if (!std::filesystem::exists(path)) { + return Error(ErrorEnum::eFailed, "file not found"); + } - ofs << ifs.rdbuf(); + std::filesystem::copy_file(path, outfilename.CStr(), std::filesystem::copy_options::overwrite_existing); return ErrorEnum::eNone; } catch (const std::exception& e) { @@ -323,14 +360,24 @@ Error Downloader::CopyFile(const Poco::URI& uri, const String& outfilename) } } -Error Downloader::RetryDownload(const String& url, const String& path, ProgressContext* context) +Error Downloader::RetryDownload(CancelContext& ctx, const String& url, const String& path, ProgressContext* context) { + if (mCertProvider) { + try { + if (Poco::URI(url.CStr()).getScheme() != "https") { + return Error(ErrorEnum::eInvalidArgument, "mutual TLS download requires HTTPS"); + } + } catch (const std::exception& e) { + return AOS_ERROR_WRAP(utils::ToAosError(e)); + } + } + auto delay = cDelay; Error err; - for (int retryCount = 0; (retryCount < cMaxRetryCount) && (!mShutdown); ++retryCount) { - if (context->mCancelFlag && context->mCancelFlag->load()) { - return Error(ErrorEnum::eRuntime, "download cancelled"); + for (int retryCount = 0; retryCount < cMaxRetryCount; ++retryCount) { + if (IsCanceled(context)) { + return Error(ErrorEnum::eCanceled, "download canceled"); } LOG_DBG() << "Downloading" << Log::Field("url", url) << Log::Field("retry", retryCount); @@ -341,13 +388,18 @@ Error Downloader::RetryDownload(const String& url, const String& path, ProgressC return ErrorEnum::eNone; } + if (IsCanceled(context)) { + return Error(ErrorEnum::eCanceled, "download canceled"); + } + LOG_ERR() << "Failed to download" << Log::Field("retry", retryCount) << Log::Field(AOS_ERROR_WRAP(err)); - { - std::unique_lock lock {mMutex}; + if (retryCount + 1 == cMaxRetryCount) { + break; + } - mCondVar.wait_for(lock, delay, - [this, context] { return mShutdown || (context->mCancelFlag && context->mCancelFlag->load()); }); + if (auto waitErr = ctx.Wait(delay.count() * Time::cMilliseconds); !waitErr.Is(ErrorEnum::eTimeout)) { + return AOS_ERROR_WRAP(waitErr); } delay = std::min(delay * 2, cMaxDelay); @@ -360,13 +412,18 @@ Error Downloader::RetryDownload(const String& url, const String& path, ProgressC * Progress callback **********************************************************************************************************************/ +bool Downloader::IsCanceled(const ProgressContext* context) +{ + return context->mCancelContext && context->mCancelContext->IsCanceled(); +} + int Downloader::XferInfoCallback( void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { auto* ctx = static_cast(clientp); - if (ctx->mCancelFlag && ctx->mCancelFlag->load()) { - LOG_DBG() << "Download cancelled via progress callback"; + if (IsCanceled(ctx)) { + LOG_DBG() << "Download canceled via progress callback"; return 1; } @@ -374,6 +431,11 @@ int Downloader::XferInfoCallback( return ctx->mDownloader->OnProgress(ctx, dltotal, dlnow, ultotal, ulnow); } +int Downloader::CancelXferInfoCallback(void* clientp, curl_off_t, curl_off_t, curl_off_t, curl_off_t) +{ + return IsCanceled(static_cast(clientp)) ? 1 : 0; +} + int Downloader::OnProgress(ProgressContext* context, curl_off_t dltotal, curl_off_t dlnow, [[maybe_unused]] curl_off_t ultotal, [[maybe_unused]] curl_off_t ulnow) { @@ -428,19 +490,4 @@ void Downloader::SendAlert(ProgressContext* context, DownloadState state, size_t mSender->SendAlert(param); } -Error Downloader::Cancel(const String& digest) -{ - std::lock_guard lock {mMutex}; - - if (auto it = mCancelFlags.find(digest.CStr()); it != mCancelFlags.end()) { - it->second.store(true); - - LOG_DBG() << "Cancel requested for download:" << Log::Field("digest", digest); - - return ErrorEnum::eNone; - } - - return Error(ErrorEnum::eNotFound, "download not found"); -} - } // namespace aos::common::downloader diff --git a/src/common/downloader/downloader.hpp b/src/common/downloader/downloader.hpp index 5ffef09a4..848d24bba 100644 --- a/src/common/downloader/downloader.hpp +++ b/src/common/downloader/downloader.hpp @@ -7,13 +7,11 @@ #ifndef AOS_COMMON_DOWNLOADER_DOWNLOADER_HPP_ #define AOS_COMMON_DOWNLOADER_DOWNLOADER_HPP_ -#include #include -#include #include +#include #include #include -#include #include #include @@ -58,28 +56,17 @@ class Downloader : public aos::downloader::DownloaderItf { aos::alerts::SenderItf* sender = nullptr, std::chrono::seconds progressInterval = std::chrono::seconds {30}); /** - * Destructor. - */ - ~Downloader(); - - /** - * Downloads file. + * Downloads file. Tries the URLs in order, each URL is retried with the downloader retry policy. * - * @param digest image digest. - * @param url URL. + * @param ctx cancel context. + * @param digest file digest. + * @param urls file URLs. * @param path path to file. - * @return Error. - */ - Error Download(const String& digest, const String& url, const String& path) override; - - /** - * Cancels ongoing download. - * - * @param digest image digest. - * - * @return Error. + * @return Error: eCanceled if the download is canceled by the cancel context, eInvalidArgument if urls is empty, + * eAlreadyExist if a download of the same digest is in progress, otherwise the error of the last attempt. */ - Error Cancel(const String& digest) override; + Error Download(CancelContext& ctx, const String& digest, const Array>& urls, + const String& path) override; private: constexpr static std::chrono::milliseconds cDelay {1000}; @@ -89,7 +76,7 @@ class Downloader : public aos::downloader::DownloaderItf { struct ProgressContext { Downloader* mDownloader {}; - std::atomic* mCancelFlag {}; + CancelContext* mCancelContext {}; std::string mDigest; std::string mURL; std::chrono::steady_clock::time_point mLastProgressTime; @@ -101,21 +88,22 @@ class Downloader : public aos::downloader::DownloaderItf { Error ConfigureTLS(CURL* curl); static CURLcode SSLContextCallback(CURL* curl, void* sslContext, void* userData); - bool SupportsRangeRequests(const String& url); - Error DownloadImage(const String& url, const String& path, ProgressContext* context); - Error CopyFile(const Poco::URI& uri, const String& outfilename); - Error RetryDownload(const String& url, const String& path, ProgressContext* context); + RetWithError SupportsRangeRequests(const String& url, ProgressContext* context); + Error DownloadImage(const String& url, const String& path, ProgressContext* context); + Error CopyFile(const Poco::URI& uri, const String& outfilename); + Error RetryDownload(CancelContext& ctx, const String& url, const String& path, ProgressContext* context); void SendAlert(ProgressContext* context, DownloadState state, size_t downloadedBytes, size_t totalBytes, const std::string& reason = "", const Error& error = ErrorEnum::eNone); - static int XferInfoCallback( + static bool IsCanceled(const ProgressContext* context); + static int XferInfoCallback( + void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow); + static int CancelXferInfoCallback( void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow); int OnProgress( ProgressContext* context, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow); - bool mShutdown {false}; - std::mutex mMutex; - std::condition_variable mCondVar; + std::mutex mMutex; std::chrono::seconds mProgressInterval {std::chrono::seconds {30}}; @@ -126,7 +114,7 @@ class Downloader : public aos::downloader::DownloaderItf { crypto::CertLoaderItf* mCertLoader {}; crypto::x509::ProviderItf* mCryptoProvider {}; - std::unordered_map> mCancelFlags; + std::set mDigestsInProgress; }; } // namespace aos::common::downloader diff --git a/src/common/downloader/tests/downloader.cpp b/src/common/downloader/tests/downloader.cpp index a43e01d5a..c07379f86 100644 --- a/src/common/downloader/tests/downloader.cpp +++ b/src/common/downloader/tests/downloader.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include @@ -28,6 +30,31 @@ using namespace testing; +namespace { + +/*********************************************************************************************************************** + * Types + **********************************************************************************************************************/ + +using URLArray = aos::StaticArray, 4>; + +/*********************************************************************************************************************** + * Utils + **********************************************************************************************************************/ + +std::unique_ptr CreateURLs(std::initializer_list urls) +{ + auto result = std::make_unique(); + + for (const auto& url : urls) { + EXPECT_TRUE(result->EmplaceBack(url.c_str()).IsNone()); + } + + return result; +} + +} // namespace + /*********************************************************************************************************************** * Suite **********************************************************************************************************************/ @@ -83,6 +110,7 @@ class DownloaderTest : public ::testing::Test { std::optional mServer; aos::common::downloader::Downloader mDownloader; + aos::CancelContext mCancelCtx; StrictMock mAlertSender; std::string mFilePath = "download/test_file.dat"; }; @@ -95,7 +123,8 @@ TEST_F(DownloaderTest, Download) { StartServer(); - auto err = mDownloader.Download("digest1", "http://localhost:8000/test_file.dat", mFilePath.c_str()); + auto err = mDownloader.Download( + mCancelCtx, "digest1", *CreateURLs({"http://localhost:8000/test_file.dat"}), mFilePath.c_str()); EXPECT_EQ(err, aos::ErrorEnum::eNone); EXPECT_TRUE(std::filesystem::exists(mFilePath)); @@ -110,7 +139,7 @@ TEST_F(DownloaderTest, Download) TEST_F(DownloaderTest, DownloadFileScheme) { - auto err = mDownloader.Download("digest2", "file://test_file.dat", mFilePath.c_str()); + auto err = mDownloader.Download(mCancelCtx, "digest2", *CreateURLs({"file://test_file.dat"}), mFilePath.c_str()); EXPECT_EQ(err, aos::ErrorEnum::eNone); EXPECT_TRUE(std::filesystem::exists(mFilePath)); @@ -133,7 +162,8 @@ TEST_F(DownloaderTest, DownloadLargeFileWithProgress) EXPECT_CALL(mAlertSender, SendAlert(_)).WillRepeatedly(Return(aos::ErrorEnum::eNone)); - auto err = mDownloader.Download("digest3", "http://localhost:8001/large_test_file.dat", mFilePath.c_str()); + auto err = mDownloader.Download( + mCancelCtx, "digest3", *CreateURLs({"http://localhost:8001/large_test_file.dat"}), mFilePath.c_str()); EXPECT_EQ(err, aos::ErrorEnum::eNone); EXPECT_TRUE(std::filesystem::exists(mFilePath)); @@ -146,39 +176,6 @@ TEST_F(DownloaderTest, DownloadLargeFileWithProgress) StopServer(); } -TEST_F(DownloaderTest, CancelDownload) -{ - const size_t fileSizeMB = 10; - - CreateLargeFile("large_test_file.dat", fileSizeMB); - - mDownloader.Init(&mAlertSender, std::chrono::seconds {1}); - - StartServer("large_test_file.dat", 8002, 100); - - EXPECT_CALL(mAlertSender, SendAlert(_)).Times(AtLeast(1)); - - std::future downloadFuture = std::async(std::launch::async, [this]() { - return mDownloader.Download("digest_cancel", "http://localhost:8002/large_test_file.dat", mFilePath.c_str()); - }); - - std::this_thread::sleep_for(std::chrono::milliseconds(500)); - - auto cancelErr = mDownloader.Cancel("digest_cancel"); - EXPECT_EQ(cancelErr, aos::ErrorEnum::eNone); - - auto downloadErr = downloadFuture.get(); - EXPECT_EQ(downloadErr, aos::ErrorEnum::eRuntime); - - StopServer(); -} - -TEST_F(DownloaderTest, CancelNonExistentDownload) -{ - auto err = mDownloader.Cancel("non_existent_digest"); - EXPECT_EQ(err, aos::ErrorEnum::eNotFound); -} - TEST_F(DownloaderTest, ParallelDownloads) { CreateLargeFile("large_test_file.dat", 2); @@ -186,13 +183,13 @@ TEST_F(DownloaderTest, ParallelDownloads) StartServer("large_test_file.dat", 8003, 100); std::future download1 = std::async(std::launch::async, [this]() { - return mDownloader.Download( - "digest_parallel_1", "http://localhost:8003/large_test_file.dat", "download/file1.dat"); + return mDownloader.Download(mCancelCtx, "digest_parallel_1", + *CreateURLs({"http://localhost:8003/large_test_file.dat"}), "download/file1.dat"); }); std::future download2 = std::async(std::launch::async, [this]() { - return mDownloader.Download( - "digest_parallel_2", "http://localhost:8003/large_test_file.dat", "download/file2.dat"); + return mDownloader.Download(mCancelCtx, "digest_parallel_2", + *CreateURLs({"http://localhost:8003/large_test_file.dat"}), "download/file2.dat"); }); auto err1 = download1.get(); @@ -217,13 +214,14 @@ TEST_F(DownloaderTest, DuplicateDownload) StartServer("large_test_file.dat", 8004, 100); std::future download1 = std::async(std::launch::async, [this]() { - return mDownloader.Download("digest_dup", "http://localhost:8004/large_test_file.dat", "download/file_dup.dat"); + return mDownloader.Download(mCancelCtx, "digest_dup", + *CreateURLs({"http://localhost:8004/large_test_file.dat"}), "download/file_dup.dat"); }); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - auto err2 - = mDownloader.Download("digest_dup", "http://localhost:8004/large_test_file.dat", "download/file_dup2.dat"); + auto err2 = mDownloader.Download( + mCancelCtx, "digest_dup", *CreateURLs({"http://localhost:8004/large_test_file.dat"}), "download/file_dup2.dat"); EXPECT_EQ(err2, aos::ErrorEnum::eAlreadyExist); auto err1 = download1.get(); @@ -242,10 +240,12 @@ TEST_F(DownloaderTest, MutualTLSRejectsUnencryptedURLs) ASSERT_TRUE(mDownloader.Init("sm", certProvider, certLoader, cryptoProvider).IsNone()); - EXPECT_TRUE(mDownloader.Download("http", "http://localhost:8000/test_file.dat", mFilePath.c_str()) - .Is(aos::ErrorEnum::eInvalidArgument)); EXPECT_TRUE( - mDownloader.Download("file", "file://test_file.dat", mFilePath.c_str()).Is(aos::ErrorEnum::eInvalidArgument)); + mDownloader + .Download(mCancelCtx, "http", *CreateURLs({"http://localhost:8000/test_file.dat"}), mFilePath.c_str()) + .Is(aos::ErrorEnum::eInvalidArgument)); + EXPECT_TRUE(mDownloader.Download(mCancelCtx, "file", *CreateURLs({"file://test_file.dat"}), mFilePath.c_str()) + .Is(aos::ErrorEnum::eInvalidArgument)); EXPECT_FALSE(std::filesystem::exists(mFilePath)); } @@ -259,7 +259,8 @@ TEST_F(DownloaderTest, MutualTLSDoesNotChangeOtherDownloaders) aos::common::downloader::Downloader downloader; ASSERT_TRUE(downloader.Init().IsNone()); - EXPECT_TRUE(downloader.Download("file", "file://test_file.dat", mFilePath.c_str()).IsNone()); + EXPECT_TRUE( + downloader.Download(mCancelCtx, "file", *CreateURLs({"file://test_file.dat"}), mFilePath.c_str()).IsNone()); } class DownloaderTLSFailureTest : public DownloaderTest, public ::testing::WithParamInterface { }; @@ -282,7 +283,7 @@ TEST_P(DownloaderTLSFailureTest, CredentialFailureNeverDownloadsUnverifiedConten ASSERT_TRUE(mDownloader.Init("sm", certProvider, certLoader, cryptoProvider).IsNone()); EXPECT_CALL(certProvider, GetCertKey(_, _, _, _)) - .Times(4) // One HEAD for the partial file, then three failed GET attempts. + .Times(3) .WillRepeatedly(Invoke([&](const aos::String& storageID, const aos::Array&, const aos::Array&, aos::CertInfo&) -> aos::Error { EXPECT_EQ(storageID, "sm"); @@ -298,10 +299,13 @@ TEST_P(DownloaderTLSFailureTest, CredentialFailureNeverDownloadsUnverifiedConten std::ofstream(mFilePath) << "partial download"; auto url = "https://localhost:" + std::to_string(socket.address().port()) + "/test_file.dat"; - EXPECT_FALSE(mDownloader.Download("tls_failure", url.c_str(), mFilePath.c_str()).IsNone()); - if (std::filesystem::exists(mFilePath)) { - EXPECT_EQ(std::filesystem::file_size(mFilePath), 0); - } + EXPECT_FALSE(mDownloader.Download(mCancelCtx, "tls_failure", *CreateURLs({url}), mFilePath.c_str()).IsNone()); + + std::ifstream ifs(mFilePath, std::ios::binary); + std::string content((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); + + EXPECT_EQ(content, "partial download"); + server.stopAll(true); } @@ -313,6 +317,223 @@ TEST_F(DownloaderTest, MutualTLSRejectsMalformedURL) StrictMock certLoader; StrictMock cryptoProvider; ASSERT_TRUE(mDownloader.Init("sm", certProvider, certLoader, cryptoProvider).IsNone()); - EXPECT_FALSE(mDownloader.Download("invalid", "https://[invalid", mFilePath.c_str()).IsNone()); + EXPECT_FALSE( + mDownloader.Download(mCancelCtx, "invalid", *CreateURLs({"https://[invalid"}), mFilePath.c_str()).IsNone()); EXPECT_FALSE(std::filesystem::exists(mFilePath)); } + +TEST_F(DownloaderTest, FallbackToNextURL) +{ + StartServer("test_file.dat", 8005); + + auto err = mDownloader.Download(mCancelCtx, "digest_fallback", + *CreateURLs({"http://127.0.0.1:1/test_file.dat", "http://localhost:8005/test_file.dat"}), mFilePath.c_str()); + EXPECT_EQ(err, aos::ErrorEnum::eNone); + + std::ifstream ifs(mFilePath, std::ios::binary); + std::string content((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); + + EXPECT_EQ(content, "This is a test file"); + + StopServer(); +} + +TEST_F(DownloaderTest, AllURLsFailReturnsLastError) +{ + StartServer("missing_file.dat", 8006); + + auto err = mDownloader.Download(mCancelCtx, "digest_all_fail", + *CreateURLs({"file://missing_file.dat", "http://localhost:8006/missing_file.dat"}), mFilePath.c_str()); + EXPECT_EQ(err.Code(), 404); + + StopServer(); +} + +TEST_F(DownloaderTest, EmptyURLs) +{ + auto err = mDownloader.Download(mCancelCtx, "digest_empty", *CreateURLs({}), mFilePath.c_str()); + EXPECT_TRUE(err.Is(aos::ErrorEnum::eInvalidArgument)); +} + +TEST_F(DownloaderTest, PreCanceledContext) +{ + ASSERT_TRUE(mCancelCtx.Cancel().IsNone()); + + auto err = mDownloader.Download( + mCancelCtx, "digest_precanceled", *CreateURLs({"file://test_file.dat"}), mFilePath.c_str()); + EXPECT_TRUE(err.Is(aos::ErrorEnum::eCanceled)); + EXPECT_FALSE(std::filesystem::exists(mFilePath)); + + aos::CancelContext ctx; + + err = mDownloader.Download(ctx, "digest_precanceled", *CreateURLs({"file://test_file.dat"}), mFilePath.c_str()); + EXPECT_TRUE(err.IsNone()); +} + +TEST_F(DownloaderTest, CancelDuringRetryDelay) +{ + auto downloadFuture = std::async(std::launch::async, [this]() { + return mDownloader.Download(mCancelCtx, "digest_cancel_delay", + *CreateURLs({"http://127.0.0.1:1/test_file.dat", "file://test_file.dat"}), mFilePath.c_str()); + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + ASSERT_TRUE(mCancelCtx.Cancel().IsNone()); + ASSERT_EQ(downloadFuture.wait_for(std::chrono::milliseconds(500)), std::future_status::ready); + + EXPECT_TRUE(downloadFuture.get().Is(aos::ErrorEnum::eCanceled)); + EXPECT_TRUE(!std::filesystem::exists(mFilePath) || std::filesystem::file_size(mFilePath) == 0); +} + +TEST_F(DownloaderTest, CancelDuringTransfer) +{ + const size_t fileSizeMB = 10; + + CreateLargeFile("large_test_file.dat", fileSizeMB); + + StartServer("large_test_file.dat", 8007, 100); + + auto downloadFuture = std::async(std::launch::async, [this]() { + return mDownloader.Download(mCancelCtx, "digest_cancel_transfer", + *CreateURLs({"http://localhost:8007/large_test_file.dat", "file://large_test_file.dat"}), + mFilePath.c_str()); + }); + + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + + while (!std::filesystem::exists(mFilePath) || std::filesystem::file_size(mFilePath) == 0) { + ASSERT_LT(std::chrono::steady_clock::now(), deadline); + + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + ASSERT_TRUE(mCancelCtx.Cancel().IsNone()); + ASSERT_EQ(downloadFuture.wait_for(std::chrono::seconds(5)), std::future_status::ready); + + EXPECT_TRUE(downloadFuture.get().Is(aos::ErrorEnum::eCanceled)); + + auto size = std::filesystem::file_size(mFilePath); + + EXPECT_GT(size, 0); + EXPECT_LT(size, fileSizeMB * 1024 * 1024); + + StopServer(); +} + +TEST_F(DownloaderTest, HeadFailureKeepsPartialFile) +{ + std::atomic headCount {0}; + std::atomic getCount {0}; + Poco::Net::ServerSocket socket(0); + Poco::Net::HTTPRequestHandlerFactory::Ptr factory = new FunctionRequestHandlerFactory( + [&](Poco::Net::HTTPServerRequest& request, Poco::Net::HTTPServerResponse& response) { + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD) { + if (++headCount == 2) { + EXPECT_TRUE(mCancelCtx.Cancel().IsNone()); + } + } else { + ++getCount; + } + + response.setStatus(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + response.send(); + }); + Poco::Net::HTTPServer server(factory, socket, new Poco::Net::HTTPServerParams); + + server.start(); + + std::ofstream(mFilePath, std::ios::binary) << "partial download"; + + auto url = "http://127.0.0.1:" + std::to_string(socket.address().port()) + "/test_file.dat"; + auto err = mDownloader.Download(mCancelCtx, "digest_head_failure", *CreateURLs({url}), mFilePath.c_str()); + + EXPECT_TRUE(err.Is(aos::ErrorEnum::eCanceled)); + EXPECT_EQ(headCount, 2); + EXPECT_EQ(getCount, 0); + + std::ifstream ifs(mFilePath, std::ios::binary); + std::string content((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); + + EXPECT_EQ(content, "partial download"); + + server.stopAll(true); +} + +TEST_F(DownloaderTest, ResumesWithLowercaseAcceptRanges) +{ + const std::string fileContent = "This is a test file"; + const size_t partialSize = 8; + std::atomic rangeRequested {false}; + Poco::Net::ServerSocket socket(0); + Poco::Net::HTTPRequestHandlerFactory::Ptr factory = new FunctionRequestHandlerFactory( + [&](Poco::Net::HTTPServerRequest& request, Poco::Net::HTTPServerResponse& response) { + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD) { + response.set("accept-ranges", "bytes"); + response.setContentLength(fileContent.size()); + response.send(); + + return; + } + + if (request.get("Range", "") != "bytes=" + std::to_string(partialSize) + "-") { + response.setContentLength(fileContent.size()); + response.send() << fileContent; + + return; + } + + rangeRequested = true; + + response.setStatus(Poco::Net::HTTPResponse::HTTP_PARTIAL_CONTENT); + response.set("Content-Range", + "bytes " + std::to_string(partialSize) + "-" + std::to_string(fileContent.size() - 1) + "/" + + std::to_string(fileContent.size())); + response.setContentLength(fileContent.size() - partialSize); + response.send() << fileContent.substr(partialSize); + }); + Poco::Net::HTTPServer server(factory, socket, new Poco::Net::HTTPServerParams); + + server.start(); + + std::ofstream(mFilePath, std::ios::binary) << fileContent.substr(0, partialSize); + + auto url = "http://127.0.0.1:" + std::to_string(socket.address().port()) + "/test_file.dat"; + auto err = mDownloader.Download(mCancelCtx, "digest_lowercase_ranges", *CreateURLs({url}), mFilePath.c_str()); + + EXPECT_TRUE(err.IsNone()); + EXPECT_TRUE(rangeRequested); + + std::ifstream ifs(mFilePath, std::ios::binary); + std::string content((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); + + EXPECT_EQ(content, fileContent); + + server.stopAll(true); +} + +TEST_F(DownloaderTest, DownloadFileSchemeDirectoryFails) +{ + std::filesystem::create_directory("source_dir"); + + auto err = mDownloader.Download(mCancelCtx, "digest_dir", *CreateURLs({"file://source_dir"}), mFilePath.c_str()); + + EXPECT_FALSE(err.IsNone()); + + std::filesystem::remove_all("source_dir"); +} + +TEST_F(DownloaderTest, DownloadFileSchemeEmptyFile) +{ + std::ofstream("empty_file.dat", std::ios::binary); + std::ofstream(mFilePath, std::ios::binary) << "previous content"; + + auto err = mDownloader.Download( + mCancelCtx, "digest_empty_file", *CreateURLs({"file://empty_file.dat"}), mFilePath.c_str()); + + EXPECT_TRUE(err.IsNone()); + EXPECT_TRUE(std::filesystem::exists(mFilePath)); + EXPECT_EQ(std::filesystem::file_size(mFilePath), 0); + + std::remove("empty_file.dat"); +} diff --git a/src/common/downloader/tests/stubs/httpserverstub.hpp b/src/common/downloader/tests/stubs/httpserverstub.hpp index f58044c92..dce4b07e8 100644 --- a/src/common/downloader/tests/stubs/httpserverstub.hpp +++ b/src/common/downloader/tests/stubs/httpserverstub.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -112,6 +113,55 @@ class FileRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory { int mDelayMs; }; +/** + * Function request handler factory. + */ +class FunctionRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory { +public: + using HandlerFunc = std::function; + + /** + * Constructor. + * + * @param handler request handler function. + */ + explicit FunctionRequestHandlerFactory(HandlerFunc handler) + : mHandler(std::move(handler)) + { + } + + /** + * Create request handler. + * + * @param request request. + * @return Poco::Net::HTTPRequestHandler. + */ + Poco::Net::HTTPRequestHandler* createRequestHandler( + [[maybe_unused]] const Poco::Net::HTTPServerRequest& request) override + { + return new Handler(mHandler); + } + +private: + class Handler : public Poco::Net::HTTPRequestHandler { + public: + explicit Handler(const HandlerFunc& handler) + : mHandler(handler) + { + } + + void handleRequest(Poco::Net::HTTPServerRequest& request, Poco::Net::HTTPServerResponse& response) override + { + mHandler(request, response); + } + + private: + HandlerFunc mHandler; + }; + + HandlerFunc mHandler; +}; + /** * HTTP server. */ From f340533b0b08eefe8e79c5555799dcea761dbf0f Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 4 Nov 2026 11:17:21 +0200 Subject: [PATCH 02/18] cm: database: implement common image manager storage Signed-off-by: Mykola Solianko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/cm/database/CMakeLists.txt | 4 +- src/cm/database/database.cpp | 145 +------ src/cm/database/database.hpp | 72 +--- src/cm/database/database.md | 62 +-- src/cm/database/migration/2_update.down.sql | 16 + src/cm/database/migration/2_update.up.sql | 32 ++ src/cm/database/tests/database.cpp | 276 ++++++------- src/common/CMakeLists.txt | 1 + src/common/database/CMakeLists.txt | 47 +++ src/common/database/database.cpp | 353 ++++++++++++++++ src/common/database/database.hpp | 167 ++++++++ src/common/database/database.md | 89 ++++ src/common/database/tests/CMakeLists.txt | 37 ++ src/common/database/tests/database.cpp | 423 ++++++++++++++++++++ 14 files changed, 1328 insertions(+), 396 deletions(-) create mode 100644 src/cm/database/migration/2_update.down.sql create mode 100644 src/cm/database/migration/2_update.up.sql create mode 100644 src/common/database/CMakeLists.txt create mode 100644 src/common/database/database.cpp create mode 100644 src/common/database/database.hpp create mode 100644 src/common/database/database.md create mode 100644 src/common/database/tests/CMakeLists.txt create mode 100644 src/common/database/tests/database.cpp diff --git a/src/cm/database/CMakeLists.txt b/src/cm/database/CMakeLists.txt index aca420fb0..ee7a254f8 100644 --- a/src/cm/database/CMakeLists.txt +++ b/src/cm/database/CMakeLists.txt @@ -16,7 +16,9 @@ set(SOURCES database.cpp) # Libraries # ###################################################################################################################### -set(LIBRARIES aos::common::cloudprotocol aos::common::utils aos::common::migration Poco::DataSQLite Poco::JSON) +set(LIBRARIES aos::common::cloudprotocol aos::common::database aos::common::utils aos::common::migration + Poco::DataSQLite Poco::JSON +) # ###################################################################################################################### # Target diff --git a/src/cm/database/database.cpp b/src/cm/database/database.cpp index 5013a23df..ceeabcfe8 100644 --- a/src/cm/database/database.cpp +++ b/src/cm/database/database.cpp @@ -934,118 +934,9 @@ Error Database::SaveRunRequests(const Array& reque } /*********************************************************************************************************************** - * imagemanager::StorageItf implementation + * updatemanager::StorageItf implementation **********************************************************************************************************************/ -Error Database::AddItem(const imagemanager::ItemInfo& item) -{ - std::lock_guard lock {mMutex}; - - try { - ImageManagerItemInfoRow row; - - FromAos(item, row); - *mSession << "INSERT INTO imagemanager (itemID, type, version, indexDigest, state, timestamp) VALUES " - "(?, ?, ?, ?, ?, ?);", - bind(row), now; - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - -Error Database::RemoveItem(const String& id, const String& version) -{ - std::lock_guard lock {mMutex}; - - try { - Poco::Data::Statement statement {*mSession}; - - statement << "DELETE FROM imagemanager WHERE itemID = ? AND version = ?;", bind(id.CStr()), - bind(version.CStr()); - - if (statement.execute() != 1) { - return ErrorEnum::eNotFound; - } - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - -Error Database::UpdateItemState(const String& id, const String& version, ItemState state, Time timestamp) -{ - std::lock_guard lock {mMutex}; - - try { - Poco::Data::Statement statement {*mSession}; - - statement << "UPDATE imagemanager SET state = ?, timestamp = ? WHERE itemID = ? AND version = ?;", - bind(state.ToString().CStr()), bind(timestamp.UnixNano()), bind(id.CStr()), bind(version.CStr()); - - if (statement.execute() != 1) { - return ErrorEnum::eNotFound; - } - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - -Error Database::GetAllItemsInfos(Array& items) -{ - std::lock_guard lock {mMutex}; - - try { - std::vector rows; - - *mSession << "SELECT itemID, type, version, indexDigest, state, timestamp FROM imagemanager;", into(rows), now; - - auto itemInfo = std::make_unique(); - - items.Clear(); - - for (const auto& row : rows) { - ToAos(row, *itemInfo); - auto err = items.PushBack(*itemInfo); - AOS_ERROR_CHECK_AND_THROW(AOS_ERROR_WRAP(err), "can't add item info"); - } - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - -Error Database::GetItemInfos(const String& id, Array& items) -{ - std::lock_guard lock {mMutex}; - - try { - std::vector rows; - - *mSession << "SELECT itemID, type, version, indexDigest, state, timestamp FROM imagemanager WHERE itemID = ?;", - bind(id.CStr()), into(rows), now; - - auto itemInfo = std::make_unique(); - - items.Clear(); - - for (const auto& row : rows) { - ToAos(row, *itemInfo); - auto err = items.PushBack(*itemInfo); - AOS_ERROR_CHECK_AND_THROW(AOS_ERROR_WRAP(err), "can't add item info"); - } - } catch (const std::exception& e) { - return AOS_ERROR_WRAP(common::utils::ToAosError(e)); - } - - return ErrorEnum::eNone; -} - Error Database::StoreDesiredStatus(const DesiredStatus& desiredStatus) { std::lock_guard lock {mMutex}; @@ -1171,17 +1062,6 @@ void Database::CreateTables() ");", now; - *mSession << "CREATE TABLE IF NOT EXISTS imagemanager (" - "itemID TEXT," - "type TEXT," - "version TEXT," - "indexDigest TEXT," - "state TEXT," - "timestamp INTEGER," - "PRIMARY KEY(itemID,version)" - ");", - now; - *mSession << "CREATE TABLE IF NOT EXISTS networks (" "networkID TEXT," "subnet TEXT," @@ -1466,29 +1346,6 @@ void Database::ToAos(const LauncherInstanceInfoRow& src, launcher::InstanceInfo& dst.mDisableRebalancing = src.get(); } -void Database::FromAos(const imagemanager::ItemInfo& src, ImageManagerItemInfoRow& dst) -{ - dst.set(src.mItemID.CStr()); - dst.set(src.mVersion.CStr()); - dst.set(src.mType.ToString().CStr()); - dst.set(src.mIndexDigest.CStr()); - dst.set(src.mState.ToString().CStr()); - dst.set(src.mTimestamp.UnixNano()); -} - -void Database::ToAos(const ImageManagerItemInfoRow& src, imagemanager::ItemInfo& dst) -{ - dst.mItemID = src.get().c_str(); - auto err = dst.mType.FromString(src.get().c_str()); - AOS_ERROR_CHECK_AND_THROW(err, "failed to parse item type"); - dst.mVersion = src.get().c_str(); - dst.mIndexDigest = src.get().c_str(); - err = dst.mState.FromString(src.get().c_str()); - AOS_ERROR_CHECK_AND_THROW(err, "failed to parse item state"); - auto timestamp = src.get(); - dst.mTimestamp = Time::Unix(timestamp / Time::cSeconds.Nanoseconds(), timestamp % Time::cSeconds.Nanoseconds()); -} - void Database::FromAos(const EnvVarsInstanceInfo& src, LauncherOverrideEnvVarsRow& dst) { std::string itemID; diff --git a/src/cm/database/database.hpp b/src/cm/database/database.hpp index 26af9e845..5e8964957 100644 --- a/src/cm/database/database.hpp +++ b/src/cm/database/database.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -27,7 +28,7 @@ namespace aos::cm::database { /** * Database class. */ -class Database : public DatabaseItf, public networkmanager::StorageItf { +class Database : public DatabaseItf, public common::database::Database, public networkmanager::StorageItf { public: /** * Creates database instance. @@ -296,54 +297,9 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { Error SaveRunRequests(const Array& requests) override; // - // imagemanager::StorageItf interface + // updatemanager::StorageItf interface // - /** - * Adds item. - * - * @param item Item info. - * @return Error. - */ - Error AddItem(const imagemanager::ItemInfo& item) override; - - /** - * Removes item. - * - * @param id ID. - * @param version Version. - * @return Error. - */ - Error RemoveItem(const String& id, const String& version) override; - - /** - * Updates item state. - * - * @param id ID. - * @param version Version. - * @param state Item state. - * @param timestamp Timestamp. - * @return Error. - */ - Error UpdateItemState(const String& id, const String& version, ItemState state, Time timestamp = {}) override; - - /** - * Gets all items info. - * - * @param items Items info. - * @return Error. - */ - Error GetAllItemsInfos(Array& items) override; - - /** - * Gets items info by ID. - * - * @param id Item ID. - * @param items Items info. - * @return Error. - */ - Error GetItemInfos(const String& id, Array& items) override; - /** * Stores desired status in storage. * @@ -352,10 +308,6 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { */ Error StoreDesiredStatus(const DesiredStatus& desiredStatus) override; - // - // updatemanager::StorageItf interface - // - /** * Stores update state in storage. * @@ -380,7 +332,7 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { RetWithError GetUpdateState() override; private: - static constexpr int cVersion = 1; + static constexpr int cVersion = 2; static constexpr auto cDBFileName = "cm.db"; enum class StorageStateInstanceInfoColumns : int { @@ -461,17 +413,6 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { std::string, std::string, std::string, uint32_t, uint32_t, uint64_t, std::string, bool, std::string, std::string, std::string, std::string, size_t, bool>; - enum class ImageManagerItemInfoColumns : int { - eItemID = 0, - eType, - eVersion, - eIndexDigest, - eState, - eTimestamp, - }; - using ImageManagerItemInfoRow - = Poco::Tuple; - enum class LauncherOverrideEnvVarsColumns : int { eItemID = 0, eSubjectID, eInstance, eVariables }; using LauncherOverrideEnvVarsRow = Poco::Tuple; @@ -512,18 +453,13 @@ class Database : public DatabaseItf, public networkmanager::StorageItf { static void FromAos(const launcher::InstanceInfo& src, LauncherInstanceInfoRow& dst); static void ToAos(const LauncherInstanceInfoRow& src, launcher::InstanceInfo& dst); - static void FromAos(const imagemanager::ItemInfo& src, ImageManagerItemInfoRow& dst); - static void ToAos(const ImageManagerItemInfoRow& src, imagemanager::ItemInfo& dst); - static void FromAos(const EnvVarsInstanceInfo& src, LauncherOverrideEnvVarsRow& dst); static void ToAos(const LauncherOverrideEnvVarsRow& src, EnvVarsInstanceInfo& dst); static void FromAos(const launcher::RunInstanceRequest& src, LauncherRunRequestRow& dst); static void ToAos(const LauncherRunRequestRow& src, launcher::RunInstanceRequest& dst); - std::unique_ptr mSession; std::optional mDatabase; - mutable std::mutex mMutex; }; } // namespace aos::cm::database diff --git a/src/cm/database/database.md b/src/cm/database/database.md index 0ce8f0bee..22893eaba 100644 --- a/src/cm/database/database.md +++ b/src/cm/database/database.md @@ -8,7 +8,8 @@ storage interfaces using SQLite as the backend database. The `Database` class implements the following interfaces to persist data for different CM modules: - [aos::cm::storagestate::StorageItf][storagestate-storage-itf] - storage state interface; -- [aos::cm::imagemanager::StorageItf][iamgemanager-storage-itf] - image manager interface; +- [aos::common::imagemanager::StorageItf][imagemanager-storage-itf] - image manager interface, implemented by the + [common database](../../common/database/database.md) base class `aos::common::database::Database`; - [aos::cm::networkmanager::StorageItf](../networkmanager/itf/storage.hpp) - network manager interface; - [aos::cm::launcher::StorageItf][launcher-storage-itf] - launcher interface. @@ -17,10 +18,14 @@ The `Database` class implements the following interfaces to persist data for dif ```mermaid classDiagram class Database { - -Session mSession -Migration mDatabase +Init(config) Error } + + class CommonDatabase["aos::common::database::Database"] { + #Session mSession + #mutex mMutex + } class StorageStateItf { <> @@ -33,11 +38,15 @@ classDiagram class ImageManagerStorageItf { <> - +AddItem() - +SetItemState() - +RemoveItem() - +GetItemsInfo() - +GetItemVersionsByID() + +AddDeployableItem() + +UpdateDeployableItem() + +RemoveDeployableItem() + +GetDeployableItem() + +GetAllDeployableItems() + +GetDeployableItemsCount() + +AddItemBlob() + +GetItemBlobs() + +IsBlobUsed() } class NetworkManagerStorageItf { @@ -62,8 +71,9 @@ classDiagram +GetActiveInstances() } + Database --|> CommonDatabase + CommonDatabase ..|> ImageManagerStorageItf Database ..|> StorageStateItf - Database ..|> ImageManagerStorageItf Database ..|> NetworkManagerStorageItf Database ..|> LauncherStorageItf ``` @@ -87,25 +97,17 @@ Stores instance storage and state quota information. **Primary Key**: `(itemID, subjectID, instance)` -#### 2. `imagemanager` +#### 2. `imagemanager_items` -Stores image manager item information and metadata. +Stores image manager deployable items. See [common database](../../common/database/database.md) for the columns +description. -| Column | Type | Description | -|-------------|---------|----------------------------------- | -| id | TEXT | Item identifier | -| type | TEXT | Item type (service/layer/component)| -| version | TEXT | Item version | -| state | TEXT | Item state | -| path | TEXT | File system path | -| totalSize | INTEGER | Total size in bytes | -| gid | INTEGER | Group ID | -| timestamp | INTEGER | Timestamp in nanoseconds | -| images | TEXT | JSON array of image metadata | +#### 3. `imagemanager_blobs` -**Primary Key**: `(id, version)` +Stores blobs referenced by deployable items. See [common database](../../common/database/database.md) for the columns +description. -#### 3. `networks` +#### 4. `networks` Stores network definitions. @@ -117,7 +119,7 @@ Stores network definitions. **Primary Key**: `(networkID)` -#### 4. `hosts` +#### 5. `hosts` Stores host information within networks. @@ -129,7 +131,7 @@ Stores host information within networks. **Primary Key**: `(networkID, nodeID)` -#### 5. `networkmanager_instances` +#### 6. `networkmanager_instances` Stores network instance configurations. @@ -146,7 +148,7 @@ Stores network instance configurations. **Primary Key**: `(itemID, subjectID, instance)` -#### 6. `launcher_instances` +#### 7. `launcher_instances` Stores launcher instance information. @@ -216,6 +218,12 @@ Helper functions in the anonymous namespace handle serialization: The database uses the common migration framework to support schema evolution across versions. Migration files should be placed in the configured migration path. +The current schema version is 2: + +- version 1 — adds `hostnames` column to `networkmanager_instances`; +- version 2 — drops the `imagemanager` table, creates `imagemanager_items`, `imagemanager_blobs` and the + `imagemanager_blobs_digest` index on `imagemanager_blobs(digest)`. + ## Configuration Database configuration is provided through `Config`: @@ -229,5 +237,5 @@ struct Config { ``` [storagestate-storage-itf]: https://github.com/aosedge/aos_core_lib_cpp/blob/main/src/core/cm/storagestate/itf/storage.hpp -[iamgemanager-storage-itf]: https://github.com/aosedge/aos_core_lib_cpp/blob/main/src/core/cm/imagemanager/itf/storage.hpp +[imagemanager-storage-itf]: https://github.com/aosedge/aos_core_lib_cpp/blob/main/src/core/common/imagemanager/itf/storage.hpp [launcher-storage-itf]: https://github.com/aosedge/aos_core_lib_cpp/blob/main/src/core/cm/launcher/itf/storage.hpp diff --git a/src/cm/database/migration/2_update.down.sql b/src/cm/database/migration/2_update.down.sql new file mode 100644 index 000000000..5bcd070b5 --- /dev/null +++ b/src/cm/database/migration/2_update.down.sql @@ -0,0 +1,16 @@ +BEGIN TRANSACTION; + +DROP TABLE IF EXISTS imagemanager_blobs; +DROP TABLE IF EXISTS imagemanager_items; + +CREATE TABLE IF NOT EXISTS imagemanager ( + itemID TEXT, + type TEXT, + version TEXT, + indexDigest TEXT, + state TEXT, + timestamp INTEGER, + PRIMARY KEY (itemID, version) +); + +COMMIT; diff --git a/src/cm/database/migration/2_update.up.sql b/src/cm/database/migration/2_update.up.sql new file mode 100644 index 000000000..590653ae7 --- /dev/null +++ b/src/cm/database/migration/2_update.up.sql @@ -0,0 +1,32 @@ +BEGIN TRANSACTION; + +DROP TABLE IF EXISTS imagemanager; + +CREATE TABLE IF NOT EXISTS imagemanager_items ( + itemID TEXT, + type TEXT, + version TEXT, + preinstalled INTEGER, + state TEXT, + errorAosCode INTEGER, + errorSysCode INTEGER, + errorMessage TEXT, + rootDigest TEXT, + timestamp INTEGER, + PRIMARY KEY (itemID, version) +); + +CREATE TABLE IF NOT EXISTS imagemanager_blobs ( + itemID TEXT, + version TEXT, + digest TEXT, + mediaType TEXT, + size INTEGER, + isLayer INTEGER, + diffID TEXT, + PRIMARY KEY (itemID, version, digest) +); + +CREATE INDEX IF NOT EXISTS imagemanager_blobs_digest ON imagemanager_blobs(digest); + +COMMIT; diff --git a/src/cm/database/tests/database.cpp b/src/cm/database/tests/database.cpp index b8502547a..b34274c04 100644 --- a/src/cm/database/tests/database.cpp +++ b/src/cm/database/tests/database.cpp @@ -162,19 +162,59 @@ launcher::InstanceInfo CreateLauncherInstanceInfo(const char* itemID, const char return info; } -imagemanager::ItemInfo CreateImageManagerItemInfo( - const char* itemID, const UpdateItemType& type, const char* version, const char* indexDigest, ItemState state) +common::imagemanager::DeployableItemInfo CreateDeployableItem(const char* itemID, const char* version, + ItemState state = ItemStateEnum::eInstalled, UpdateItemType type = UpdateItemTypeEnum::eService, + const char* rootDigest = "sha256:root", bool preinstalled = false, const Error& error = ErrorEnum::eNone) { - imagemanager::ItemInfo info; + common::imagemanager::DeployableItemInfo item; - info.mItemID = itemID; - info.mType = type; - info.mVersion = version; - info.mIndexDigest = indexDigest; - info.mState = state; - info.mTimestamp = Time::Now(); + item.mItemID = itemID; + item.mType = type; + item.mVersion = version; + item.mPreinstalled = preinstalled; + item.mState = state; + item.mError = error; + item.mRootDigest = rootDigest; + item.mTimestamp = Time::Now(); - return info; + return item; +} + +common::imagemanager::ItemBlobInfo CreateItemBlob( + const char* digest, bool isLayer = false, const char* diffID = "", uint64_t size = 1024) +{ + common::imagemanager::ItemBlobInfo blob; + + blob.mMediaType + = isLayer ? "application/vnd.oci.image.layer.v1.tar+gzip" : "application/vnd.oci.image.manifest.v1+json"; + blob.mDigest = digest; + blob.mSize = size; + blob.mIsLayer = isLayer; + blob.mDiffID = diffID; + + return blob; +} + +size_t CountTables(const std::string& dbPath, const std::string& tableName) +{ + Poco::Data::Session session("SQLite", dbPath); + size_t count = 0; + + session << "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?;", + Poco::Data::Keywords::bind(tableName), Poco::Data::Keywords::into(count), Poco::Data::Keywords::now; + + return count; +} + +int GetSchemaVersion(const std::string& dbPath) +{ + Poco::Data::Session session("SQLite", dbPath); + int version = 0; + + session << "SELECT version FROM SchemaVersion LIMIT 1;", Poco::Data::Keywords::into(version), + Poco::Data::Keywords::now; + + return version; } EnvVarInfo CreateEnvVarInfo(const char* name, const char* value, const Optional