Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion include/miniocpp/args.h
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ struct ComposeSource : public ObjectConditionalReadArgs {
utils::Multimap Headers() const;

private:
long object_size_ = -1;
std::optional<size_t> object_size_;
utils::Multimap headers_;
}; // struct ComposeSource

Expand Down
12 changes: 7 additions & 5 deletions include/miniocpp/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <map>
#include <memory>
#include <nlohmann/json_fwd.hpp>
#include <optional>
#include <ostream>
#include <string>
#include <type_traits>
Expand Down Expand Up @@ -293,17 +294,18 @@ struct SelectRequest {
struct SelectResult {
error::Error err = error::SUCCESS;
bool ended = false;
long int bytes_scanned = -1;
long int bytes_processed = -1;
long int bytes_returned = -1;
std::optional<long long> bytes_scanned;
std::optional<long long> bytes_processed;
std::optional<long long> bytes_returned;
std::string records;

SelectResult() : ended(true) {}

explicit SelectResult(error::Error err) : err(std::move(err)), ended(true) {}

SelectResult(long int bytes_scanned, long int bytes_processed,
long int bytes_returned)
SelectResult(std::optional<long long> bytes_scanned,
std::optional<long long> bytes_processed,
std::optional<long long> bytes_returned)
: bytes_scanned(bytes_scanned),
bytes_processed(bytes_processed),
bytes_returned(bytes_returned) {}
Expand Down
6 changes: 3 additions & 3 deletions src/args.cc
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ error::Error ComposeSource::BuildHeaders(size_t object_size,
}
}

object_size_ = static_cast<long>(object_size);
object_size_ = object_size;
headers_ = CopyHeaders();
if (!headers_.Contains("x-amz-copy-source-if-match")) {
headers_.Add("x-amz-copy-source-if-match", etag);
Expand All @@ -391,14 +391,14 @@ error::Error ComposeSource::BuildHeaders(size_t object_size,
}

size_t ComposeSource::ObjectSize() const {
if (object_size_ == -1) {
if (!object_size_.has_value()) {
std::cerr << "ABORT: ComposeSource::BuildHeaders() must be called prior to "
"this method invocation. This should not happen."
<< std::endl;
std::terminate();
}

return object_size_;
return *object_size_;
}

utils::Multimap ComposeSource::Headers() const {
Expand Down
2 changes: 1 addition & 1 deletion src/baseclient.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2049,7 +2049,7 @@ Result<StatObjectResponse> BaseClient::StatObject(StatObjectArgs args) {
resp.etag = utils::Trim(response->headers.GetFront("etag"), '"');

std::string value = response->headers.GetFront("content-length");
if (!value.empty()) resp.size = std::stol(value);
if (!value.empty()) resp.size = std::stoll(value);

value = response->headers.GetFront("last-modified");
if (!value.empty()) {
Expand Down
3 changes: 2 additions & 1 deletion src/client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1165,7 +1165,8 @@ Result<DownloadObjectResponse> Client::DownloadObject(DownloadObjectArgs args) {

std::string temp_filename =
args.filename + "." + curlpp::escape(etag) + ".part.minio";
std::ofstream fout(temp_filename, std::ios::trunc | std::ios::out);
std::ofstream fout(temp_filename,
std::ios::trunc | std::ios::out | std::ios::binary);
if (!fout.is_open()) {
return error::make<DownloadObjectResponse>("unable to open file " +
temp_filename);
Expand Down
12 changes: 6 additions & 6 deletions src/select.cc
Original file line number Diff line number Diff line change
Expand Up @@ -195,21 +195,21 @@ bool SelectHandler::process(const http::DataFunctionArgs& /* args */,
auto root = xdoc.select_node(xpath.c_str());
pugi::xpath_node text;
std::string value;
long int bytes_scanned = -1;
long int bytes_processed = -1;
long int bytes_returned = -1;
std::optional<long long> bytes_scanned;
std::optional<long long> bytes_processed;
std::optional<long long> bytes_returned;

text = root.node().select_node("BytesScanned/text()");
value = text.node().value();
if (!value.empty()) bytes_scanned = std::stol(value);
if (!value.empty()) bytes_scanned = std::stoll(value);

text = root.node().select_node("BytesProcessed/text()");
value = text.node().value();
if (!value.empty()) bytes_processed = std::stol(value);
if (!value.empty()) bytes_processed = std::stoll(value);

text = root.node().select_node("BytesReturned/text()");
value = text.node().value();
if (!value.empty()) bytes_returned = std::stol(value);
if (!value.empty()) bytes_returned = std::stoll(value);

cont = result_func_(
SelectResult(bytes_scanned, bytes_processed, bytes_returned));
Expand Down
92 changes: 92 additions & 0 deletions tests/tests.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <miniocpp/request.h>
#include <miniocpp/response.h>
#include <miniocpp/result.h>
#include <miniocpp/select.h>
#include <miniocpp/types.h>

using minio::Result;
Expand All @@ -38,6 +39,7 @@ using minio::Result;
#include <iosfwd>
#include <iostream>
#include <list>
#include <map>
#include <ostream>
#include <random>
#include <sstream>
Expand Down Expand Up @@ -102,6 +104,43 @@ std::string RandBucketName() {

std::string RandObjectName() { return RandomString(charset, 8); }

std::string PutUint32BigEndian(unsigned int v) {
std::string s(4, '\0');
s[0] = static_cast<char>((v >> 24) & 0xFF);
s[1] = static_cast<char>((v >> 16) & 0xFF);
s[2] = static_cast<char>((v >> 8) & 0xFF);
s[3] = static_cast<char>(v & 0xFF);
return s;
}

// Build a single S3 Select protocol frame (prelude + prelude CRC + headers
// + payload + message CRC) for the given event headers and XML payload.
std::string MakeSelectFrame(const std::map<std::string, std::string>& headers,
const std::string& payload) {
std::string headerdata;
for (const auto& [name, value] : headers) {
headerdata += static_cast<char>(name.length());
headerdata += name;
headerdata += static_cast<char>(7); // header value type: string
headerdata += static_cast<char>((value.length() >> 8) & 0xFF);
headerdata += static_cast<char>(value.length() & 0xFF);
headerdata += value;
}
headerdata += static_cast<char>(0); // header terminator

std::string data = headerdata + payload;
unsigned int total_length = 16 + static_cast<unsigned int>(data.length());
std::string prelude =
PutUint32BigEndian(total_length) +
PutUint32BigEndian(static_cast<unsigned int>(headerdata.length()));
std::string prelude_crc = PutUint32BigEndian(
static_cast<unsigned int>(minio::utils::CRC32(prelude)));
std::string message = prelude + prelude_crc + data;
std::string message_crc = PutUint32BigEndian(
static_cast<unsigned int>(minio::utils::CRC32(message)));
return message + message_crc;
}

struct MakeBucketError : public std::runtime_error {
MakeBucketError(std::string err) : runtime_error(err) {}
};
Expand Down Expand Up @@ -1537,6 +1576,58 @@ class Tests {
RemoveObject(bucket_name_, object_name);
}
} // TestAsyncOperations

// Regression test for SelectHandler Stats metric parsing: metrics larger
// than INT32_MAX must round-trip as exact long long values (any fallback
// to std::stol on 32-bit Windows LLP64 would truncate them). Uses a
// synthetic Stats event frame, independent of large objects.
void SelectStatsMetrics() {
std::cout << "SelectStatsMetrics()" << std::endl;

const long long scanned = 5000000000LL;
const long long processed = 6000000000LL;
const long long returned = 7000000000LL;

std::map<std::string, std::string> headers = {
{":message-type", "event"},
{":event-type", "Stats"},
};
std::string payload = "<Stats><BytesScanned>" + std::to_string(scanned) +
"</BytesScanned><BytesProcessed>" +
std::to_string(processed) +
"</BytesProcessed><BytesReturned>" +
std::to_string(returned) + "</BytesReturned></Stats>";

bool stats_delivered = false;
minio::s3::SelectHandler handler(
[&](minio::s3::SelectResult result) -> bool {
if (result.err) {
throw std::runtime_error("SelectStatsMetrics(): " +
result.err.String());
}
if (!result.bytes_scanned.has_value() ||
*result.bytes_scanned != scanned ||
!result.bytes_processed.has_value() ||
*result.bytes_processed != processed ||
!result.bytes_returned.has_value() ||
*result.bytes_returned != returned) {
throw std::runtime_error(
"SelectStatsMetrics(): unexpected metrics");
}
stats_delivered = true;
return true;
});

minio::http::DataFunctionArgs args;
args.datachunk = MakeSelectFrame(headers, payload);
if (!handler.DataFunction(args)) {
throw std::runtime_error("SelectStatsMetrics(): DataFunction failed");
}
if (!stats_delivered) {
throw std::runtime_error(
"SelectStatsMetrics(): Stats result was not delivered");
}
}
}; // class Tests

int main(int /*argc*/, char* /*argv*/[]) {
Expand Down Expand Up @@ -1596,6 +1687,7 @@ int main(int /*argc*/, char* /*argv*/[]) {
tests.SelectObjectContent();
tests.ListenBucketNotification();
tests.TestAsyncOperations();
tests.SelectStatsMetrics();

return EXIT_SUCCESS;
}
Loading