diff --git a/include/trx/detail/dtype_helpers.h b/include/trx/detail/dtype_helpers.h index e1a0d96..c77c869 100644 --- a/include/trx/detail/dtype_helpers.h +++ b/include/trx/detail/dtype_helpers.h @@ -3,6 +3,8 @@ #include +#include + #include #include #include @@ -16,25 +18,25 @@ namespace detail { // // MapType must be an Eigen::Map> type. template -inline void remap(MapType &map, void *data, int rows, int cols) { +inline void remap(MapType &map, void *data, long long rows, long long cols) { using Scalar = typename MapType::Scalar; new (&map) MapType(reinterpret_cast(data), rows, cols); // NOLINT } // Overload for const data pointers (read-only maps). template -inline void remap(MapType &map, const void *data, int rows, int cols) { +inline void remap(MapType &map, const void *data, long long rows, long long cols) { using Scalar = typename MapType::Scalar; new (&map) MapType(const_cast(reinterpret_cast(data)), rows, cols); // NOLINT } // Convenience overloads that unpack a (rows, cols) shape tuple. template -inline void remap(MapType &map, void *data, const std::tuple &shape) { +inline void remap(MapType &map, void *data, const std::tuple &shape) { remap(map, data, std::get<0>(shape), std::get<1>(shape)); } template -inline void remap(MapType &map, const void *data, const std::tuple &shape) { +inline void remap(MapType &map, const void *data, const std::tuple &shape) { remap(map, data, std::get<0>(shape), std::get<1>(shape)); } @@ -45,7 +47,7 @@ std::tuple _split_ext_with_dimensionality(const s template inline Eigen::Matrix _compute_lengths(const Eigen::MatrixBase
&offsets, - int nb_vertices) { + long long nb_vertices) { static_cast(nb_vertices); if (offsets.size() > 1) { const auto casted = offsets.template cast(); diff --git a/include/trx/trx.h b/include/trx/trx.h index dedf8e7..6674089 100644 --- a/include/trx/trx.h +++ b/include/trx/trx.h @@ -84,6 +84,16 @@ inline json::object _json_object(const json &value) { return json::object(); } +// This is a workaround for JSON files +// json11 stores every number as a double +// Json::int_value() truncates to a 32-bit int +// NB_VERTICES can exceed INT32_MAX +// so counts must be read through number_value(). +// Doubles represent integers exactly up to 2^53 +inline long long _json_int64(const json &value) { + return static_cast(value.number_value()); +} + inline json _json_set(const json &value, const std::string &key, const json &field) { auto obj = _json_object(value); obj[key] = field; @@ -247,8 +257,8 @@ template class TrxFile { public: struct GroupBackingInfo { std::string filename; - int rows = 0; - int cols = 0; + long long rows = 0; + long long cols = 0; std::string dtype; long long mem_offset = 0; }; @@ -269,8 +279,8 @@ template class TrxFile { // Member Functions() // TrxFile(int nb_vertices = 0, int nb_streamlines = 0); - TrxFile(int nb_vertices = 0, - int nb_streamlines = 0, + TrxFile(long long nb_vertices = 0, + long long nb_streamlines = 0, const TrxFile
*init_as = nullptr, std::string reference = ""); ~TrxFile(); @@ -308,7 +318,7 @@ template class TrxFile { * @param nb_vertices The number of vertices to keep * @param delete_dpg Remove data_per_group when resizing */ - void resize(int nb_streamlines = -1, int nb_vertices = -1, bool delete_dpg = false); + void resize(long long nb_streamlines = -1, long long nb_vertices = -1, bool delete_dpg = false); /** * @brief Save a TrxFile @@ -373,7 +383,7 @@ template class TrxFile { return static_cast(streamlines->_data.rows()); } if (header["NB_VERTICES"].is_number()) { - return static_cast(header["NB_VERTICES"].int_value()); + return static_cast(_json_int64(header["NB_VERTICES"])); } return 0; } @@ -386,7 +396,7 @@ template class TrxFile { return static_cast(streamlines->_lengths.size()); } if (header["NB_STREAMLINES"].is_number()) { - return static_cast(header["NB_STREAMLINES"].int_value()); + return static_cast(_json_int64(header["NB_STREAMLINES"])); } return 0; } @@ -521,11 +531,11 @@ template class TrxFile { * @param strs_start The start index of the streamline * @param pts_start The start index of the point * @param nb_strs_to_copy The number of streamlines to copy. If not set will copy all - * @return std::tuple A tuple representing the end of the copied streamlines and end + * @return std::tuple A tuple representing the end of the copied streamlines and end * of copied points */ - std::tuple - _copy_fixed_arrays_from(TrxFile
*trx, int strs_start = 0, int pts_start = 0, int nb_strs_to_copy = -1); + std::tuple + _copy_fixed_arrays_from(TrxFile
*trx, long long strs_start = 0, long long pts_start = 0, long long nb_strs_to_copy = -1); int len(); private: @@ -535,10 +545,10 @@ template class TrxFile { /** * @brief Get the real size of data (ignoring zeros of preallocation) * - * @return std::tuple A tuple representing the index of the last streamline and the + * @return std::tuple A tuple representing the index of the last streamline and the * total length of all the streamlines */ - std::tuple _get_real_len(); + std::tuple _get_real_len(); }; namespace detail { @@ -573,8 +583,8 @@ inline std::string make_unique_temp_path(const std::string &prefix) { struct TypedArray { std::string dtype; - int rows = 0; - int cols = 0; + long long rows = 0; + long long cols = 0; mio::shared_mmap_sink mmap; std::vector owned; @@ -1188,7 +1198,8 @@ TrxFile
::compute_group_connectivity(ConnectivityMeasure measure, const std:: if (b == this->group_backing_info_.end()) { continue; } - const size_t expected_ids = static_cast((std::max)(0, b->second.rows)) * static_cast((std::max)(0, b->second.cols)); + const size_t expected_ids = static_cast((std::max)(0, b->second.rows)) * + static_cast((std::max)(0, b->second.cols)); tmp_ids.resize(expected_ids); if (expected_ids > 0) { std::ifstream in(b->second.filename, std::ios::binary); @@ -1269,7 +1280,7 @@ void allocate_file(const std::string &path, std::size_t size); // Known limitations: only row-major order supported; shape uses tuple (sufficient for 2D); // dtype parameter is used only for byte-size computation. mio::shared_mmap_sink _create_memmap(std::string filename, - const std::tuple &shape, + const std::tuple &shape, const std::string &mode = "r", const std::string &dtype = "float32", long long offset = 0); @@ -1287,7 +1298,7 @@ std::string _generate_filename_from_data(const Eigen::MatrixBase
&arr, const */ template std::unique_ptr> -_initialize_empty_trx(int nb_streamlines, int nb_vertices, const TrxFile
*init_as = nullptr); +_initialize_empty_trx(long long nb_streamlines, long long nb_vertices, const TrxFile
*init_as = nullptr); template void ediff1d(Eigen::Matrix &lengths, diff --git a/include/trx/trx.tpp b/include/trx/trx.tpp index e9b2244..957db34 100644 --- a/include/trx/trx.tpp +++ b/include/trx/trx.tpp @@ -55,8 +55,8 @@ inline std::string folder_from_path(const std::string &elem_filename, const std: } template void materialize_matrix_map_and_unmap(MMappedMatrix &mapped_matrix) { - const int rows = mapped_matrix._matrix.rows(); - const int cols = mapped_matrix._matrix.cols(); + const long long rows = mapped_matrix._matrix.rows(); + const long long cols = mapped_matrix._matrix.cols(); const size_t n = static_cast(rows) * static_cast(cols); mapped_matrix._matrix_owned.resize(n); if (n > 0) { @@ -69,8 +69,8 @@ template void materialize_matrix_map_and_unmap(MMappedMatrix &ma } template void materialize_sequence_data_and_unmap(ArraySequence &sequence) { - const int rows = sequence._data.rows(); - const int cols = sequence._data.cols(); + const long long rows = sequence._data.rows(); + const long long cols = sequence._data.cols(); const size_t n = static_cast(rows) * static_cast(cols); sequence._data_owned.resize(n); if (n > 0) { @@ -225,7 +225,7 @@ template std::unique_ptr> TrxFile
::make_empty_like } template -TrxFile
::TrxFile(int nb_vertices, int nb_streamlines, const TrxFile
*init_as, std::string reference) { +TrxFile
::TrxFile(long long nb_vertices, long long nb_streamlines, const TrxFile
*init_as, std::string reference) { std::vector> affine(4); std::vector dimensions(3); @@ -289,15 +289,16 @@ TrxFile
::TrxFile(int nb_vertices, int nb_streamlines, const TrxFile
*ini json::object header_obj; header_obj["VOXEL_TO_RASMM"] = affine; header_obj["DIMENSIONS"] = dimensions; - header_obj["NB_VERTICES"] = nb_vertices; - header_obj["NB_STREAMLINES"] = nb_streamlines; + // json11 has no 64-bit integer constructor; store counts as double (exact up to 2^53). + header_obj["NB_VERTICES"] = static_cast(nb_vertices); + header_obj["NB_STREAMLINES"] = static_cast(nb_streamlines); this->header = json(header_obj); this->_copy_safe = true; } template -std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_vertices, const TrxFile
*init_as) { +std::unique_ptr> _initialize_empty_trx(long long nb_streamlines, long long nb_vertices, const TrxFile
*init_as) { auto trx = std::make_unique>(); std::string tmp_dir = make_temp_dir("trx"); @@ -306,8 +307,8 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve if (init_as != nullptr) { header = init_as->header; } - header = _json_set(header, "NB_VERTICES", nb_vertices); - header = _json_set(header, "NB_STREAMLINES", nb_streamlines); + header = _json_set(header, "NB_VERTICES", static_cast(nb_vertices)); + header = _json_set(header, "NB_STREAMLINES", static_cast(nb_streamlines)); std::string positions_dtype; std::string offsets_dtype; @@ -327,7 +328,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve std::string positions_filename(tmp_dir); positions_filename += "/positions.3." + positions_dtype; - std::tuple shape = std::make_tuple(nb_vertices, 3); + std::tuple shape = std::make_tuple(nb_vertices, static_cast(3)); trx->streamlines = std::make_unique>(); trx->streamlines->mmap_pos = trx::_create_memmap(positions_filename, shape, "w+", positions_dtype); @@ -337,7 +338,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve std::string offsets_filename(tmp_dir); offsets_filename += "/offsets." + offsets_dtype; - std::tuple shape_off = std::make_tuple(nb_streamlines + 1, 1); + std::tuple shape_off = std::make_tuple(nb_streamlines + 1, static_cast(1)); trx->streamlines->mmap_off = trx::_create_memmap(offsets_filename, shape_off, "w+", offsets_dtype); trx::detail::remap(trx->streamlines->_offsets, trx->streamlines->mmap_off.data(), shape_off); @@ -358,7 +359,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve } for (auto const &x : init_as->data_per_vertex) { - int rows, cols; + long long rows, cols; std::string dpv_dtype = dtype_from_scalar
(); Map> tmp_as = init_as->data_per_vertex.find(x.first)->second->_data; @@ -374,7 +375,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve dpv_filename = dpv_dirname + x.first + "." + std::to_string(cols) + "." + dpv_dtype; } - std::tuple dpv_shape = std::make_tuple(rows, cols); + std::tuple dpv_shape = std::make_tuple(rows, cols); trx->data_per_vertex[x.first] = std::make_unique>(); trx->data_per_vertex[x.first]->mmap_pos = trx::_create_memmap(dpv_filename, dpv_shape, "w+", dpv_dtype); trx::detail::remap( @@ -382,14 +383,14 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve trx::detail::remap(trx->data_per_vertex[x.first]->_offsets, trx->streamlines->_offsets.data(), - int(trx->streamlines->_offsets.rows()), - int(trx->streamlines->_offsets.cols())); + trx->streamlines->_offsets.rows(), + trx->streamlines->_offsets.cols()); trx->data_per_vertex[x.first]->_lengths = trx->streamlines->_lengths; } for (auto const &x : init_as->data_per_streamline) { std::string dps_dtype = dtype_from_scalar
(); - int rows, cols; + long long rows, cols; Map> tmp_as = init_as->data_per_streamline.find(x.first)->second->_matrix; std::string dps_filename; @@ -404,7 +405,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve dps_filename = dps_dirname + x.first + "." + std::to_string(cols) + "." + dps_dtype; } - std::tuple dps_shape = std::make_tuple(rows, cols); + std::tuple dps_shape = std::make_tuple(rows, cols); trx->data_per_streamline[x.first] = std::make_unique>(); trx->data_per_streamline[x.first]->mmap = trx::_create_memmap(dps_filename, dps_shape, std::string("w+"), dps_dtype); @@ -452,22 +453,22 @@ TrxFile
::_create_trx_from_pointer(json header, long long size = std::get<1>(x->second); if (base == "positions" && (folder.empty() || folder == ".")) { - const auto nb_vertices = static_cast(trx->header["NB_VERTICES"].int_value()); + const auto nb_vertices = _json_int64(trx->header["NB_VERTICES"]); const auto expected = nb_vertices * 3; if (size != expected || dim != 3) { throw TrxFormatError("Wrong data size/dimensionality: size=" + std::to_string(size) + " expected=" + std::to_string(expected) + " dim=" + std::to_string(dim) + " filename=" + elem_filename); } - std::tuple shape = std::make_tuple(static_cast(trx->header["NB_VERTICES"].int_value()), 3); + std::tuple shape = std::make_tuple(nb_vertices, static_cast(3)); trx->streamlines->mmap_pos = trx::_create_memmap(filename, shape, "r+", ext, mem_address); trx::detail::remap(trx->streamlines->_data, trx->streamlines->mmap_pos.data(), shape); } else if (base == "offsets" && (folder.empty() || folder == ".")) { - const auto nb_streamlines = static_cast(trx->header["NB_STREAMLINES"].int_value()); - const auto nb_vertices = static_cast(trx->header["NB_VERTICES"].int_value()); + const auto nb_streamlines = _json_int64(trx->header["NB_STREAMLINES"]); + const auto nb_vertices = static_cast(_json_int64(trx->header["NB_VERTICES"])); const auto expected = nb_streamlines + 1; const bool missing_sentinel = (size == nb_streamlines && dim == 1); if ((size != expected && !missing_sentinel) || dim != 1) { @@ -475,17 +476,17 @@ TrxFile
::_create_trx_from_pointer(json header, std::to_string(expected) + " dim=" + std::to_string(dim) + " filename=" + elem_filename); } - const int nb_str = static_cast(trx->header["NB_STREAMLINES"].int_value()); - const int offsets_rows = missing_sentinel ? (nb_str + 1) : static_cast(size); - std::tuple shape = std::make_tuple(offsets_rows, 1); + const long long nb_str = nb_streamlines; + const long long offsets_rows = missing_sentinel ? (nb_str + 1) : static_cast(size); + std::tuple shape = std::make_tuple(offsets_rows, static_cast(1)); trx->streamlines->mmap_off = - trx::_create_memmap(filename, std::make_tuple(static_cast(size), 1), "r+", ext, mem_address); + trx::_create_memmap(filename, std::make_tuple(static_cast(size), static_cast(1)), "r+", ext, mem_address); if (ext == "uint64") { if (missing_sentinel) { trx->streamlines->_offsets_owned.resize(static_cast(offsets_rows)); auto *src = reinterpret_cast(trx->streamlines->mmap_off.data()); // NOLINT - for (int i = 0; i < static_cast(size); ++i) { + for (long long i = 0; i < size; ++i) { trx->streamlines->_offsets_owned[static_cast(i)] = src[i]; } trx->streamlines->_offsets_owned.back() = nb_vertices; @@ -496,7 +497,7 @@ TrxFile
::_create_trx_from_pointer(json header, } else if (ext == "uint32") { trx->streamlines->_offsets_owned.resize(static_cast(offsets_rows)); auto *src = reinterpret_cast(trx->streamlines->mmap_off.data()); // NOLINT - for (int i = 0; i < static_cast(size); ++i) { + for (long long i = 0; i < size; ++i) { trx->streamlines->_offsets_owned[static_cast(i)] = static_cast(src[i]); } if (missing_sentinel) { @@ -509,18 +510,19 @@ TrxFile
::_create_trx_from_pointer(json header, Matrix offsets = trx->streamlines->_offsets; trx->streamlines->_lengths = - trx::detail::_compute_lengths(offsets, static_cast(trx->header["NB_VERTICES"].int_value())); + trx::detail::_compute_lengths(offsets, _json_int64(trx->header["NB_VERTICES"])); } else if (folder == "dps") { - std::tuple shape; + std::tuple shape; trx->data_per_streamline[base] = std::make_unique>(); - int nb_scalar = size / static_cast(trx->header["NB_STREAMLINES"].int_value()); + const long long nb_streamlines = _json_int64(trx->header["NB_STREAMLINES"]); + const long long nb_scalar = nb_streamlines > 0 ? size / nb_streamlines : 0; - if (size % static_cast(trx->header["NB_STREAMLINES"].int_value()) != 0 || nb_scalar != dim) { + if (nb_streamlines == 0 || size % nb_streamlines != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dps size/dimensionality"); } else { - shape = std::make_tuple(static_cast(trx->header["NB_STREAMLINES"].int_value()), nb_scalar); + shape = std::make_tuple(nb_streamlines, nb_scalar); } trx->data_per_streamline[base]->mmap = trx::_create_memmap(filename, shape, "r+", ext, mem_address); const std::string expected_dtype = dtype_from_scalar
(); @@ -538,14 +540,15 @@ TrxFile
::_create_trx_from_pointer(json header, } else if (folder == "dpv") { - std::tuple shape; + std::tuple shape; trx->data_per_vertex[base] = std::make_unique>(); - int nb_scalar = size / static_cast(trx->header["NB_VERTICES"].int_value()); + const long long nb_vertices = _json_int64(trx->header["NB_VERTICES"]); + const long long nb_scalar = nb_vertices > 0 ? size / nb_vertices : 0; - if (size % static_cast(trx->header["NB_VERTICES"].int_value()) != 0 || nb_scalar != dim) { + if (nb_vertices == 0 || size % nb_vertices != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dpv size/dimensionality"); } else { - shape = std::make_tuple(static_cast(trx->header["NB_VERTICES"].int_value()), nb_scalar); + shape = std::make_tuple(nb_vertices, nb_scalar); } trx->data_per_vertex[base]->mmap_pos = trx::_create_memmap(filename, shape, "r+", ext, mem_address); const std::string expected_dtype = dtype_from_scalar
(); @@ -560,19 +563,19 @@ TrxFile
::_create_trx_from_pointer(json header, } trx::detail::remap(trx->data_per_vertex[base]->_offsets, trx->streamlines->_offsets.data(), - int(trx->streamlines->_offsets.rows()), - int(trx->streamlines->_offsets.cols())); + trx->streamlines->_offsets.rows(), + trx->streamlines->_offsets.cols()); trx->data_per_vertex[base]->_lengths = trx->streamlines->_lengths; } else if (folder.rfind("dpg", 0) == 0) { - std::tuple shape; + std::tuple shape; if (size != dim) { throw TrxFormatError("Wrong dpg size/dimensionality"); } else { - shape = std::make_tuple(1, static_cast(size)); + shape = std::make_tuple(static_cast(1), static_cast(size)); } std::string data_name = path_basename(base); @@ -600,11 +603,11 @@ TrxFile
::_create_trx_from_pointer(json header, } else if (folder == "groups") { - std::tuple shape; + std::tuple shape; if (dim != 1) { throw TrxFormatError("Wrong group dimensionality"); } else { - shape = std::make_tuple(static_cast(size), 1); + shape = std::make_tuple(static_cast(size), static_cast(1)); } trx->groups[base] = nullptr; typename TrxFile
::GroupBackingInfo info; @@ -635,15 +638,16 @@ template std::unique_ptr> TrxFile
::deepcopy() { // Determine effective counts (handle sliced/non-copy-safe data) json tmp_header = this->header; - int nb_streamlines, nb_vertices; + long long nb_streamlines = 0; + long long nb_vertices = 0; if (!this->_copy_safe) { - nb_streamlines = static_cast(this->num_streamlines()); - nb_vertices = static_cast(this->streamlines->_data.size() / 3); - tmp_header = _json_set(tmp_header, "NB_STREAMLINES", nb_streamlines); - tmp_header = _json_set(tmp_header, "NB_VERTICES", nb_vertices); + nb_streamlines = static_cast(this->num_streamlines()); + nb_vertices = static_cast(this->streamlines->_data.size() / 3); + tmp_header = _json_set(tmp_header, "NB_STREAMLINES", static_cast(nb_streamlines)); + tmp_header = _json_set(tmp_header, "NB_VERTICES", static_cast(nb_vertices)); } else { - nb_streamlines = tmp_header["NB_STREAMLINES"].int_value(); - nb_vertices = tmp_header["NB_VERTICES"].int_value(); + nb_streamlines = _json_int64(tmp_header["NB_STREAMLINES"]); + nb_vertices = _json_int64(tmp_header["NB_VERTICES"]); } // Allocate a fresh TrxFile with memory-mapped storage @@ -694,12 +698,12 @@ template std::unique_ptr> TrxFile
::deepcopy() { } for (auto const &kv : this->groups) { std::string group_dtype = dtype_from_scalar(); - int rows = static_cast(kv.second->_matrix.rows()); - int cols = static_cast(kv.second->_matrix.cols()); + const long long rows = static_cast(kv.second->_matrix.rows()); + const long long cols = static_cast(kv.second->_matrix.cols()); std::string group_filename = groups_dirname + kv.first; group_filename = _generate_filename_from_data(kv.second->_matrix, group_filename); - std::tuple group_shape = std::make_tuple(rows, cols); + std::tuple group_shape = std::make_tuple(rows, cols); copy->groups[kv.first] = std::make_unique>(); copy->groups[kv.first]->mmap = _create_memmap(group_filename, group_shape, "w+", group_dtype); trx::detail::remap(copy->groups[kv.first]->_matrix, copy->groups[kv.first]->mmap.data(), rows, cols); @@ -717,12 +721,12 @@ template std::unique_ptr> TrxFile
::deepcopy() { } for (auto const &field : group_kv.second) { std::string dpg_dtype = dtype_from_scalar
(); - int rows = static_cast(field.second->_matrix.rows()); - int cols = static_cast(field.second->_matrix.cols()); + const long long rows = static_cast(field.second->_matrix.rows()); + const long long cols = static_cast(field.second->_matrix.cols()); std::string dpg_filename = dpg_subdirname + SEPARATOR + field.first; dpg_filename = _generate_filename_from_data(field.second->_matrix, dpg_filename); - std::tuple dpg_shape = std::make_tuple(rows, cols); + std::tuple dpg_shape = std::make_tuple(rows, cols); copy->data_per_group[group_kv.first][field.first] = std::make_unique>(); copy->data_per_group[group_kv.first][field.first]->mmap = _create_memmap(dpg_filename, dpg_shape, "w+", dpg_dtype); @@ -739,34 +743,36 @@ template std::unique_ptr> TrxFile
::deepcopy() { /// Compute the used range in a preallocated TrxFile by finding the last non-zero length. /// Returns (nb_streamlines_used, nb_vertices_used). -template std::tuple TrxFile
::_get_real_len() { +template std::tuple TrxFile
::_get_real_len() { if (this->streamlines->_lengths.size() == 0) - return std::make_tuple(0, 0); + return std::make_tuple(static_cast(0), static_cast(0)); - int last_elem_pos = trx::detail::_dichotomic_search(this->streamlines->_lengths); + long long last_elem_pos = trx::detail::_dichotomic_search(this->streamlines->_lengths); if (last_elem_pos != -1) { - int strs_end = last_elem_pos + 1; - int pts_end = this->streamlines->_lengths(Eigen::seq(0, last_elem_pos), 0).sum(); + const long long strs_end = last_elem_pos + 1; + const long long pts_end = + this->streamlines->_lengths(Eigen::seq(0, last_elem_pos), 0).template cast().sum(); return std::make_tuple(strs_end, pts_end); } - return std::make_tuple(0, 0); + return std::make_tuple(static_cast(0), static_cast(0)); } template -std::tuple -TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, int strs_start, int pts_start, int nb_strs_to_copy) { - int curr_strs_len, curr_pts_len; +std::tuple +TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, long long strs_start, long long pts_start, long long nb_strs_to_copy) { + long long curr_strs_len = 0; + long long curr_pts_len = 0; if (nb_strs_to_copy == -1) { - std::tuple curr = this->_get_real_len(); + std::tuple curr = this->_get_real_len(); curr_strs_len = std::get<0>(curr); curr_pts_len = std::get<1>(curr); } else { curr_strs_len = nb_strs_to_copy; - curr_pts_len = trx->streamlines->_lengths(Eigen::seq(0, curr_strs_len - 1)).sum(); + curr_pts_len = trx->streamlines->_lengths(Eigen::seq(0, curr_strs_len - 1)).template cast().sum(); } if (pts_start == -1) { @@ -776,8 +782,8 @@ TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, int strs_start, int pts_s strs_start = 0; } - int strs_end = strs_start + curr_strs_len; - int pts_end = pts_start + curr_pts_len; + const long long strs_end = strs_start + curr_strs_len; + const long long pts_end = pts_start + curr_pts_len; if (curr_pts_len == 0) return std::make_tuple(strs_start, pts_start); @@ -862,21 +868,21 @@ void TrxFile
::_cleanup_temporary_directory() { template // Caveats: downsizing vertices is not supported; reducing streamlines truncates data; same-size // resize is a no-op. -void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { +void TrxFile
::resize(long long nb_streamlines, long long nb_vertices, bool delete_dpg) { if (!this->_copy_safe) { throw TrxArgumentError("Cannot resize a sliced dataset."); } - std::tuple sp_end = this->_get_real_len(); - int strs_end = std::get<0>(sp_end); - int ptrs_end = std::get<1>(sp_end); + std::tuple sp_end = this->_get_real_len(); + long long strs_end = std::get<0>(sp_end); + long long ptrs_end = std::get<1>(sp_end); if (nb_streamlines != -1 && nb_streamlines < strs_end) { strs_end = nb_streamlines; } if (nb_vertices == -1) { - ptrs_end = this->streamlines->_lengths.sum(); + ptrs_end = this->streamlines->_lengths.template cast().sum(); nb_vertices = ptrs_end; } else if (nb_vertices < ptrs_end) { return; @@ -886,14 +892,14 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { nb_streamlines = strs_end; } - if (nb_streamlines == this->header["NB_STREAMLINES"].int_value() && - nb_vertices == this->header["NB_VERTICES"].int_value()) { + if (nb_streamlines == _json_int64(this->header["NB_STREAMLINES"]) && + nb_vertices == _json_int64(this->header["NB_VERTICES"])) { return; } auto trx = _initialize_empty_trx(nb_streamlines, nb_vertices, this); - if (nb_streamlines < this->header["NB_STREAMLINES"].int_value()) + if (nb_streamlines < _json_int64(this->header["NB_STREAMLINES"])) trx->_copy_fixed_arrays_from(this, -1, -1, nb_streamlines); else { trx->_copy_fixed_arrays_from(this); @@ -926,7 +932,7 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { // std::cout << "Cols " << keep_rows.at(1) << std::endl; Matrix tmp = this->groups[x.first]->_matrix(keep_rows, keep_cols); - std::tuple group_shape = std::make_tuple(tmp.size(), 1); + std::tuple group_shape = std::make_tuple(static_cast(tmp.size()), static_cast(1)); trx->groups[x.first] = std::make_unique>(); trx->groups[x.first]->mmap = trx::_create_memmap(group_name, group_shape, "w+", group_dtype); @@ -965,8 +971,9 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { std::string dpg_filename = dpg_subdir + SEPARATOR + y.first; dpg_filename = _generate_filename_from_data(this->data_per_group[x.first][y.first]->_matrix, dpg_filename); - std::tuple dpg_shape = std::make_tuple(this->data_per_group[x.first][y.first]->_matrix.rows(), - this->data_per_group[x.first][y.first]->_matrix.cols()); + std::tuple dpg_shape = + std::make_tuple(static_cast(this->data_per_group[x.first][y.first]->_matrix.rows()), + static_cast(this->data_per_group[x.first][y.first]->_matrix.cols())); if (trx->data_per_group[x.first].find(y.first) == trx->data_per_group[x.first].end()) { trx->data_per_group[x.first][y.first] = std::make_unique>(); @@ -1184,13 +1191,15 @@ template void TrxFile
::normalize_for_save() { if (used_vertices > data_rows) { throw TrxFormatError("TRX offsets exceed positions row count"); } - if (used_vertices > static_cast(std::numeric_limits::max()) || - used_streamlines > static_cast(std::numeric_limits::max())) { - throw TrxFormatError("TRX normalize_for_save exceeds supported int range"); + // json11 has no 64-bit integer constructor; store counts as double + // here, we check that the counts are within the exact range representable in double + constexpr uint64_t kMaxExactInDouble = (1ULL << 53); + if (used_vertices > kMaxExactInDouble || static_cast(used_streamlines) > kMaxExactInDouble) { + throw TrxFormatError("TRX normalize_for_save exceeds the 2^53 range representable in the JSON header"); } if (used_streamlines < total_streamlines || used_vertices < data_rows) { - this->resize(static_cast(used_streamlines), static_cast(used_vertices)); + this->resize(static_cast(used_streamlines), static_cast(used_vertices)); } const size_t normalized_streamlines = this->num_streamlines(); @@ -1207,8 +1216,8 @@ template void TrxFile
::normalize_for_save() { this->streamlines->_lengths(static_cast(i)) = static_cast(diff); } - this->header = _json_set(this->header, "NB_STREAMLINES", static_cast(normalized_streamlines)); - this->header = _json_set(this->header, "NB_VERTICES", static_cast(this->num_vertices())); + this->header = _json_set(this->header, "NB_STREAMLINES", static_cast(normalized_streamlines)); + this->header = _json_set(this->header, "NB_VERTICES", static_cast(this->num_vertices())); } template void TrxFile
::save(const std::string &filename, const TrxSaveOptions &options) { @@ -1224,13 +1233,13 @@ template void TrxFile
::save(const std::string &filename, const throw TrxFormatError("Cannot save TRX without offsets data"); } if (save_trx->header["NB_STREAMLINES"].is_number()) { - const auto nb_streamlines = static_cast(save_trx->header["NB_STREAMLINES"].int_value()); + const auto nb_streamlines = static_cast(_json_int64(save_trx->header["NB_STREAMLINES"])); if (save_trx->streamlines->_offsets.size() != static_cast(nb_streamlines + 1)) { throw TrxFormatError("TRX offsets size does not match NB_STREAMLINES"); } } if (save_trx->header["NB_VERTICES"].is_number()) { - const auto nb_vertices = static_cast(save_trx->header["NB_VERTICES"].int_value()); + const auto nb_vertices = static_cast(_json_int64(save_trx->header["NB_VERTICES"])); const auto last = static_cast(save_trx->num_vertices()); if (last != nb_vertices) { throw TrxFormatError("TRX offsets sentinel does not match NB_VERTICES"); @@ -1419,7 +1428,7 @@ void TrxFile
::add_dps_from_vector(const std::string &name, const std::string if (this->streamlines) { nb_streamlines = static_cast(this->streamlines->_lengths.size()); } else if (this->header["NB_STREAMLINES"].is_number()) { - nb_streamlines = static_cast(this->header["NB_STREAMLINES"].int_value()); + nb_streamlines = static_cast(_json_int64(this->header["NB_STREAMLINES"])); } if (values.size() != nb_streamlines) { @@ -1443,10 +1452,10 @@ void TrxFile
::add_dps_from_vector(const std::string &name, const std::string this->data_per_streamline.erase(existing); } - const int rows = static_cast(nb_streamlines); - const int cols = 1; - std::tuple shape = std::make_tuple(rows, cols); - const size_t n = static_cast(rows * cols); + const long long rows = static_cast(nb_streamlines); + const long long cols = 1; + std::tuple shape = std::make_tuple(rows, cols); + const size_t n = static_cast(rows) * static_cast(cols); auto matrix = std::make_unique>(); matrix->mmap = trx::_create_memmap(dps_filename, shape, "w+", dtype_norm); @@ -1455,7 +1464,7 @@ void TrxFile
::add_dps_from_vector(const std::string &name, const std::string if (dtype_norm == expected_dtype) { // On-disk dtype matches DT: memory-map directly and write as DT. trx::detail::remap(matrix->_matrix, matrix->mmap.data(), rows, cols); - for (int i = 0; i < rows; ++i) { + for (long long i = 0; i < rows; ++i) { matrix->_matrix(i, 0) = static_cast
(values[static_cast(i)]); } } else { @@ -1510,7 +1519,7 @@ void TrxFile
::add_dpv_from_vector(const std::string &name, const std::string if (this->streamlines) { nb_vertices = static_cast(this->streamlines->_data.rows()); } else if (this->header["NB_VERTICES"].is_number()) { - nb_vertices = static_cast(this->header["NB_VERTICES"].int_value()); + nb_vertices = static_cast(_json_int64(this->header["NB_VERTICES"])); } if (values.size() != nb_vertices) { @@ -1534,10 +1543,10 @@ void TrxFile
::add_dpv_from_vector(const std::string &name, const std::string this->data_per_vertex.erase(existing); } - const int rows = static_cast(nb_vertices); - const int cols = 1; - std::tuple shape = std::make_tuple(rows, cols); - const size_t n = static_cast(rows * cols); + const long long rows = static_cast(nb_vertices); + const long long cols = 1; + std::tuple shape = std::make_tuple(rows, cols); + const size_t n = static_cast(rows) * static_cast(cols); auto seq = std::make_unique>(); seq->mmap_pos = trx::_create_memmap(dpv_filename, shape, "w+", dtype_norm); @@ -1546,7 +1555,7 @@ void TrxFile
::add_dpv_from_vector(const std::string &name, const std::string if (dtype_norm == expected_dtype) { // On-disk dtype matches DT: memory-map directly and write as DT. trx::detail::remap(seq->_data, seq->mmap_pos.data(), rows, cols); - for (int i = 0; i < rows; ++i) { + for (long long i = 0; i < rows; ++i) { seq->_data(i, 0) = static_cast
(values[static_cast(i)]); } } else { @@ -1595,7 +1604,7 @@ void TrxFile
::add_group_from_indices(const std::string &name, const std::vec if (this->streamlines) { nb_streamlines = static_cast(this->streamlines->_lengths.size()); } else if (this->header["NB_STREAMLINES"].is_number()) { - nb_streamlines = static_cast(this->header["NB_STREAMLINES"].int_value()); + nb_streamlines = static_cast(_json_int64(this->header["NB_STREAMLINES"])); } for (const auto idx : indices) { @@ -1624,14 +1633,14 @@ void TrxFile
::add_group_from_indices(const std::string &name, const std::vec this->group_backing_info_.erase(backing); } - const int rows = static_cast(indices.size()); - const int cols = 1; - std::tuple shape = std::make_tuple(rows, cols); + const long long rows = static_cast(indices.size()); + const long long cols = 1; + std::tuple shape = std::make_tuple(rows, cols); auto group = std::make_unique>(); group->mmap = trx::_create_memmap(group_filename, shape, "w+", "uint32"); trx::detail::remap(group->_matrix, group->mmap.data(), shape); - for (int i = 0; i < rows; ++i) { + for (long long i = 0; i < rows; ++i) { group->_matrix(i, 0) = indices[static_cast(i)]; } this->groups[name] = std::move(group); @@ -2040,11 +2049,11 @@ template void TrxStream::finalize(const std::string &filename, Trx const size_t nb_streamlines = lengths_.size(); const size_t nb_vertices = total_vertices_; - TrxFile
trx(static_cast(nb_vertices), static_cast(nb_streamlines)); + TrxFile
trx(static_cast(nb_vertices), static_cast(nb_streamlines)); json header_out = header; - header_out = _json_set(header_out, "NB_VERTICES", static_cast(nb_vertices)); - header_out = _json_set(header_out, "NB_STREAMLINES", static_cast(nb_streamlines)); + header_out = _json_set(header_out, "NB_VERTICES", static_cast(nb_vertices)); + header_out = _json_set(header_out, "NB_STREAMLINES", static_cast(nb_streamlines)); trx.header = header_out; auto &positions = trx.streamlines->_data; @@ -2180,8 +2189,8 @@ inline void TrxStream::finalize_directory_impl(const std::string &directory, boo ec.clear(); json header_out = header; - header_out = _json_set(header_out, "NB_VERTICES", static_cast(nb_vertices)); - header_out = _json_set(header_out, "NB_STREAMLINES", static_cast(nb_streamlines)); + header_out = _json_set(header_out, "NB_VERTICES", static_cast(nb_vertices)); + header_out = _json_set(header_out, "NB_STREAMLINES", static_cast(nb_streamlines)); const std::string header_path = directory + SEPARATOR + "header.json"; std::ofstream out_header(header_path, std::ios::out | std::ios::trunc); if (!out_header.is_open()) { @@ -2556,22 +2565,22 @@ void TrxFile
::add_dpv_from_tsf(const std::string &name, const std::string &d this->data_per_vertex.erase(existing); } - const int rows = static_cast(nb_vertices); - const int cols = 1; - std::tuple shape = std::make_tuple(rows, cols); + const long long rows = static_cast(nb_vertices); + const long long cols = 1; + std::tuple shape = std::make_tuple(rows, cols); auto seq = std::make_unique>(); seq->mmap_pos = trx::_create_memmap(dpv_filename, shape, "w+", dtype_norm); trx::detail::remap(seq->_data, seq->mmap_pos.data(), rows, cols); - for (int i = 0; i < rows; ++i) { + for (long long i = 0; i < rows; ++i) { seq->_data(i, 0) = static_cast
(values[static_cast(i)]); } trx::detail::remap(seq->_offsets, this->streamlines->_offsets.data(), - static_cast(this->streamlines->_offsets.rows()), - static_cast(this->streamlines->_offsets.cols())); + this->streamlines->_offsets.rows(), + this->streamlines->_offsets.cols()); seq->_lengths = this->streamlines->_lengths; this->data_per_vertex[name] = std::move(seq); @@ -2892,11 +2901,11 @@ template const MMappedMatrix *TrxFile
::get_group_mem if (b == this->group_backing_info_.end()) { return nullptr; } - const int rows = b->second.rows; - const int cols = b->second.cols; - std::tuple shape = std::make_tuple(rows, cols); + const long long rows = b->second.rows; + const long long cols = b->second.cols; + std::tuple shape = std::make_tuple(rows, cols); it->second = std::make_unique>(); - const size_t n = static_cast((std::max)(0, rows)) * static_cast((std::max)(0, cols)); + const size_t n = static_cast((std::max)(0, rows)) * static_cast((std::max)(0, cols)); it->second->_matrix_owned.resize(n); if (n > 0) { @@ -3036,12 +3045,12 @@ void TrxFile
::add_dpg_from_vector(const std::string &group, auto &group_map = this->data_per_group[group]; group_map.erase(name); - std::tuple shape = std::make_tuple(rows, cols); + std::tuple shape = std::make_tuple(static_cast(rows), static_cast(cols)); group_map[name] = std::make_unique>(); group_map[name]->mmap = _create_memmap(dpg_filename, shape, "w+", dtype_norm); trx::detail::remap(group_map[name]->_matrix, group_map[name]->mmap.data(), rows, cols); - for (int i = 0; i < rows * cols; ++i) { + for (long long i = 0; i < rows * cols; ++i) { group_map[name]->_matrix(i) = static_cast
(values[static_cast(i)]); } } @@ -3161,19 +3170,20 @@ std::unique_ptr> TrxFile
::subset_streamlines(const std::vectormake_empty_like(); } - std::vector old_to_new(nb_streamlines, -1); + std::vector old_to_new(nb_streamlines, -1); size_t total_vertices = 0; for (size_t i = 0; i < selected.size(); ++i) { const uint32_t idx = selected[i]; - old_to_new[idx] = static_cast(i); + old_to_new[idx] = static_cast(i); const uint64_t start = offsets[idx]; const uint64_t end = offsets[idx + 1]; total_vertices += static_cast(end - start); } - auto out = std::make_unique>(static_cast(total_vertices), static_cast(selected.size()), this); - out->header = _json_set(this->header, "NB_VERTICES", static_cast(total_vertices)); - out->header = _json_set(out->header, "NB_STREAMLINES", static_cast(selected.size())); + auto out = std::make_unique>( + static_cast(total_vertices), static_cast(selected.size()), this); + out->header = _json_set(this->header, "NB_VERTICES", static_cast(total_vertices)); + out->header = _json_set(out->header, "NB_STREAMLINES", static_cast(selected.size())); auto &out_positions = out->streamlines->_data; auto &out_offsets = out->streamlines->_offsets; @@ -3236,7 +3246,7 @@ std::unique_ptr> TrxFile
::subset_streamlines(const std::vector= old_to_new.size()) { continue; } - const int new_idx = old_to_new[old_idx]; + const long long new_idx = old_to_new[old_idx]; if (new_idx >= 0) { indices.push_back(static_cast(new_idx)); } @@ -3273,8 +3283,8 @@ std::unique_ptr> TrxFile
::subset_streamlines(const std::vector_matrix, dpg_filename); - std::tuple dpg_shape = - std::make_tuple(field_kv.second->_matrix.rows(), field_kv.second->_matrix.cols()); + std::tuple dpg_shape = std::make_tuple( + static_cast(field_kv.second->_matrix.rows()), static_cast(field_kv.second->_matrix.cols())); out->data_per_group[group_name][field_name] = std::make_unique>(); out->data_per_group[group_name][field_name]->mmap = _create_memmap(dpg_filename, dpg_shape, "w+", dpg_dtype); diff --git a/src/trx.cpp b/src/trx.cpp index 59d8823..d8aa2d9 100644 --- a/src/trx.cpp +++ b/src/trx.cpp @@ -180,7 +180,7 @@ std::array read_xyz_as_double(const TypedArray &positions, size_t row throw TrxDTypeError("Unsupported positions dtype for streamline extraction: " + positions.dtype); } -TypedArray make_typed_array(const std::string &filename, int rows, int cols, const std::string &dtype) { +TypedArray make_typed_array(const std::string &filename, long long rows, long long cols, const std::string &dtype) { TypedArray array; array.dtype = dtype; array.rows = rows; @@ -369,7 +369,7 @@ size_t AnyTrxFile::num_vertices() const { return static_cast(positions.rows); } if (header["NB_VERTICES"].is_number()) { - return static_cast(header["NB_VERTICES"].int_value()); + return static_cast(_json_int64(header["NB_VERTICES"])); } return 0; } @@ -379,7 +379,7 @@ size_t AnyTrxFile::num_streamlines() const { return lengths.size(); } if (header["NB_STREAMLINES"].is_number()) { - return static_cast(header["NB_STREAMLINES"].int_value()); + return static_cast(_json_int64(header["NB_STREAMLINES"])); } return 0; } @@ -469,8 +469,8 @@ AnyTrxFile AnyTrxFile::load_from_zip(const std::string &filename) { throw TrxFormatError("Missing NB_VERTICES or NB_STREAMLINES in header.json"); } - const int nb_vertices = trx.header["NB_VERTICES"].int_value(); - const int nb_streamlines = trx.header["NB_STREAMLINES"].int_value(); + const long long nb_vertices = _json_int64(trx.header["NB_VERTICES"]); + const long long nb_streamlines = _json_int64(trx.header["NB_STREAMLINES"]); // Build the offset map ONCE in a single pass over the archive, then reuse // it for every entry. The previous approach called find_uncompressed_zip_entry_offset @@ -504,7 +504,7 @@ AnyTrxFile AnyTrxFile::load_from_zip(const std::string &filename) { const int dtype_size = trx::detail::_sizeof_dtype(ext); long long count_elems = (dtype_size > 0) ? (raw_size_bytes / dtype_size) : 0; - auto read_entry_to_typed_array = [&](int rows, int cols) -> TypedArray { + auto read_entry_to_typed_array = [&](long long rows, long long cols) -> TypedArray { TypedArray arr; arr.dtype = ext; arr.rows = rows; @@ -560,13 +560,13 @@ AnyTrxFile AnyTrxFile::load_from_zip(const std::string &filename) { } trx.offsets = read_entry_to_typed_array(nb_streamlines + 1, 1); } else if (folder == "dps") { - const int nb_scalar = nb_streamlines > 0 ? static_cast(count_elems / nb_streamlines) : 0; + const long long nb_scalar = nb_streamlines > 0 ? count_elems / nb_streamlines : 0; if (nb_streamlines == 0 || count_elems % nb_streamlines != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dps size/dimensionality"); } trx.data_per_streamline.emplace(base, read_entry_to_typed_array(nb_streamlines, nb_scalar)); } else if (folder == "dpv") { - const int nb_scalar = nb_vertices > 0 ? static_cast(count_elems / nb_vertices) : 0; + const long long nb_scalar = nb_vertices > 0 ? count_elems / nb_vertices : 0; if (nb_vertices == 0 || count_elems % nb_vertices != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dpv size/dimensionality"); } @@ -577,13 +577,13 @@ AnyTrxFile AnyTrxFile::load_from_zip(const std::string &filename) { } std::string data_name = path_basename(base); std::string sub_folder = path_basename(folder); - trx.data_per_group[sub_folder].emplace(data_name, read_entry_to_typed_array(1, static_cast(count_elems))); + trx.data_per_group[sub_folder].emplace(data_name, read_entry_to_typed_array(1, static_cast(count_elems))); } else if (folder == "groups") { if (dim != 1) { throw TrxFormatError("Wrong group dimensionality"); } if (ext == "uint32") { - auto arr = read_entry_to_typed_array(static_cast(count_elems), 1); + auto arr = read_entry_to_typed_array(static_cast(count_elems), 1); arr.materialize_to_owned(); const uint64_t nb_streamlines_u64 = static_cast(trx.header["NB_STREAMLINES"].number_value()); const auto *vals = reinterpret_cast(arr.owned.data()); @@ -601,12 +601,12 @@ AnyTrxFile AnyTrxFile::load_from_zip(const std::string &filename) { throw TrxFormatError("Cannot normalize group '" + group_name + "' to uint32: NB_STREAMLINES exceeds uint32 limit"); } - auto tmp_arr = read_entry_to_typed_array(static_cast(count_elems), 1); + auto tmp_arr = read_entry_to_typed_array(static_cast(count_elems), 1); tmp_arr.materialize_to_owned(); TypedArray arr; arr.dtype = "uint32"; - arr.rows = static_cast(count_elems); + arr.rows = static_cast(count_elems); arr.cols = 1; arr.owned.resize(static_cast(count_elems) * sizeof(uint32_t)); uint32_t *dst = reinterpret_cast(arr.owned.data()); @@ -766,8 +766,8 @@ AnyTrxFile::_create_from_pointer(json header, throw TrxFormatError("Missing NB_VERTICES or NB_STREAMLINES in header.json"); } - const int nb_vertices = header["NB_VERTICES"].int_value(); - const int nb_streamlines = header["NB_STREAMLINES"].int_value(); + const long long nb_vertices = _json_int64(header["NB_VERTICES"]); + const long long nb_streamlines = _json_int64(header["NB_STREAMLINES"]); for (auto x = dict_pointer_size.rbegin(); x != dict_pointer_size.rend(); ++x) { const std::string elem_filename = x->first; @@ -797,7 +797,7 @@ AnyTrxFile::_create_from_pointer(json header, } trx.offsets = make_typed_array(elem_filename, nb_streamlines + 1, 1, ext); } else if (folder == "dps") { - const int nb_scalar = nb_streamlines > 0 ? static_cast(size / nb_streamlines) : 0; + const long long nb_scalar = nb_streamlines > 0 ? size / nb_streamlines : 0; if (nb_streamlines == 0 || size % nb_streamlines != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dps size/dimensionality"); } @@ -805,7 +805,7 @@ AnyTrxFile::_create_from_pointer(json header, arr.materialize_to_owned(); trx.data_per_streamline.emplace(base, std::move(arr)); } else if (folder == "dpv") { - const int nb_scalar = nb_vertices > 0 ? static_cast(size / nb_vertices) : 0; + const long long nb_scalar = nb_vertices > 0 ? size / nb_vertices : 0; if (nb_vertices == 0 || size % nb_vertices != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dpv size/dimensionality"); } @@ -818,7 +818,7 @@ AnyTrxFile::_create_from_pointer(json header, } std::string data_name = path_basename(base); std::string sub_folder = path_basename(folder); - auto arr = make_typed_array(elem_filename, 1, static_cast(size), ext); + auto arr = make_typed_array(elem_filename, 1, static_cast(size), ext); arr.materialize_to_owned(); trx.data_per_group[sub_folder].emplace(data_name, std::move(arr)); } else if (folder == "groups") { @@ -826,7 +826,7 @@ AnyTrxFile::_create_from_pointer(json header, throw TrxFormatError("Wrong group dimensionality"); } if (ext == "uint32") { - auto arr = make_typed_array(elem_filename, static_cast(size), 1, ext); + auto arr = make_typed_array(elem_filename, static_cast(size), 1, ext); arr.materialize_to_owned(); const uint64_t nb_streamlines_u64 = static_cast(header["NB_STREAMLINES"].number_value()); const auto *vals = reinterpret_cast(arr.owned.data()); @@ -853,12 +853,12 @@ AnyTrxFile::_create_from_pointer(json header, "' to uint32: NB_STREAMLINES exceeds the uint32 limit"); } - auto tmp_arr = make_typed_array(elem_filename, static_cast(size), 1, ext); + auto tmp_arr = make_typed_array(elem_filename, static_cast(size), 1, ext); tmp_arr.materialize_to_owned(); TypedArray arr; arr.dtype = "uint32"; - arr.rows = static_cast(size); + arr.rows = static_cast(size); arr.cols = 1; arr.owned.resize(static_cast(size) * sizeof(uint32_t)); uint32_t *dst = reinterpret_cast(arr.owned.data()); @@ -1080,7 +1080,7 @@ void AnyTrxFile::save(const std::string &filename, const TrxSaveOptions &options } const bool is_empty_tractogram = header["NB_VERTICES"].is_number() && header["NB_STREAMLINES"].is_number() && - header["NB_VERTICES"].int_value() == 0 && header["NB_STREAMLINES"].int_value() == 0; + _json_int64(header["NB_VERTICES"]) == 0 && _json_int64(header["NB_STREAMLINES"]) == 0; if (!is_empty_tractogram) { if (offsets.empty()) { @@ -1090,13 +1090,13 @@ void AnyTrxFile::save(const std::string &filename, const TrxSaveOptions &options throw TrxFormatError("Cannot save TRX without decoded offsets"); } if (header["NB_STREAMLINES"].is_number()) { - const auto nb_streamlines = static_cast(header["NB_STREAMLINES"].int_value()); + const auto nb_streamlines = static_cast(_json_int64(header["NB_STREAMLINES"])); if (offsets_u64.size() != nb_streamlines + 1) { throw TrxFormatError("TRX offsets size does not match NB_STREAMLINES"); } } if (header["NB_VERTICES"].is_number()) { - const auto nb_vertices = static_cast(header["NB_VERTICES"].int_value()); + const auto nb_vertices = static_cast(_json_int64(header["NB_VERTICES"])); const auto last = offsets_u64.back(); if (last != nb_vertices) { throw TrxFormatError("TRX offsets sentinel does not match NB_VERTICES"); @@ -1433,7 +1433,7 @@ void allocate_file(const std::string &path, std::size_t size) { } mio::shared_mmap_sink _create_memmap(std::string filename, - const std::tuple &shape, + const std::tuple &shape, const std::string &mode, const std::string &dtype, long long offset) { @@ -2162,8 +2162,8 @@ void merge_trx_shards(const MergeTrxShardsOptions &options) { ensure_schema_match("groups", groups_schema, shard_dir); const json shard_header = read_header(shard_dir); - const uint64_t shard_vertices = static_cast(shard_header["NB_VERTICES"].int_value()); - const uint64_t shard_streamlines = static_cast(shard_header["NB_STREAMLINES"].int_value()); + const uint64_t shard_vertices = static_cast(_json_int64(shard_header["NB_VERTICES"])); + const uint64_t shard_streamlines = static_cast(_json_int64(shard_header["NB_STREAMLINES"])); const std::string shard_positions = find_file_with_prefix(shard_dir, "positions."); const std::string shard_offsets = find_file_with_prefix(shard_dir, "offsets."); @@ -2201,8 +2201,13 @@ void merge_trx_shards(const MergeTrxShardsOptions &options) { total_streamlines += shard_streamlines; } - merged_header = _json_set(merged_header, "NB_VERTICES", static_cast(total_vertices)); - merged_header = _json_set(merged_header, "NB_STREAMLINES", static_cast(total_streamlines)); + constexpr uint64_t kMaxExactInDouble = (1ULL << 53); + if (total_vertices > kMaxExactInDouble || total_streamlines > kMaxExactInDouble) { + throw TrxFormatError("Merged TRX exceeds the 2^53 range representable in the JSON header"); + } + + merged_header = _json_set(merged_header, "NB_VERTICES", static_cast(total_vertices)); + merged_header = _json_set(merged_header, "NB_STREAMLINES", static_cast(total_streamlines)); { const std::string merged_header_path = output_dir + SEPARATOR + "header.json"; std::ofstream out(merged_header_path, std::ios::out | std::ios::trunc); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 89d5944..60d26dc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -118,6 +118,10 @@ add_executable(test_legacy_io test_trx_legacy_io.cpp) target_link_libraries(test_legacy_io PRIVATE trx GTest::gtest_main) target_compile_features(test_legacy_io PRIVATE cxx_std_17) +add_executable(test_large test_trx_large.cpp) +target_link_libraries(test_large PRIVATE trx GTest::gtest_main) +target_compile_features(test_large PRIVATE cxx_std_17) + include(GoogleTest) gtest_discover_tests(test_mmap PROPERTIES ENVIRONMENT "TRX_TEST_DATA_DIR=${TRX_TEST_DATA_DIR}" @@ -148,3 +152,5 @@ gtest_discover_tests(test_gs_consistency PROPERTIES gtest_discover_tests(test_legacy_io PROPERTIES ENVIRONMENT "TRX_TEST_DATA_DIR=${TRX_TEST_DATA_DIR}" ) + +gtest_discover_tests(test_large) diff --git a/tests/test_trx_large.cpp b/tests/test_trx_large.cpp new file mode 100644 index 0000000..a431084 --- /dev/null +++ b/tests/test_trx_large.cpp @@ -0,0 +1,79 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace trx; +namespace fs = std::filesystem; + +namespace { +fs::path make_temp_test_dir(const std::string &prefix) { + std::error_code ec; + auto base = fs::temp_directory_path(ec); + if (ec) { + throw std::runtime_error("Failed to get temp directory: " + ec.message()); + } + + thread_local std::mt19937_64 rng(std::random_device{}()); + std::uniform_int_distribution dist; + + for (int attempt = 0; attempt < 100; ++attempt) { + fs::path candidate = base / (prefix + "_" + std::to_string(dist(rng))); + std::error_code dir_ec; + if (fs::create_directory(candidate, dir_ec)) { + return candidate; + } + if (dir_ec && dir_ec != std::errc::file_exists) { + throw std::runtime_error("Failed to create temporary directory: " + dir_ec.message()); + } + } + throw std::runtime_error("Unable to create unique temporary directory"); +} + +void write_header_file(const fs::path &dir, const json &header) { + std::ofstream out((dir / "header.json").string()); + if (!out.is_open()) { + throw std::runtime_error("Failed to write header.json"); + } + out << header.dump() << '\n'; +} + +} + +TEST(Uint64Offsets, LoadVertexCountAboveInt32Max) { + const uint64_t base = 2147483647ULL + 100ULL; // just past INT32_MAX + const uint64_t nb_vertices = base + 5; + + const fs::path dir = make_temp_test_dir("trx_issue50"); + + json header = json::object{ + {"DIMENSIONS", json::array{1, 1, 1}}, + {"NB_STREAMLINES", 2.0}, + {"NB_VERTICES", static_cast(nb_vertices)}, + {"VOXEL_TO_RASMM", json::array{json::array{1, 0, 0, 0}, json::array{0, 1, 0, 0}, + json::array{0, 0, 1, 0}, json::array{0, 0, 0, 1}}}, + }; + write_header_file(dir, header); + + std::ofstream pos((dir / "positions.3.float32").string(), std::ios::binary); + pos.seekp(static_cast(nb_vertices * 3ULL * sizeof(float) - 1)); + pos.put('\0'); + pos.close(); + + const uint64_t offsets[3] = {base, base + 2, nb_vertices}; + std::ofstream((dir / "offsets.uint64").string(), std::ios::binary) + .write(reinterpret_cast(offsets), sizeof(offsets)); + + auto trx = AnyTrxFile::load(dir.string()); // threw before the fix + EXPECT_EQ(trx.num_vertices(), static_cast(nb_vertices)); + trx.close(); + + std::error_code ec; + fs::remove_all(dir, ec); +}