Skip to content

Allow tractograms with large offsets, 2 #59

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
36000 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Allow tractograms with large offsets, 2 #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
76f867a
allow tractograms with large offsets
36000 Sep 9, 2026
abeb22b
check merged TRXs; make test look more like others
36000 Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions include/trx/detail/dtype_helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

#include <Eigen/Core>

#include <cstdint>

#include <new>
#include <string>
#include <tuple>
Expand All @@ -16,25 +18,25 @@ namespace detail {
//
// MapType must be an Eigen::Map<Matrix<...>> type.
template <typename MapType>
inline void remap(MapType &map, void *data, int rows, int cols) {
inline void remap(MapType &map, void *data, long long rows, long long cols) {
using Scalar = typename MapType::Scalar;
new (&map) MapType(reinterpret_cast<Scalar *>(data), rows, cols); // NOLINT
}

// Overload for const data pointers (read-only maps).
template <typename MapType>
inline void remap(MapType &map, const void *data, int rows, int cols) {
inline void remap(MapType &map, const void *data, long long rows, long long cols) {
using Scalar = typename MapType::Scalar;
new (&map) MapType(const_cast<Scalar *>(reinterpret_cast<const Scalar *>(data)), rows, cols); // NOLINT
}

// Convenience overloads that unpack a (rows, cols) shape tuple.
template <typename MapType>
inline void remap(MapType &map, void *data, const std::tuple<int, int> &shape) {
inline void remap(MapType &map, void *data, const std::tuple<long long, long long> &shape) {
remap(map, data, std::get<0>(shape), std::get<1>(shape));
}
template <typename MapType>
inline void remap(MapType &map, const void *data, const std::tuple<int, int> &shape) {
inline void remap(MapType &map, const void *data, const std::tuple<long long, long long> &shape) {
remap(map, data, std::get<0>(shape), std::get<1>(shape));
}

Expand All @@ -45,7 +47,7 @@ std::tuple<std::string, int, std::string> _split_ext_with_dimensionality(const s

template <typename DT>
inline Eigen::Matrix<uint32_t, Eigen::Dynamic, 1> _compute_lengths(const Eigen::MatrixBase<DT> &offsets,
int nb_vertices) {
long long nb_vertices) {
static_cast<void>(nb_vertices);
if (offsets.size() > 1) {
const auto casted = offsets.template cast<uint64_t>();
Expand Down
45 changes: 28 additions & 17 deletions include/trx/trx.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,16 @@ inline json::object _json_object(const json &value) {
return json::object();
}

// This is a workaround for JSON files
// json11 stores every number as a double
// Json::int_value() truncates to a 32-bit int
// NB_VERTICES can exceed INT32_MAX
// so counts must be read through number_value().
// Doubles represent integers exactly up to 2^53
inline long long _json_int64(const json &value) {
return static_cast<long long>(value.number_value());
}

inline json _json_set(const json &value, const std::string &key, const json &field) {
auto obj = _json_object(value);
obj[key] = field;
Expand Down Expand Up @@ -247,8 +257,8 @@ template <typename DT> class TrxFile {
public:
struct GroupBackingInfo {
std::string filename;
int rows = 0;
int cols = 0;
long long rows = 0;
long long cols = 0;
std::string dtype;
long long mem_offset = 0;
};
Expand All @@ -269,8 +279,8 @@ template <typename DT> class TrxFile {

// Member Functions()
// TrxFile(int nb_vertices = 0, int nb_streamlines = 0);
TrxFile(int nb_vertices = 0,
int nb_streamlines = 0,
TrxFile(long long nb_vertices = 0,
long long nb_streamlines = 0,
const TrxFile<DT> *init_as = nullptr,
std::string reference = "");
~TrxFile();
Expand Down Expand Up @@ -308,7 +318,7 @@ template <typename DT> class TrxFile {
* @param nb_vertices The number of vertices to keep
* @param delete_dpg Remove data_per_group when resizing
*/
void resize(int nb_streamlines = -1, int nb_vertices = -1, bool delete_dpg = false);
void resize(long long nb_streamlines = -1, long long nb_vertices = -1, bool delete_dpg = false);

/**
* @brief Save a TrxFile
Expand Down Expand Up @@ -373,7 +383,7 @@ template <typename DT> class TrxFile {
return static_cast<size_t>(streamlines->_data.rows());
}
if (header["NB_VERTICES"].is_number()) {
return static_cast<size_t>(header["NB_VERTICES"].int_value());
return static_cast<size_t>(_json_int64(header["NB_VERTICES"]));
}
return 0;
}
Expand All @@ -386,7 +396,7 @@ template <typename DT> class TrxFile {
return static_cast<size_t>(streamlines->_lengths.size());
}
if (header["NB_STREAMLINES"].is_number()) {
return static_cast<size_t>(header["NB_STREAMLINES"].int_value());
return static_cast<size_t>(_json_int64(header["NB_STREAMLINES"]));
}
return 0;
}
Expand Down Expand Up @@ -521,11 +531,11 @@ template <typename DT> class TrxFile {
* @param strs_start The start index of the streamline
* @param pts_start The start index of the point
* @param nb_strs_to_copy The number of streamlines to copy. If not set will copy all
* @return std::tuple<int, int> A tuple representing the end of the copied streamlines and end
* @return std::tuple<long long, long long> A tuple representing the end of the copied streamlines and end
* of copied points
*/
std::tuple<int, int>
_copy_fixed_arrays_from(TrxFile<DT> *trx, int strs_start = 0, int pts_start = 0, int nb_strs_to_copy = -1);
std::tuple<long long, long long>
_copy_fixed_arrays_from(TrxFile<DT> *trx, long long strs_start = 0, long long pts_start = 0, long long nb_strs_to_copy = -1);
int len();

private:
Expand All @@ -535,10 +545,10 @@ template <typename DT> class TrxFile {
/**
* @brief Get the real size of data (ignoring zeros of preallocation)
*
* @return std::tuple<int, int> A tuple representing the index of the last streamline and the
* @return std::tuple<long long, long long> A tuple representing the index of the last streamline and the
* total length of all the streamlines
*/
std::tuple<int, int> _get_real_len();
std::tuple<long long, long long> _get_real_len();
};

namespace detail {
Expand Down Expand Up @@ -573,8 +583,8 @@ inline std::string make_unique_temp_path(const std::string &prefix) {

struct TypedArray {
std::string dtype;
int rows = 0;
int cols = 0;
long long rows = 0;
long long cols = 0;
mio::shared_mmap_sink mmap;
std::vector<std::uint8_t> owned;

Expand Down Expand Up @@ -1188,7 +1198,8 @@ TrxFile<DT>::compute_group_connectivity(ConnectivityMeasure measure, const std::
if (b == this->group_backing_info_.end()) {
continue;
}
const size_t expected_ids = static_cast<size_t>((std::max)(0, b->second.rows)) * static_cast<size_t>((std::max)(0, b->second.cols));
const size_t expected_ids = static_cast<size_t>((std::max<long long>)(0, b->second.rows)) *
static_cast<size_t>((std::max<long long>)(0, b->second.cols));
tmp_ids.resize(expected_ids);
if (expected_ids > 0) {
std::ifstream in(b->second.filename, std::ios::binary);
Expand Down Expand Up @@ -1269,7 +1280,7 @@ void allocate_file(const std::string &path, std::size_t size);
// Known limitations: only row-major order supported; shape uses tuple (sufficient for 2D);
// dtype parameter is used only for byte-size computation.
mio::shared_mmap_sink _create_memmap(std::string filename,
const std::tuple<int, int> &shape,
const std::tuple<long long, long long> &shape,
const std::string &mode = "r",
const std::string &dtype = "float32",
long long offset = 0);
Expand All @@ -1287,7 +1298,7 @@ std::string _generate_filename_from_data(const Eigen::MatrixBase<DT> &arr, const
*/
template <typename DT>
std::unique_ptr<TrxFile<DT>>
_initialize_empty_trx(int nb_streamlines, int nb_vertices, const TrxFile<DT> *init_as = nullptr);
_initialize_empty_trx(long long nb_streamlines, long long nb_vertices, const TrxFile<DT> *init_as = nullptr);

template <typename DT>
void ediff1d(Eigen::Matrix<DT, Eigen::Dynamic, 1> &lengths,
Expand Down
Loading