From 1b7f72e365fd6ca0f174b690d8a0e92e0a2fb58b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 15:19:14 +0000 Subject: [PATCH 01/10] Move the C ABI to int64_t and uint64_t, over BART's fixed-width types external/bart now points at the pulserver/bart branch that keeps dimensions, strides and flag sets in bart_dim_t, bart_stride_t and bart_flags_t, 64 bits on every data model, rather than in long, which is 32 bits on Windows. The header spells every shape, stride, count and size int64_t and every bit mask uint64_t; gen_abi.py maps them to ctypes.c_int64 and c_uint64 and no longer knows long. api.c asserts that BART's types are the ABI's, so a dimension vector still crosses without a copy, and test_abi.py holds the header to naming no long. The library's own code uses BART's names. fft.cpp checks that each length and stride fits MKL's LP64 MKL_LONG before handing a plan to DFTI, and walks or leaves to pocketfft one that does not. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- external/bart | 2 +- scripts/gen_abi.py | 12 +- src/bartorch/_abi.py | 288 +++++++++-------- src/bartorch/_marshal.py | 26 +- src/bartorch/nlop/base.py | 2 +- src/bartorch/nlop/callback.py | 4 +- src/bartorch/optim/linear.py | 12 +- src/csrc/abi/api.c | 6 + src/csrc/abi/cuda.c | 46 +-- src/csrc/abi/host_reads.c | 12 +- src/csrc/abi/memcfl.c | 18 +- src/csrc/include/bartorch.h | 227 +++++++------- src/csrc/ops/coset.cuh | 16 +- src/csrc/ops/fft_callbacks.cu | 22 +- src/csrc/ops/fft_callbacks_lto.cu | 10 +- src/csrc/ops/grid.c | 406 ++++++++++++------------ src/csrc/ops/grid.cuh | 6 +- src/csrc/ops/iter.c | 52 ++-- src/csrc/ops/kernels.cu | 178 +++++------ src/csrc/ops/ops.c | 308 +++++++++--------- src/csrc/ops/paired.cu | 24 +- src/csrc/ops/sense.c | 466 ++++++++++++++-------------- src/csrc/substitute/fft.cpp | 54 +++- src/csrc/substitute/finufft.c | 6 +- src/csrc/substitute/nufft_finufft.c | 408 ++++++++++++------------ src/csrc/substitute/psf.c | 116 +++---- tests/test_abi.py | 9 + 27 files changed, 1399 insertions(+), 1337 deletions(-) diff --git a/external/bart b/external/bart index 731bfd3e..04991fb5 160000 --- a/external/bart +++ b/external/bart @@ -1 +1 @@ -Subproject commit 731bfd3ec4f09ffa59b497ded1274a2c0357c8af +Subproject commit 04991fb50501a927b7f8ab3743c72ad7cef7e843 diff --git a/scripts/gen_abi.py b/scripts/gen_abi.py index cd956180..bbf0ce9b 100644 --- a/scripts/gen_abi.py +++ b/scripts/gen_abi.py @@ -28,16 +28,17 @@ # Every one of these appears in bartorch.h; a type that does not is a type the # generator has not been taught, and it stops rather than guessing. Pointers # to the opaque operator structs are void pointers here: the host never reads -# through them. +# through them. There is no ``long``: it is 32 bits on Windows and 64 +# elsewhere, and every integer in the ABI has one width on every platform. SCALARS = { "void": "None", "int": "ctypes.c_int", - "long": "ctypes.c_long", + "int64_t": "ctypes.c_int64", + "uint64_t": "ctypes.c_uint64", "float": "ctypes.c_float", "double": "ctypes.c_double", "size_t": "ctypes.c_size_t", - "unsigned long": "ctypes.c_ulong", "unsigned int": "ctypes.c_uint", } @@ -46,7 +47,8 @@ "char**": "ctypes.POINTER(ctypes.c_char_p)", "void*": "ctypes.c_void_p", "void**": "ctypes.POINTER(ctypes.c_void_p)", - "long*": "ctypes.POINTER(ctypes.c_long)", + "int64_t*": "ctypes.POINTER(ctypes.c_int64)", + "uint64_t*": "ctypes.POINTER(ctypes.c_uint64)", "int*": "ctypes.POINTER(ctypes.c_int)", "float*": "ctypes.POINTER(ctypes.c_float)", "double*": "ctypes.POINTER(ctypes.c_double)", @@ -84,7 +86,7 @@ def normalise(decl: str) -> str: #: Words that begin a type rather than name a parameter. -TYPE_WORDS = {"void", "int", "long", "float", "double", "size_t", "unsigned", "char"} +TYPE_WORDS = {"void", "int", "int64_t", "uint64_t", "float", "double", "size_t", "unsigned", "char"} def ctype(decl: str, callbacks: dict[str, str]) -> str: diff --git a/src/bartorch/_abi.py b/src/bartorch/_abi.py index 862151ec..d905b7fb 100644 --- a/src/bartorch/_abi.py +++ b/src/bartorch/_abi.py @@ -59,17 +59,17 @@ class Encoding(ctypes.Structure): ("traj_dims", ctypes.c_void_p), ("traj", ctypes.c_void_p), ("stacked", ctypes.c_int), - ("stack_count", ctypes.c_long), + ("stack_count", ctypes.c_int64), ("stack_positions", ctypes.c_void_p), - ("frames", ctypes.c_long), - ("shots", ctypes.c_long), + ("frames", ctypes.c_int64), + ("shots", ctypes.c_int64), ("components", ctypes.c_int), ("positions", ctypes.c_void_p), ("kspace_readout", ctypes.c_int), - ("readout", ctypes.c_long), + ("readout", ctypes.c_int64), ("psf", ctypes.c_void_p), ("centred", ctypes.c_int), - ("segments", ctypes.c_long), + ("segments", ctypes.c_int64), ("segment_sample_dims", ctypes.c_void_p), ("segment_sample", ctypes.c_void_p), ("segment_image_dims", ctypes.c_void_p), @@ -89,7 +89,7 @@ class Encoding(ctypes.Structure): ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ) FREE_FN = ctypes.CFUNCTYPE( None, @@ -361,7 +361,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_register.argtypes = [ ctypes.c_char_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_exists.restype = ctypes.c_int @@ -370,7 +370,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_lookup.argtypes = [ ctypes.c_char_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.POINTER(ctypes.c_void_p), ] lib.bartorch_unlink.restype = ctypes.c_int @@ -427,7 +427,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nufft_bf16.restype = ctypes.c_int lib.bartorch_nufft_bf16.argtypes = [] lib.bartorch_host_alloc.restype = ctypes.c_void_p - lib.bartorch_host_alloc.argtypes = [ctypes.c_long, ctypes.c_int] + lib.bartorch_host_alloc.argtypes = [ctypes.c_int64, ctypes.c_int] lib.bartorch_host_free.restype = None lib.bartorch_host_free.argtypes = [ctypes.c_void_p] lib.bartorch_cuda_stage_open.restype = ctypes.c_int @@ -440,20 +440,20 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, - ctypes.c_long, + ctypes.c_int64, ] lib.bartorch_cuda_stage_wait.restype = ctypes.c_int lib.bartorch_cuda_stage_wait.argtypes = [ctypes.c_void_p, ctypes.c_int] lib.bartorch_cuda_stage_release.restype = ctypes.c_int lib.bartorch_cuda_stage_release.argtypes = [ctypes.c_void_p, ctypes.c_int] lib.bartorch_cuda_copy_pageable.restype = ctypes.c_int - lib.bartorch_cuda_copy_pageable.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long] + lib.bartorch_cuda_copy_pageable.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64] lib.bartorch_host_prefault_begin.restype = ctypes.c_void_p - lib.bartorch_host_prefault_begin.argtypes = [ctypes.c_void_p, ctypes.c_long] + lib.bartorch_host_prefault_begin.argtypes = [ctypes.c_void_p, ctypes.c_int64] lib.bartorch_host_prefault_end.restype = None lib.bartorch_host_prefault_end.argtypes = [ctypes.c_void_p] lib.bartorch_cuda_host_register.restype = ctypes.c_int - lib.bartorch_cuda_host_register.argtypes = [ctypes.c_void_p, ctypes.c_long] + lib.bartorch_cuda_host_register.argtypes = [ctypes.c_void_p, ctypes.c_int64] lib.bartorch_cuda_host_unregister.restype = None lib.bartorch_cuda_host_unregister.argtypes = [ctypes.c_void_p] lib.bartorch_sense_set_coil_batch.restype = None @@ -464,7 +464,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_sense_set_fold_maps.argtypes = [ctypes.c_int] lib.bartorch_sense_fold_maps.restype = ctypes.c_int lib.bartorch_sense_fold_maps.argtypes = [] - lib.bartorch_sense_counter.restype = ctypes.c_long + lib.bartorch_sense_counter.restype = ctypes.c_int64 lib.bartorch_sense_counter.argtypes = [ctypes.c_int] lib.bartorch_sense_reset_counters.restype = None lib.bartorch_sense_reset_counters.argtypes = [] @@ -472,16 +472,16 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_fft_set.argtypes = [ctypes.c_char_p, ctypes.c_void_p] lib.bartorch_fft_usable.restype = ctypes.c_int lib.bartorch_fft_usable.argtypes = [] - lib.bartorch_fft_counter.restype = ctypes.c_long + lib.bartorch_fft_counter.restype = ctypes.c_int64 lib.bartorch_fft_counter.argtypes = [ctypes.c_int] lib.bartorch_fft_reset_counters.restype = None lib.bartorch_fft_reset_counters.argtypes = [] lib.bartorch_linop_callback.restype = ctypes.c_void_p lib.bartorch_linop_callback.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), APPLY_FN, APPLY_FN, APPLY_FN, @@ -491,42 +491,42 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_fft.restype = ctypes.c_void_p lib.bartorch_linop_fft.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ctypes.c_int, ctypes.c_int, ] lib.bartorch_linop_cdiag.restype = ctypes.c_void_p lib.bartorch_linop_cdiag.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ctypes.c_void_p, ] lib.bartorch_linop_fmac.restype = ctypes.c_void_p lib.bartorch_linop_fmac.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_sampling.restype = ctypes.c_void_p lib.bartorch_linop_sampling.argtypes = [ - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_nufft.restype = ctypes.c_void_p lib.bartorch_linop_nufft.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ctypes.c_int, ctypes.c_float, @@ -536,31 +536,31 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_set_diagonal.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_blocks.restype = ctypes.c_void_p lib.bartorch_linop_blocks.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.c_long, + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.c_int64, ] lib.bartorch_linop_reshaped.restype = ctypes.c_void_p lib.bartorch_linop_reshaped.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_encoding.restype = ctypes.c_void_p lib.bartorch_linop_encoding.argtypes = [ctypes.POINTER(Encoding)] - lib.bartorch_encoding_counter.restype = ctypes.c_long + lib.bartorch_encoding_counter.restype = ctypes.c_int64 lib.bartorch_encoding_counter.argtypes = [ctypes.c_int] lib.bartorch_encoding_reset_counters.restype = None lib.bartorch_encoding_reset_counters.argtypes = [] - lib.bartorch_grid_fused.restype = ctypes.c_long + lib.bartorch_grid_fused.restype = ctypes.c_int64 lib.bartorch_grid_fused.argtypes = [] lib.bartorch_linop_with_normal.restype = ctypes.c_void_p lib.bartorch_linop_with_normal.argtypes = [ctypes.c_void_p, ctypes.c_void_p] @@ -588,80 +588,92 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_scale.restype = ctypes.c_void_p lib.bartorch_linop_scale.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ctypes.c_float, ] lib.bartorch_linop_zconj.restype = ctypes.c_void_p - lib.bartorch_linop_zconj.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_linop_zconj.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_linop_identity.restype = ctypes.c_void_p - lib.bartorch_linop_identity.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_linop_identity.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_linop_null.restype = ctypes.c_void_p lib.bartorch_linop_null.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_maxeigen.restype = ctypes.c_double lib.bartorch_linop_maxeigen.argtypes = [ctypes.c_void_p] lib.bartorch_linop_zreal.restype = ctypes.c_void_p - lib.bartorch_linop_zreal.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_linop_zreal.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_linop_rdiag.restype = ctypes.c_void_p lib.bartorch_linop_rdiag.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ctypes.c_void_p, ] lib.bartorch_linop_matrix.restype = ctypes.c_void_p lib.bartorch_linop_matrix.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_conv.restype = ctypes.c_void_p lib.bartorch_linop_conv.argtypes = [ ctypes.c_int, - ctypes.c_ulong, + ctypes.c_uint64, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_grad.restype = ctypes.c_void_p lib.bartorch_linop_grad.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.c_ulong, + ctypes.c_uint64, ] lib.bartorch_linop_sum.restype = ctypes.c_void_p - lib.bartorch_linop_sum.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_ulong] + lib.bartorch_linop_sum.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, + ] lib.bartorch_linop_scaled_sum.restype = ctypes.c_void_p lib.bartorch_linop_scaled_sum.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ] lib.bartorch_linop_avg.restype = ctypes.c_void_p - lib.bartorch_linop_avg.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_ulong] + lib.bartorch_linop_avg.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, + ] lib.bartorch_linop_repmat.restype = ctypes.c_void_p lib.bartorch_linop_repmat.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ] lib.bartorch_linop_flip.restype = ctypes.c_void_p - lib.bartorch_linop_flip.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_ulong] + lib.bartorch_linop_flip.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, + ] lib.bartorch_linop_hankel.restype = ctypes.c_void_p lib.bartorch_linop_hankel.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, ctypes.c_int, ctypes.c_int, @@ -669,51 +681,51 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_reshape.restype = ctypes.c_void_p lib.bartorch_linop_reshape.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_resize.restype = ctypes.c_void_p lib.bartorch_linop_resize.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_extract.restype = ctypes.c_void_p lib.bartorch_linop_extract.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_transpose.restype = ctypes.c_void_p lib.bartorch_linop_transpose.argtypes = [ ctypes.c_int, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_permute.restype = ctypes.c_void_p lib.bartorch_linop_permute.argtypes = [ ctypes.c_int, ctypes.POINTER(ctypes.c_int), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_shift.restype = ctypes.c_void_p lib.bartorch_linop_shift.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.c_long, + ctypes.c_int64, ctypes.c_int, ] lib.bartorch_linop_padding.restype = ctypes.c_void_p lib.bartorch_linop_padding.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_has_pseudo_inv.restype = ctypes.c_int lib.bartorch_linop_has_pseudo_inv.argtypes = [ctypes.c_void_p] @@ -728,13 +740,13 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_domain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_codomain.restype = ctypes.c_int lib.bartorch_linop_codomain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_forward.restype = ctypes.c_int lib.bartorch_linop_forward.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] @@ -759,8 +771,8 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_char_p), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_uint64), + ctypes.POINTER(ctypes.c_uint64), ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_void_p), @@ -791,13 +803,13 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.POINTER(ctypes.c_float), ctypes.c_void_p, ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_solve_error.restype = ctypes.c_char_p lib.bartorch_solve_error.argtypes = [ctypes.c_int] lib.bartorch_scaling_norm.restype = ctypes.c_float lib.bartorch_scaling_norm.argtypes = [ - ctypes.c_long, + ctypes.c_int64, ctypes.c_void_p, ctypes.c_float, ctypes.c_int, @@ -806,22 +818,22 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_prox_create.restype = ctypes.c_int lib.bartorch_prox_create.argtypes = [ ctypes.c_char_p, - ctypes.c_long, - ctypes.c_long, + ctypes.c_uint64, + ctypes.c_uint64, ctypes.c_float, ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.POINTER(ctypes.c_void_p), ] lib.bartorch_prox_set_create.restype = ctypes.c_int lib.bartorch_prox_set_create.argtypes = [ ctypes.c_int, ctypes.POINTER(ctypes.c_char_p), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_uint64), + ctypes.POINTER(ctypes.c_uint64), ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_int), ctypes.c_int, @@ -829,17 +841,17 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_float), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_int), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_prox_domain.restype = ctypes.c_int lib.bartorch_prox_domain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_prox_apply.restype = ctypes.c_int lib.bartorch_prox_apply.argtypes = [ @@ -866,9 +878,9 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nlop_callback.restype = ctypes.c_void_p lib.bartorch_nlop_callback.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), APPLY_FN, APPLY_FN, APPLY_FN, @@ -879,10 +891,10 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nlop_callback_generic.argtypes = [ ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), GENERIC_APPLY_FN, PAIR_APPLY_FN, PAIR_APPLY_FN, @@ -897,13 +909,13 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nlop_domain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_codomain.restype = ctypes.c_int lib.bartorch_nlop_codomain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_apply.restype = ctypes.c_int lib.bartorch_nlop_apply.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] @@ -920,14 +932,14 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_output_codomain.restype = ctypes.c_int lib.bartorch_nlop_output_codomain.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_apply_generic.restype = ctypes.c_int lib.bartorch_nlop_apply_generic.argtypes = [ @@ -942,14 +954,14 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_reshape_out.restype = ctypes.c_void_p lib.bartorch_nlop_reshape_out.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_chain2.restype = ctypes.c_void_p lib.bartorch_nlop_chain2.argtypes = [ @@ -992,62 +1004,62 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nlop_tenmul.restype = ctypes.c_void_p lib.bartorch_nlop_tenmul.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_zdiv.restype = ctypes.c_void_p - lib.bartorch_nlop_zdiv.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_float] + lib.bartorch_nlop_zdiv.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_float] lib.bartorch_nlop_zaxpbz.restype = ctypes.c_void_p lib.bartorch_nlop_zaxpbz.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ctypes.c_float, ] lib.bartorch_nlop_zexp.restype = ctypes.c_void_p - lib.bartorch_nlop_zexp.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_nlop_zexp.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_nlop_zlog.restype = ctypes.c_void_p - lib.bartorch_nlop_zlog.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_nlop_zlog.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_nlop_zinv.restype = ctypes.c_void_p - lib.bartorch_nlop_zinv.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_float] + lib.bartorch_nlop_zinv.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_float] lib.bartorch_nlop_zsqrt.restype = ctypes.c_void_p - lib.bartorch_nlop_zsqrt.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_nlop_zsqrt.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_nlop_zspow.restype = ctypes.c_void_p lib.bartorch_nlop_zspow.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ctypes.c_float, ] lib.bartorch_nlop_zsadd.restype = ctypes.c_void_p lib.bartorch_nlop_zsadd.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ctypes.c_float, ] lib.bartorch_nlop_zabs.restype = ctypes.c_void_p - lib.bartorch_nlop_zabs.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_nlop_zabs.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_nlop_smo_abs.restype = ctypes.c_void_p lib.bartorch_nlop_smo_abs.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ] lib.bartorch_nlop_zrss.restype = ctypes.c_void_p lib.bartorch_nlop_zrss.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ctypes.c_float, ] lib.bartorch_nlop_zss.restype = ctypes.c_void_p - lib.bartorch_nlop_zss.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_ulong] + lib.bartorch_nlop_zss.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_uint64] lib.bartorch_nlop_const.restype = ctypes.c_void_p lib.bartorch_nlop_const.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_nlop_set_input_const.restype = ctypes.c_void_p @@ -1055,7 +1067,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_nlop_free.restype = None @@ -1066,31 +1078,31 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_int, ctypes.c_float, ctypes.c_float, - ctypes.c_long, + ctypes.c_int64, ] lib.bartorch_noir_create.restype = ctypes.c_void_p lib.bartorch_noir_create.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int, - ctypes.c_ulong, - ctypes.c_ulong, + ctypes.c_uint64, + ctypes.c_uint64, ctypes.c_int, ctypes.c_int, ctypes.c_float, @@ -1114,7 +1126,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_noir_free.restype = None lib.bartorch_noir_free.argtypes = [ctypes.c_void_p] @@ -1165,7 +1177,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_cuda_wait_for_stream.argtypes = [ctypes.c_void_p] lib.bartorch_cuda_signal_stream.restype = ctypes.c_int lib.bartorch_cuda_signal_stream.argtypes = [ctypes.c_void_p] - lib.bartorch_cuda_free_memory.restype = ctypes.c_long + lib.bartorch_cuda_free_memory.restype = ctypes.c_int64 lib.bartorch_cuda_free_memory.argtypes = [] lib.bartorch_finufft_set.restype = ctypes.c_int lib.bartorch_finufft_set.argtypes = [ctypes.c_char_p, ctypes.c_void_p] @@ -1195,7 +1207,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_finufft_usable_on.argtypes = [ctypes.c_int] lib.bartorch_finufft_usable.restype = ctypes.c_int lib.bartorch_finufft_usable.argtypes = [] - lib.bartorch_finufft_live_plans.restype = ctypes.c_long + lib.bartorch_finufft_live_plans.restype = ctypes.c_int64 lib.bartorch_finufft_live_plans.argtypes = [] lib.bartorch_last_error.restype = ctypes.c_char_p lib.bartorch_last_error.argtypes = [] @@ -1209,11 +1221,11 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nufft_allow_fallback.argtypes = [ctypes.c_int] lib.bartorch_nufft_fallback_allowed.restype = ctypes.c_int lib.bartorch_nufft_fallback_allowed.argtypes = [] - lib.bartorch_nufft_counter.restype = ctypes.c_long + lib.bartorch_nufft_counter.restype = ctypes.c_int64 lib.bartorch_nufft_counter.argtypes = [ctypes.c_int] lib.bartorch_nufft_reset_counters.restype = None lib.bartorch_nufft_reset_counters.argtypes = [] - lib.bartorch_toeplitz_counter.restype = ctypes.c_long + lib.bartorch_toeplitz_counter.restype = ctypes.c_int64 lib.bartorch_toeplitz_counter.argtypes = [ctypes.c_int] lib.bartorch_toeplitz_reset_counters.restype = None lib.bartorch_toeplitz_reset_counters.argtypes = [] diff --git a/src/bartorch/_marshal.py b/src/bartorch/_marshal.py index 5b3cb980..a6c17c03 100644 --- a/src/bartorch/_marshal.py +++ b/src/bartorch/_marshal.py @@ -21,7 +21,7 @@ "float_buffer", "floats", "ints", - "longs", + "int64s", "null_apply", "null_release", "out_pointer", @@ -29,6 +29,7 @@ "padded_dims", "shape_from_dims", "text_buffer", + "uint64s", ] @@ -37,7 +38,7 @@ def padded_dims(shape: tuple[int, ...]) -> ctypes.Array: rev = list(shape)[::-1] if len(rev) > DIMS: raise ValueError(f"BART supports at most {DIMS} dimensions, got {len(rev)}") - return (ctypes.c_long * DIMS)(*(rev + [1] * (DIMS - len(rev)))) + return (ctypes.c_int64 * DIMS)(*(rev + [1] * (DIMS - len(rev)))) def padded_offsets(values, ndim: int, what: str) -> ctypes.Array: @@ -53,7 +54,7 @@ def padded_offsets(values, ndim: int, what: str) -> ctypes.Array: if ndim > DIMS: raise ValueError(f"BART supports at most {DIMS} dimensions, got {ndim}") rev = [int(v) for v in values][::-1] - return (ctypes.c_long * DIMS)(*(rev + [0] * (DIMS - ndim))) + return (ctypes.c_int64 * DIMS)(*(rev + [0] * (DIMS - ndim))) def padded_order(order, ndim: int) -> ctypes.Array: @@ -84,7 +85,7 @@ def dims(shape: tuple[int, ...]) -> tuple[int, ctypes.Array]: rev = list(shape)[::-1] or [1] if len(rev) > DIMS: raise ValueError(f"BART supports at most {DIMS} dimensions, got {len(rev)}") - return len(rev), (ctypes.c_long * len(rev))(*rev) + return len(rev), (ctypes.c_int64 * len(rev))(*rev) def shape_from_dims(vector: ctypes.Array, min_ndim: int) -> list[int]: @@ -100,8 +101,13 @@ def argv(args: list[str]) -> ctypes.Array: return (ctypes.c_char_p * len(args))(*[a.encode() for a in args]) -def longs(values) -> ctypes.Array: - return (ctypes.c_long * len(values))(*[int(v) for v in values]) +def int64s(values) -> ctypes.Array: + return (ctypes.c_int64 * len(values))(*[int(v) for v in values]) + + +def uint64s(values) -> ctypes.Array: + """Bit masks, as the ABI's ``uint64_t``.""" + return (ctypes.c_uint64 * len(values))(*[int(v) for v in values]) def ints(values) -> ctypes.Array: @@ -141,7 +147,7 @@ def null_release() -> ctypes.Array: def dim_vector() -> ctypes.Array: """A dimension vector of :data:`DIMS` entries for the library to fill in.""" - return (ctypes.c_long * DIMS)() + return (ctypes.c_int64 * DIMS)() def wide_dim_vector() -> ctypes.Array: @@ -151,12 +157,12 @@ def wide_dim_vector() -> ctypes.Array: thresholds a gradient's components, an axis beyond the image's, and total generalized variation's symmetric gradient adds a second (``tgv.c``). """ - return (ctypes.c_long * (DIMS + 2))() + return (ctypes.c_int64 * (DIMS + 2))() -def long_out() -> ctypes.c_long: +def int64_out() -> ctypes.c_int64: """A number the library writes into; read it back off ``.value``.""" - return ctypes.c_long() + return ctypes.c_int64() def int_out() -> ctypes.c_int: diff --git a/src/bartorch/nlop/base.py b/src/bartorch/nlop/base.py index 708b0601..87cc7f1f 100644 --- a/src/bartorch/nlop/base.py +++ b/src/bartorch/nlop/base.py @@ -741,7 +741,7 @@ def _create(self) -> Built: self.x._h.ptr, self.at, len(self.shape), - _marshal.longs(list(self.shape)[::-1]), + _marshal.int64s(list(self.shape)[::-1]), device=self.x.device, ) ishapes, oshapes = list(self.x.ishapes), list(self.x.oshapes) diff --git a/src/bartorch/nlop/callback.py b/src/bartorch/nlop/callback.py index f33b2494..0d6da806 100644 --- a/src/bartorch/nlop/callback.py +++ b/src/bartorch/nlop/callback.py @@ -110,10 +110,10 @@ def _create(self) -> Built: library().bartorch_nlop_callback_generic, len(self.oshapes), DIMS, - _marshal.longs(_flat(self.oshapes)), + _marshal.int64s(_flat(self.oshapes)), len(self.ishapes), DIMS, - _marshal.longs(_flat(self.ishapes)), + _marshal.int64s(_flat(self.ishapes)), fwd, der, adj, diff --git a/src/bartorch/optim/linear.py b/src/bartorch/optim/linear.py index e3c24071..0002742b 100644 --- a/src/bartorch/optim/linear.py +++ b/src/bartorch/optim/linear.py @@ -293,14 +293,14 @@ def _solve( block, family, shift_mode = _shared_options(terms) alpha, gamma = _shared_pairs(terms) - counter = _marshal.long_out() + counter = _marshal.int64_out() with _lock, _on_device(op.device or y.device): code = lib.bartorch_solve( op._h.ptr, algorithm.encode(), _marshal.argv([term.kind for term in terms]) if terms else None, - _marshal.longs([f for f, _ in flags]) if terms else None, - _marshal.longs([j for _, j in flags]) if terms else None, + _marshal.uint64s([f for f, _ in flags]) if terms else None, + _marshal.uint64s([j for _, j in flags]) if terms else None, _marshal.floats([term.weight for term in terms]) if terms else None, _marshal.ints([term.count for term in terms]) if terms else None, _marshal.pointers(handles) if handles else None, @@ -410,13 +410,13 @@ def _penalties(terms, image_shape: tuple[int, ...]): _ensure_ready() handles = _marshal.pointer_buffer(_MAX_PENALTIES) - count, svars = _marshal.int_out(), _marshal.long_out() + count, svars = _marshal.int_out(), _marshal.int64_out() with _lock: code = library().bartorch_prox_set_create( len(terms), _marshal.argv([term.kind for term in terms]), - _marshal.longs([f for f, _ in flags]), - _marshal.longs([j for _, j in flags]), + _marshal.uint64s([f for f, _ in flags]), + _marshal.uint64s([j for _, j in flags]), _marshal.floats([term.weight for term in terms]), _marshal.ints([term.count for term in terms]), block, diff --git a/src/csrc/abi/api.c b/src/csrc/abi/api.c index 39dbf317..05b7916c 100644 --- a/src/csrc/abi/api.c +++ b/src/csrc/abi/api.c @@ -17,6 +17,12 @@ #include "include/bartorch.h" #include "substitute/backend.h" +/* The ABI spells BART's dimension, stride and flag types by their widths, so + * a dimension vector crosses as it stands, without a copy. */ +_Static_assert(_Generic((bart_dim_t*)0, int64_t*: 1, default: 0), "BART dimensions are the ABI's int64_t"); +_Static_assert(_Generic((bart_stride_t*)0, int64_t*: 1, default: 0), "BART strides are the ABI's int64_t"); +_Static_assert(_Generic((bart_flags_t*)0, uint64_t*: 1, default: 0), "BART flags are the ABI's uint64_t"); + extern int bart_command(int len, char* buf, int argc, char* argv[]); /* A failed assertion reaches the caller instead of the process. diff --git a/src/csrc/abi/cuda.c b/src/csrc/abi/cuda.c index 4d0caf60..49b401a0 100644 --- a/src/csrc/abi/cuda.c +++ b/src/csrc/abi/cuda.c @@ -34,15 +34,15 @@ struct prefault { pthread_t thread; volatile char* ptr; - long size; + bart_dim_t size; }; static void* prefault_run(void* arg) { struct prefault* p = arg; - long page = sysconf(_SC_PAGESIZE); + bart_dim_t page = sysconf(_SC_PAGESIZE); - for (long i = 0; i < p->size; i += page) + for (bart_dim_t i = 0; i < p->size; i += page) p->ptr[i] = p->ptr[i]; p->ptr[p->size - 1] = p->ptr[p->size - 1]; @@ -50,7 +50,7 @@ static void* prefault_run(void* arg) return NULL; } -void* bartorch_host_prefault_begin(void* ptr, long size) +void* bartorch_host_prefault_begin(void* ptr, int64_t size) { if ((NULL == ptr) || (0 >= size)) return NULL; @@ -266,7 +266,7 @@ int bartorch_cuda_signal_stream(void* stream) * is what the function a normal is streamed from is kept in. Where there is * no card, or where the card will not lock that much, it is ordinary * memory. */ -void* bartorch_host_alloc(long size, int pinned) +void* bartorch_host_alloc(int64_t size, int pinned) { if (0 >= size) return NULL; @@ -336,7 +336,7 @@ static bool bounce_open(void) static void host_copy(void* dst, const void* src, size_t n) { - long parts = (long)(n >> 22); + bart_dim_t parts = (bart_dim_t)(n >> 22); if (parts < 2) { @@ -345,7 +345,7 @@ static void host_copy(void* dst, const void* src, size_t n) } #pragma omp parallel for - for (long p = 0; p < parts; p++) { + for (bart_dim_t p = 0; p < parts; p++) { size_t a = n * (size_t)p / (size_t)parts; size_t b = n * (size_t)(p + 1) / (size_t)parts; @@ -354,7 +354,7 @@ static void host_copy(void* dst, const void* src, size_t n) } } -int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) +int bartorch_cuda_copy_pageable(void* dst, const void* src, int64_t size) { if ((-1 == current_device) || (0 >= size) || !bounce_open()) return -1; @@ -365,12 +365,12 @@ int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) for (int i = 0; i < 2; i++) cudaEventSynchronize(bounce_done[i]); - long chunks = (size + BOUNCE_BYTES - 1) / BOUNCE_BYTES; + bart_dim_t chunks = (size + BOUNCE_BYTES - 1) / BOUNCE_BYTES; - for (long k = 0; k <= chunks; k++) { + for (bart_dim_t k = 0; k <= chunks; k++) { - long off = k * (long)BOUNCE_BYTES; - long len = MIN((long)BOUNCE_BYTES, size - off); + bart_stride_t off = k * (bart_dim_t)BOUNCE_BYTES; + bart_dim_t len = MIN((bart_dim_t)BOUNCE_BYTES, size - off); int b = (int)(k & 1); if (to_host) { @@ -386,8 +386,8 @@ int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) if (0 < k) { - long poff = (k - 1) * (long)BOUNCE_BYTES; - long plen = MIN((long)BOUNCE_BYTES, size - poff); + bart_dim_t poff = (k - 1) * (bart_dim_t)BOUNCE_BYTES; + bart_dim_t plen = MIN((bart_dim_t)BOUNCE_BYTES, size - poff); cudaEventSynchronize(bounce_done[b ^ 1]); host_copy((char*)dst + poff, bounce[b ^ 1], (size_t)plen); @@ -413,7 +413,7 @@ int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) * its own and a copy out of it runs behind whatever the card is doing. For * gigabytes it is a fraction of a second here, where allocating as much * page-locked takes seconds. */ -int bartorch_cuda_host_register(void* ptr, long size) +int bartorch_cuda_host_register(void* ptr, int64_t size) { if ((NULL == ptr) || (0 >= size) || (-1 == current_device)) return -1; @@ -500,7 +500,7 @@ void bartorch_cuda_stage_close(void* stage) } /* Start a slot's crossing, once the card has finished reading what is in it. */ -int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, long size) +int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, int64_t size) { if ((NULL == stage) || (0 > slot) || (1 < slot)) return -1; @@ -542,7 +542,7 @@ int bartorch_on_device(const void* ptr) return cuda_ondevice(ptr) ? 1 : 0; } -long bartorch_cuda_free_memory(void) +int64_t bartorch_cuda_free_memory(void) { size_t free_bytes = 0; size_t total_bytes = 0; @@ -553,7 +553,7 @@ long bartorch_cuda_free_memory(void) return -1; } - return (long)free_bytes; + return (bart_dim_t)free_bytes; } #else /* !USE_CUDA */ @@ -569,17 +569,17 @@ int bartorch_cuda_use_memcache(int enable) { (void)enable; return -1; } void bartorch_cuda_memcache_clear_all(void) { } int bartorch_cuda_wait_for_stream(void* stream) { (void)stream; return -1; } int bartorch_cuda_signal_stream(void* stream) { (void)stream; return -1; } -void* bartorch_host_alloc(long size, int pinned) { (void)pinned; return (0 < size) ? xmalloc((size_t)size) : NULL; } +void* bartorch_host_alloc(int64_t size, int pinned) { (void)pinned; return (0 < size) ? xmalloc((size_t)size) : NULL; } void bartorch_host_free(void* ptr) { if (NULL != ptr) xfree(ptr); } int bartorch_cuda_stage_open(void** stage) { (void)stage; return -1; } -int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) { (void)dst; (void)src; (void)size; return -1; } -int bartorch_cuda_host_register(void* ptr, long size) { (void)ptr; (void)size; return -1; } +int bartorch_cuda_copy_pageable(void* dst, const void* src, int64_t size) { (void)dst; (void)src; (void)size; return -1; } +int bartorch_cuda_host_register(void* ptr, int64_t size) { (void)ptr; (void)size; return -1; } void bartorch_cuda_host_unregister(void* ptr) { (void)ptr; } void bartorch_cuda_stage_close(void* stage) { (void)stage; } -int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, long size) +int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, int64_t size) { (void)stage; (void)slot; (void)dst; (void)src; (void)size; return -1; } int bartorch_cuda_stage_wait(void* stage, int slot) { (void)stage; (void)slot; return -1; } int bartorch_cuda_stage_release(void* stage, int slot) { (void)stage; (void)slot; return -1; } -long bartorch_cuda_free_memory(void) { return -1; } +int64_t bartorch_cuda_free_memory(void) { return -1; } #endif diff --git a/src/csrc/abi/host_reads.c b/src/csrc/abi/host_reads.c index 0adc48a0..75ce39f6 100644 --- a/src/csrc/abi/host_reads.c +++ b/src/csrc/abi/host_reads.c @@ -23,12 +23,12 @@ #include "include/bartorch.h" -extern void bart_estimate_im_dims(int N, unsigned long flags, long dims[N], const long tdims[N], const complex float* traj); -extern void bart_estimate_fast_sq_im_dims(int N, long dims[3], const long tdims[N], const complex float* traj); +extern void bart_estimate_im_dims(int N, bart_flags_t flags, bart_dim_t dims[N], const bart_dim_t tdims[N], const complex float* traj); +extern void bart_estimate_fast_sq_im_dims(int N, bart_dim_t dims[3], const bart_dim_t tdims[N], const complex float* traj); extern float bart_estimate_scaling_norm(float rescale, int imsize, complex float* tmpnorm, bool compat, float p); /* A copy of the trajectory on the host, or NULL when it is already there. */ -static complex float* host_copy(int N, const long tdims[N], const complex float* traj) +static complex float* host_copy(int N, const bart_dim_t tdims[N], const complex float* traj) { if (!bartorch_on_device(traj)) return NULL; @@ -40,7 +40,7 @@ static complex float* host_copy(int N, const long tdims[N], const complex float* return host; } -void estimate_im_dims(int N, unsigned long flags, long dims[N], const long tdims[N], const complex float* traj) +void estimate_im_dims(int N, bart_flags_t flags, bart_dim_t dims[N], const bart_dim_t tdims[N], const complex float* traj) { complex float* host = host_copy(N, tdims, traj); @@ -49,7 +49,7 @@ void estimate_im_dims(int N, unsigned long flags, long dims[N], const long tdims md_free(host); } -void estimate_fast_sq_im_dims(int N, long dims[3], const long tdims[N], const complex float* traj) +void estimate_fast_sq_im_dims(int N, bart_dim_t dims[3], const bart_dim_t tdims[N], const complex float* traj) { complex float* host = host_copy(N, tdims, traj); @@ -66,7 +66,7 @@ float estimate_scaling_norm(float rescale, int imsize, complex float* tmpnorm, b if (!bartorch_on_device(tmpnorm)) return bart_estimate_scaling_norm(rescale, imsize, tmpnorm, compat, p); - long dims[1] = { imsize }; + bart_dim_t dims[1] = { imsize }; complex float* host = md_alloc(1, dims, CFL_SIZE); md_copy(1, dims, host, tmpnorm, CFL_SIZE); diff --git a/src/csrc/abi/memcfl.c b/src/csrc/abi/memcfl.c index d0f68d69..72ff87a1 100644 --- a/src/csrc/abi/memcfl.c +++ b/src/csrc/abi/memcfl.c @@ -27,7 +27,7 @@ struct entry { char* name; int D; - long* dims; + bart_dim_t* dims; complex float* data; int refcount; enum owner owner; @@ -78,13 +78,13 @@ static struct entry* find_data_locked(const complex float* data) return NULL; } -static void insert_locked(const char* name, int D, const long* dims, complex float* data, enum owner owner) +static void insert_locked(const char* name, int D, const bart_dim_t* dims, complex float* data, enum owner owner) { struct entry* e = malloc(sizeof(*e)); e->name = strdup(name); e->D = D; - e->dims = malloc(sizeof(long) * (size_t)D); - memcpy(e->dims, dims, sizeof(long) * (size_t)D); + e->dims = malloc(sizeof(bart_dim_t) * (size_t)D); + memcpy(e->dims, dims, sizeof(bart_dim_t) * (size_t)D); e->data = data; e->refcount = 1; e->owner = owner; @@ -123,14 +123,14 @@ static void unlink_locked(struct entry** ep) /* BART-facing API (misc/memcfl.h). */ -void memcfl_register(const char* name, int D, const long dims[D], complex float* data, bool managed) +void memcfl_register(const char* name, int D, const bart_dim_t dims[D], complex float* data, bool managed) { pthread_mutex_lock(&lock); insert_locked(name, D, dims, data, managed ? OWNER_XMALLOC : OWNER_HOST); pthread_mutex_unlock(&lock); } -complex float* memcfl_create(const char* name, int D, const long dims[D]) +complex float* memcfl_create(const char* name, int D, const bart_dim_t dims[D]) { pthread_mutex_lock(&lock); @@ -200,7 +200,7 @@ const char** memcfl_list_all(void) return out; } -complex float* memcfl_load(const char* name, int D, long dims[D]) +complex float* memcfl_load(const char* name, int D, bart_dim_t dims[D]) { pthread_mutex_lock(&lock); @@ -265,7 +265,7 @@ void memcfl_unlink(const char* name) /* Host-facing API (include/bartorch.h). */ -int bartorch_register(const char* name, int D, const long* dims, void* data) +int bartorch_register(const char* name, int D, const int64_t* dims, void* data) { if ((D < 1) || (D > BARTORCH_DIMS) || (NULL == data)) return -1; @@ -291,7 +291,7 @@ int bartorch_exists(const char* name) return memcfl_exists(name) ? 1 : 0; } -int bartorch_lookup(const char* name, int D, long* dims, void** data) +int bartorch_lookup(const char* name, int D, int64_t* dims, void** data) { pthread_mutex_lock(&lock); diff --git a/src/csrc/include/bartorch.h b/src/csrc/include/bartorch.h index e61b80bf..e5077bf0 100644 --- a/src/csrc/include/bartorch.h +++ b/src/csrc/include/bartorch.h @@ -13,6 +13,7 @@ #define BARTORCH_H #include +#include #ifdef __cplusplus extern "C" { @@ -41,7 +42,7 @@ enum bartorch_log_level { * free callback receives the same pointer. Returning NULL aborts the * running command with an error. */ -typedef void* (*bartorch_alloc_fn)(void* ctx, int D, const long* dims); +typedef void* (*bartorch_alloc_fn)(void* ctx, int D, const int64_t* dims); typedef void (*bartorch_free_fn)(void* ctx, void* data); /* Receives every BART log line at or below the current debug level. */ @@ -63,9 +64,9 @@ BARTORCH_API void bartorch_set_num_threads(int n); * freed by BART; arrays BART creates itself come from the allocator * callback and are released through the free callback on unlink. */ -BARTORCH_API int bartorch_register(const char* name, int D, const long* dims, void* data); +BARTORCH_API int bartorch_register(const char* name, int D, const int64_t* dims, void* data); BARTORCH_API int bartorch_exists(const char* name); -BARTORCH_API int bartorch_lookup(const char* name, int D, long* dims, void** data); +BARTORCH_API int bartorch_lookup(const char* name, int D, int64_t* dims, void** data); BARTORCH_API int bartorch_unlink(const char* name); BARTORCH_API int bartorch_unlink_all(void); @@ -146,7 +147,7 @@ BARTORCH_API int bartorch_nufft_bf16(void); * it is one. Ordinary memory when it is not asked for, or where there is no * card: page-locking is not free, and only a crossing that overlaps something * repays it. */ -BARTORCH_API void* bartorch_host_alloc(long size, int pinned); +BARTORCH_API void* bartorch_host_alloc(int64_t size, int pinned); BARTORCH_API void bartorch_host_free(void* ptr); /* A stream of its own for bringing a function over, so the set that will be @@ -155,18 +156,18 @@ BARTORCH_API void bartorch_host_free(void* ptr); * until it has arrived, `release` says BART is done reading it. */ BARTORCH_API int bartorch_cuda_stage_open(void** stage); BARTORCH_API void bartorch_cuda_stage_close(void* stage); -BARTORCH_API int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, long size); +BARTORCH_API int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, int64_t size); BARTORCH_API int bartorch_cuda_stage_wait(void* stage, int slot); BARTORCH_API int bartorch_cuda_stage_release(void* stage, int slot); /* A copy between the card and pageable host memory through two page-locked * buffers, so it runs near the bus's rate even into pages never touched. */ -BARTORCH_API int bartorch_cuda_copy_pageable(void* dst, const void* src, long size); -BARTORCH_API void* bartorch_host_prefault_begin(void* ptr, long size); +BARTORCH_API int bartorch_cuda_copy_pageable(void* dst, const void* src, int64_t size); +BARTORCH_API void* bartorch_host_prefault_begin(void* ptr, int64_t size); BARTORCH_API void bartorch_host_prefault_end(void* handle); /* Page-lock, and release, host memory that already exists. */ -BARTORCH_API int bartorch_cuda_host_register(void* ptr, long size); +BARTORCH_API int bartorch_cuda_host_register(void* ptr, int64_t size); BARTORCH_API void bartorch_cuda_host_unregister(void* ptr); BARTORCH_API void bartorch_sense_set_coil_batch(int coils); @@ -179,13 +180,13 @@ BARTORCH_API void bartorch_sense_set_fold_maps(int enable); BARTORCH_API int bartorch_sense_fold_maps(void); /* Operators built since the last reset: 0 with the coil loop, 1 as BART's * own chain because the arrangement could not be sliced. */ -BARTORCH_API long bartorch_sense_counter(int which); +BARTORCH_API int64_t bartorch_sense_counter(int which); BARTORCH_API void bartorch_sense_reset_counters(void); BARTORCH_API int bartorch_fft_set(const char* symbol, void* fn); BARTORCH_API int bartorch_fft_usable(void); /* Plans built since the last reset: 0 by MKL, 1 by the built-in transform. */ -BARTORCH_API long bartorch_fft_counter(int which); +BARTORCH_API int64_t bartorch_fft_counter(int which); BARTORCH_API void bartorch_fft_reset_counters(void); /* @@ -209,16 +210,16 @@ typedef int (*bartorch_generic_apply_fn)(void* ctx, int N, void** args); typedef int (*bartorch_pair_apply_fn)(void* ctx, int o, int i, void* dst, const void* src); typedef void (*bartorch_release_fn)(void* ctx); -BARTORCH_API bartorch_linop* bartorch_linop_callback(int ON, const long* odims, int IN, const long* idims, +BARTORCH_API bartorch_linop* bartorch_linop_callback(int ON, const int64_t* odims, int IN, const int64_t* idims, bartorch_apply_fn forward, bartorch_apply_fn adjoint, bartorch_apply_fn normal, void* ctx, bartorch_release_fn release); -BARTORCH_API bartorch_linop* bartorch_linop_fft(int N, const long* dims, unsigned long flags, int inverse, int centered); -BARTORCH_API bartorch_linop* bartorch_linop_cdiag(int N, const long* dims, unsigned long flags, const void* diag); -BARTORCH_API bartorch_linop* bartorch_linop_fmac(int N, const long* odims, const long* idims, const long* tdims, const void* tensor); -BARTORCH_API bartorch_linop* bartorch_linop_sampling(const long* dims, const long* pat_dims, const void* pattern); -BARTORCH_API bartorch_linop* bartorch_linop_nufft(int N, const long* ksp_dims, const long* cim_dims, const long* traj_dims, - const void* traj, const long* wgh_dims, const void* weights, - const long* bas_dims, const void* basis, int toeplitz, float os, float width); +BARTORCH_API bartorch_linop* bartorch_linop_fft(int N, const int64_t* dims, uint64_t flags, int inverse, int centered); +BARTORCH_API bartorch_linop* bartorch_linop_cdiag(int N, const int64_t* dims, uint64_t flags, const void* diag); +BARTORCH_API bartorch_linop* bartorch_linop_fmac(int N, const int64_t* odims, const int64_t* idims, const int64_t* tdims, const void* tensor); +BARTORCH_API bartorch_linop* bartorch_linop_sampling(const int64_t* dims, const int64_t* pat_dims, const void* pattern); +BARTORCH_API bartorch_linop* bartorch_linop_nufft(int N, const int64_t* ksp_dims, const int64_t* cim_dims, const int64_t* traj_dims, + const void* traj, const int64_t* wgh_dims, const void* weights, + const int64_t* bas_dims, const void* basis, int toeplitz, float os, float width); /* Write a new diagonal into a `cdiag` or a sampling operator, which is the * one already built rather than a second one. The values are copied and the * cached normal is dropped, so an operator this is applied to -- and every @@ -226,7 +227,7 @@ BARTORCH_API bartorch_linop* bartorch_linop_nufft(int N, const long* ksp_dims, c * the next application. `ddims` is what the operator was built with: the * shape selected by the broadcast flags, not the operator's own. Returns * non-zero where the operator is not a diagonal or the shape disagrees. */ -BARTORCH_API int bartorch_linop_set_diagonal(const bartorch_linop* op, int N, const long* ddims, +BARTORCH_API int bartorch_linop_set_diagonal(const bartorch_linop* op, int N, const int64_t* ddims, const void* diag); /* SENSE, over sensitivities held as maps or as k-space kernels, walking the * coils a slab at a time. A NULL trajectory makes the Cartesian operator, @@ -239,11 +240,11 @@ BARTORCH_API int bartorch_linop_set_diagonal(const bartorch_linop* op, int N, co * are slowest in memory, is built on BART's dimensions without a copy. The * block operator is referenced, so the caller still frees it. */ BARTORCH_API bartorch_linop* bartorch_linop_blocks(const bartorch_linop* block, int N, - const long* odims, const long* idims, long n); + const int64_t* odims, const int64_t* idims, int64_t n); /* `op` over the same memory described by `idims` and `odims`, with nothing * copied. The operator is referenced, so the caller still frees it. */ BARTORCH_API bartorch_linop* bartorch_linop_reshaped(const bartorch_linop* op, int N, - const long* odims, const long* idims); + const int64_t* odims, const int64_t* idims); /* * The MRI encoding form. Every encoding this library builds reduces to * @@ -275,26 +276,26 @@ struct bartorch_encoding { /* The whole operator's dimensions -- spatial axes, coils, sets of * maps, coefficients -- and one coil's samples with the coefficients * a basis contracts still on them. */ - const long* max_dims; - const long* ksp_dims; + const int64_t* max_dims; + const int64_t* ksp_dims; /* The image-side factor: coil sensitivities as maps, or as the * k-space kernels they band-limit to, inflated a slab at a time. */ - const long* sens_dims; + const int64_t* sens_dims; const void* sens; int kernels; /* The k-space factors. Each pointer may be NULL, and its dimensions * are then not read. */ - const long* pat_dims; + const int64_t* pat_dims; const void* pattern; - const long* bas_dims; + const int64_t* bas_dims; const void* basis; - const long* wgh_dims; + const int64_t* wgh_dims; const void* weights; /* A NUFFT's trajectory, in grid units. */ - const long* traj_dims; + const int64_t* traj_dims; const void* traj; /* A stack whose kz lies on the image's own z grid, decoupled: the * trajectory is one position's in-plane shots, kz zero, the transform @@ -304,25 +305,25 @@ struct bartorch_encoding { /* The positions along z a stack's blocks of shots lie at, one per block, * where they are not every position in order; NULL is every position, * block j at position j. */ - long stack_count; - const long* stack_positions; + int64_t stack_count; + const int64_t* stack_positions; /* A table of the phase encodes that were sampled, instead of a dense - * pattern: `frames` x `shots` places of `components` long indices + * pattern: `frames` x `shots` places of `components` indices * each -- (y) for a 2D image, (z, y) for a 3D one, -1 for padding -- * with the whole readout along each. `kspace_readout` says the * samples are in k-space along the readout rather than transformed * back along it. NULL `positions` is dense samples. */ - long frames; - long shots; + int64_t frames; + int64_t shots; int components; - const long* positions; + const int64_t* positions; int kspace_readout; /* A wave: the oversampled readout the coil images are zero-filled to, * the point spread function over it, and whether its two transforms * are centred and unitary rather than BART's own. */ - long readout; + int64_t readout; const void* psf; int centred; @@ -330,10 +331,10 @@ struct bartorch_encoding { * diag(image_l): off-resonance by time segmentation. Each weight is * laid out on its own dimensions, the terms contiguous one after * another. Zero segments is no contraction. */ - long segments; - const long* segment_sample_dims; + int64_t segments; + const int64_t* segment_sample_dims; const void* segment_sample; - const long* segment_image_dims; + const int64_t* segment_image_dims; const void* segment_image; /* The BART dimension a batch the sensitivities vary along lies on, or @@ -347,7 +348,7 @@ struct bartorch_encoding { * trajectory: the extent of each such axis, and one elsewhere, or NULL * for none. Every item is its own transform and normal kernel under one * coil loop. */ - const long* item_dims; + const int64_t* item_dims; /* A k-space factor that differs between sets of maps, applied after * the transform, with the sets summed over after it: the slice phase @@ -355,7 +356,7 @@ struct bartorch_encoding { * sensitivities rather than being contracted by them, so the transform * runs once per set -- which is what the sum being on the far side of * it costs. NULL leaves the sets where the sensitivities sum them. */ - const long* slice_dims; + const int64_t* slice_dims; const void* slice; /* The closed-form normal rather than the two applications. */ @@ -396,11 +397,11 @@ enum bartorch_encoding_count { /* Forms built with a transform per item. */ BARTORCH_ENCODING_ITEMS = 8, }; -BARTORCH_API long bartorch_encoding_counter(int which); +BARTORCH_API int64_t bartorch_encoding_counter(int which); BARTORCH_API void bartorch_encoding_reset_counters(void); /* Normals of a Cartesian encoding applied through cuFFT's callbacks since the * library was loaded; the rest were applied as BART's chain of operators. */ -BARTORCH_API long bartorch_grid_fused(void); +BARTORCH_API int64_t bartorch_grid_fused(void); /* `a`, answering `normal` when it is asked for A^H A, rather than the adjoint * chained onto the forward. */ BARTORCH_API bartorch_linop* bartorch_linop_with_normal(const bartorch_linop* a, const bartorch_linop* normal); @@ -410,33 +411,33 @@ BARTORCH_API bartorch_linop* bartorch_linop_adjoint_op(const bartorch_linop* a); BARTORCH_API bartorch_linop* bartorch_linop_stack_cod(int n, const bartorch_linop** ops, int stack_dim); BARTORCH_API bartorch_linop* bartorch_linop_stack(int cod_dim, int dom_dim, const bartorch_linop* a, const bartorch_linop* b); BARTORCH_API bartorch_linop* bartorch_linop_normal_op(const bartorch_linop* a); -BARTORCH_API bartorch_linop* bartorch_linop_scale(int N, const long* dims, float re, float im); -BARTORCH_API bartorch_linop* bartorch_linop_zconj(int N, const long* dims); -BARTORCH_API bartorch_linop* bartorch_linop_identity(int N, const long* dims); -BARTORCH_API bartorch_linop* bartorch_linop_null(int NO, const long* odims, int NI, const long* idims); +BARTORCH_API bartorch_linop* bartorch_linop_scale(int N, const int64_t* dims, float re, float im); +BARTORCH_API bartorch_linop* bartorch_linop_zconj(int N, const int64_t* dims); +BARTORCH_API bartorch_linop* bartorch_linop_identity(int N, const int64_t* dims); +BARTORCH_API bartorch_linop* bartorch_linop_null(int NO, const int64_t* odims, int NI, const int64_t* idims); BARTORCH_API double bartorch_linop_maxeigen(const bartorch_linop* a); -BARTORCH_API bartorch_linop* bartorch_linop_zreal(int N, const long* dims); -BARTORCH_API bartorch_linop* bartorch_linop_rdiag(int N, const long* dims, unsigned long flags, const void* diag); -BARTORCH_API bartorch_linop* bartorch_linop_matrix(int N, const long* odims, const long* idims, const long* mdims, const void* matrix); -BARTORCH_API bartorch_linop* bartorch_linop_conv(int N, unsigned long flags, int ctype, int cmode, const long* odims, const long* idims, const long* kdims, const void* kernel); -BARTORCH_API bartorch_linop* bartorch_linop_grad(int N, const long* dims, int d, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_sum(int N, const long* dims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_scaled_sum(int N, const long* dims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_avg(int N, const long* dims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_repmat(int N, const long* odims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_flip(int N, const long* dims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_hankel(int N, const long* dims, int dim, int window_dim, int window); -BARTORCH_API bartorch_linop* bartorch_linop_reshape(int NO, const long* odims, int NI, const long* idims); -BARTORCH_API bartorch_linop* bartorch_linop_resize(int N, const long* odims, const long* idims); -BARTORCH_API bartorch_linop* bartorch_linop_extract(int N, const long* pos, const long* odims, const long* idims); -BARTORCH_API bartorch_linop* bartorch_linop_transpose(int N, int a, int b, const long* dims); -BARTORCH_API bartorch_linop* bartorch_linop_permute(int N, const int* order, const long* idims); -BARTORCH_API bartorch_linop* bartorch_linop_shift(int N, const long* dims, int dim, long shift, int pad); -BARTORCH_API bartorch_linop* bartorch_linop_padding(int N, const long* dims, int pad, const long* before, const long* after); +BARTORCH_API bartorch_linop* bartorch_linop_zreal(int N, const int64_t* dims); +BARTORCH_API bartorch_linop* bartorch_linop_rdiag(int N, const int64_t* dims, uint64_t flags, const void* diag); +BARTORCH_API bartorch_linop* bartorch_linop_matrix(int N, const int64_t* odims, const int64_t* idims, const int64_t* mdims, const void* matrix); +BARTORCH_API bartorch_linop* bartorch_linop_conv(int N, uint64_t flags, int ctype, int cmode, const int64_t* odims, const int64_t* idims, const int64_t* kdims, const void* kernel); +BARTORCH_API bartorch_linop* bartorch_linop_grad(int N, const int64_t* dims, int d, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_sum(int N, const int64_t* dims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_scaled_sum(int N, const int64_t* dims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_avg(int N, const int64_t* dims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_repmat(int N, const int64_t* odims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_flip(int N, const int64_t* dims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_hankel(int N, const int64_t* dims, int dim, int window_dim, int window); +BARTORCH_API bartorch_linop* bartorch_linop_reshape(int NO, const int64_t* odims, int NI, const int64_t* idims); +BARTORCH_API bartorch_linop* bartorch_linop_resize(int N, const int64_t* odims, const int64_t* idims); +BARTORCH_API bartorch_linop* bartorch_linop_extract(int N, const int64_t* pos, const int64_t* odims, const int64_t* idims); +BARTORCH_API bartorch_linop* bartorch_linop_transpose(int N, int a, int b, const int64_t* dims); +BARTORCH_API bartorch_linop* bartorch_linop_permute(int N, const int* order, const int64_t* idims); +BARTORCH_API bartorch_linop* bartorch_linop_shift(int N, const int64_t* dims, int dim, int64_t shift, int pad); +BARTORCH_API bartorch_linop* bartorch_linop_padding(int N, const int64_t* dims, int pad, const int64_t* before, const int64_t* after); BARTORCH_API int bartorch_linop_has_pseudo_inv(const bartorch_linop* h); BARTORCH_API int bartorch_linop_pseudo_inv(const bartorch_linop* h, float lambda, void* dst, const void* src); -BARTORCH_API int bartorch_linop_domain(const bartorch_linop* h, int N, long* dims); -BARTORCH_API int bartorch_linop_codomain(const bartorch_linop* h, int N, long* dims); +BARTORCH_API int bartorch_linop_domain(const bartorch_linop* h, int N, int64_t* dims); +BARTORCH_API int bartorch_linop_codomain(const bartorch_linop* h, int N, int64_t* dims); BARTORCH_API int bartorch_linop_forward(const bartorch_linop* h, void* dst, const void* src); BARTORCH_API int bartorch_linop_adjoint(const bartorch_linop* h, void* dst, const void* src); BARTORCH_API int bartorch_linop_normal(const bartorch_linop* h, void* dst, const void* src); @@ -517,7 +518,7 @@ BARTORCH_API int bartorch_maxeigen(const bartorch_linop* A, const bartorch_linop BARTORCH_API int bartorch_solve(const bartorch_linop* A, const char* algorithm, - const char* const* reg_kinds, const long* reg_xflags, const long* reg_jflags, + const char* const* reg_kinds, const uint64_t* reg_xflags, const uint64_t* reg_jflags, const float* reg_lambda, const int* reg_k, const bartorch_prox* const* reg_ops, int n_reg, float cclambda, int maxiter, float step, int eigen, int hogwild, @@ -538,7 +539,7 @@ BARTORCH_API int bartorch_solve(const bartorch_linop* A, * BART's own. Ignored otherwise. */ int llr_blk, const char* wavelet, int shift_mode, const float* alpha, const float* gamma, - void* x, const void* y, long* iterations); + void* x, const void* y, int64_t* iterations); BARTORCH_API const char* bartorch_solve_error(int code); /* @@ -551,7 +552,7 @@ BARTORCH_API const char* bartorch_solve_error(int code); * sorts what it is given. `p` is a percentile in (0, 1], or negative for the * rule `pics` uses. */ -BARTORCH_API float bartorch_scaling_norm(long size, const void* image, float rescale, +BARTORCH_API float bartorch_scaling_norm(int64_t size, const void* image, float rescale, int compat, float p); /* @@ -572,9 +573,9 @@ BARTORCH_API float bartorch_scaling_norm(long size, const void* image, float res * generator of its own, which `bartorch_solve` rewinds before each solve, so a * term that is reused is a term freshly built as far as the answer goes. */ -BARTORCH_API int bartorch_prox_create(const char* kind, long xflags, long jflags, +BARTORCH_API int bartorch_prox_create(const char* kind, uint64_t xflags, uint64_t jflags, float lambda, int k, int llr_blk, const char* wavelet, int shift_mode, - const long* img_dims, bartorch_prox** out); + const int64_t* img_dims, bartorch_prox** out); /* A set of terms configured together, as `bartorch_solve` configures a set * holding a term that extends the optimisation variable. * @@ -586,13 +587,13 @@ BARTORCH_API int bartorch_prox_create(const char* kind, long xflags, long jflags * `bartorch_prox_free`. `alpha` and `gamma` are the set's pairs, or NULL. */ BARTORCH_API int bartorch_prox_set_create(int n, const char* const* kinds, - const long* xflags, const long* jflags, const float* lambda, const int* k, + const uint64_t* xflags, const uint64_t* jflags, const float* lambda, const int* k, int llr_blk, const char* wavelet, int shift_mode, - const float* alpha, const float* gamma, const long* img_dims, - int max_out, bartorch_prox** out, int* count, long* svars); + const float* alpha, const float* gamma, const int64_t* img_dims, + int max_out, bartorch_prox** out, int* count, int64_t* svars); /* The shape a term's proximal operator works on -- the image's, or the * codomain of the transform the term applies first. Returns the rank. */ -BARTORCH_API int bartorch_prox_domain(const bartorch_prox* h, int N, long* dims); +BARTORCH_API int bartorch_prox_domain(const bartorch_prox* h, int N, int64_t* dims); /* prox_{gamma f}(src) into dst, over that shape. */ BARTORCH_API int bartorch_prox_apply(const bartorch_prox* h, float gamma, void* dst, const void* src); /* The transform applied in place, for the one whose rank an operator here @@ -615,7 +616,7 @@ BARTORCH_API int bartorch_prox_transform_is_identity(const bartorch_prox* h); BARTORCH_API int bartorch_prox_rewind(const bartorch_prox* h); BARTORCH_API void bartorch_prox_free(bartorch_prox* h); -BARTORCH_API bartorch_nlop* bartorch_nlop_callback(int ON, const long* odims, int IN, const long* idims, +BARTORCH_API bartorch_nlop* bartorch_nlop_callback(int ON, const int64_t* odims, int IN, const int64_t* idims, bartorch_apply_fn forward, bartorch_apply_fn derivative, bartorch_apply_fn adjoint, void* ctx, bartorch_release_fn release); @@ -630,16 +631,16 @@ BARTORCH_API bartorch_nlop* bartorch_nlop_callback(int ON, const long* odims, in * What this is for is a Python function with more than one argument standing * inside a BART graph: a denoiser whose weights are an *input* rather than * something it closed over, so that a gradient reaches them. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const long* odims, - int II, int IN, const long* idims, +BARTORCH_API bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const int64_t* odims, + int II, int IN, const int64_t* idims, bartorch_generic_apply_fn forward, bartorch_pair_apply_fn derivative, bartorch_pair_apply_fn adjoint, void* ctx, bartorch_release_fn release); BARTORCH_API bartorch_nlop* bartorch_nlop_from_linop(const bartorch_linop* lin); BARTORCH_API bartorch_nlop* bartorch_nlop_chain(const bartorch_nlop* a, const bartorch_nlop* b); -BARTORCH_API int bartorch_nlop_domain(const bartorch_nlop* h, int N, long* dims); -BARTORCH_API int bartorch_nlop_codomain(const bartorch_nlop* h, int N, long* dims); +BARTORCH_API int bartorch_nlop_domain(const bartorch_nlop* h, int N, int64_t* dims); +BARTORCH_API int bartorch_nlop_codomain(const bartorch_nlop* h, int N, int64_t* dims); BARTORCH_API int bartorch_nlop_apply(const bartorch_nlop* h, void* dst, const void* src); BARTORCH_API int bartorch_nlop_derivative(const bartorch_nlop* h, void* dst, const void* src); BARTORCH_API int bartorch_nlop_adjoint(const bartorch_nlop* h, void* dst, const void* src); @@ -653,8 +654,8 @@ BARTORCH_API int bartorch_nlop_adjoint(const bartorch_nlop* h, void* dst, const */ BARTORCH_API int bartorch_nlop_inputs(const bartorch_nlop* h); BARTORCH_API int bartorch_nlop_outputs(const bartorch_nlop* h); -BARTORCH_API int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, long* dims); -BARTORCH_API int bartorch_nlop_output_codomain(const bartorch_nlop* h, int o, int N, long* dims); +BARTORCH_API int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, int64_t* dims); +BARTORCH_API int bartorch_nlop_output_codomain(const bartorch_nlop* h, int o, int N, int64_t* dims); /* Apply an operator of any arity; `args` is outputs then inputs. Fixes the * point every derivative is taken at, as the one-argument apply does. */ BARTORCH_API int bartorch_nlop_apply_generic(const bartorch_nlop* h, int nargs, void** args); @@ -676,8 +677,8 @@ BARTORCH_API bartorch_linop* bartorch_nlop_derivative_linop(const bartorch_nlop* * the callbacks above is built at DIMS. So two arguments of the same shape * can still refuse to meet. Padding a shape with ones is not a change to it, * and this is how that is said. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_reshape_in(const bartorch_nlop* a, int i, int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_reshape_out(const bartorch_nlop* a, int o, int N, const long* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_reshape_in(const bartorch_nlop* a, int i, int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_reshape_out(const bartorch_nlop* a, int o, int N, const int64_t* dims); BARTORCH_API bartorch_nlop* bartorch_nlop_chain2(const bartorch_nlop* a, int o, const bartorch_nlop* b, int i); BARTORCH_API bartorch_nlop* bartorch_nlop_combine(const bartorch_nlop* a, const bartorch_nlop* b); @@ -706,22 +707,22 @@ BARTORCH_API bartorch_nlop* bartorch_nlop_flatten(const bartorch_nlop* x, int in * constructor cannot be called at all. The same operator is built out of * the two pieces on the Python side, where the indices are right. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_tenmul(int N, const long* odims, const long* idims1, const long* idims2); -BARTORCH_API bartorch_nlop* bartorch_nlop_zdiv(int N, const long* dims, float eps); -BARTORCH_API bartorch_nlop* bartorch_nlop_zaxpbz(int N, const long* dims, float a, float b); -BARTORCH_API bartorch_nlop* bartorch_nlop_zexp(int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_zlog(int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_zinv(int N, const long* dims, float eps); -BARTORCH_API bartorch_nlop* bartorch_nlop_zsqrt(int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_zspow(int N, const long* dims, float re, float im); -BARTORCH_API bartorch_nlop* bartorch_nlop_zsadd(int N, const long* dims, float re, float im); -BARTORCH_API bartorch_nlop* bartorch_nlop_zabs(int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_smo_abs(int N, const long* dims, float eps); -BARTORCH_API bartorch_nlop* bartorch_nlop_zrss(int N, const long* dims, unsigned long flags, float eps); -BARTORCH_API bartorch_nlop* bartorch_nlop_zss(int N, const long* dims, unsigned long flags); +BARTORCH_API bartorch_nlop* bartorch_nlop_tenmul(int N, const int64_t* odims, const int64_t* idims1, const int64_t* idims2); +BARTORCH_API bartorch_nlop* bartorch_nlop_zdiv(int N, const int64_t* dims, float eps); +BARTORCH_API bartorch_nlop* bartorch_nlop_zaxpbz(int N, const int64_t* dims, float a, float b); +BARTORCH_API bartorch_nlop* bartorch_nlop_zexp(int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_zlog(int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_zinv(int N, const int64_t* dims, float eps); +BARTORCH_API bartorch_nlop* bartorch_nlop_zsqrt(int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_zspow(int N, const int64_t* dims, float re, float im); +BARTORCH_API bartorch_nlop* bartorch_nlop_zsadd(int N, const int64_t* dims, float re, float im); +BARTORCH_API bartorch_nlop* bartorch_nlop_zabs(int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_smo_abs(int N, const int64_t* dims, float eps); +BARTORCH_API bartorch_nlop* bartorch_nlop_zrss(int N, const int64_t* dims, uint64_t flags, float eps); +BARTORCH_API bartorch_nlop* bartorch_nlop_zss(int N, const int64_t* dims, uint64_t flags); /* An operator of no inputs, and pinning one input of an operator to a value. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_const(int N, const long* dims, const void* val); -BARTORCH_API bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int N, const long* dims, const void* val); +BARTORCH_API bartorch_nlop* bartorch_nlop_const(int N, const int64_t* dims, const void* val); +BARTORCH_API bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int N, const int64_t* dims, const void* val); BARTORCH_API void bartorch_nlop_free(bartorch_nlop* h); @@ -738,7 +739,7 @@ BARTORCH_API void bartorch_nlop_free(bartorch_nlop* h); * another; with more than one, the conjugate gradients keep their step lengths * and stopping test per item. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* normal, int maxiter, float tol, float l2lambda, long batch); +BARTORCH_API bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* normal, int maxiter, float tol, float l2lambda, int64_t batch); /* The nonlinear SENSE model `nlinv` inverts, from `noir/model2.c`. * @@ -762,15 +763,15 @@ BARTORCH_API bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* n * input, which the arity queries above report. */ BARTORCH_API bartorch_noir* bartorch_noir_create(int N, - const long* ksp_dims, const long* cim_dims, const long* img_dims, - const long* kco_dims, const long* col_dims, - const long* pat_dims, const void* pattern, - const long* trj_dims, const void* traj, - const long* wgh_dims, const void* weights, - const long* bas_dims, const void* basis, - const long* msk_dims, const void* mask, + const int64_t* ksp_dims, const int64_t* cim_dims, const int64_t* img_dims, + const int64_t* kco_dims, const int64_t* col_dims, + const int64_t* pat_dims, const void* pattern, + const int64_t* trj_dims, const void* traj, + const int64_t* wgh_dims, const void* weights, + const int64_t* bas_dims, const void* basis, + const int64_t* msk_dims, const void* mask, int noncart, int optimized, int toeplitz, - unsigned long fft_flags, unsigned long wght_flags, + uint64_t fft_flags, uint64_t wght_flags, int rvc, int sos, float a, float b, float c, float oversampling_coils, int ret_os_coils); BARTORCH_API bartorch_nlop* bartorch_noir_model(const bartorch_noir* h); @@ -778,7 +779,7 @@ BARTORCH_API bartorch_linop* bartorch_noir_coils(const bartorch_noir* h); BARTORCH_API bartorch_linop* bartorch_noir_image(const bartorch_noir* h); BARTORCH_API bartorch_linop* bartorch_noir_data(const bartorch_noir* h); BARTORCH_API bartorch_linop* bartorch_noir_transform(const bartorch_noir* h); -BARTORCH_API int bartorch_noir_dims(const bartorch_noir* h, int which, int N, long* dims); +BARTORCH_API int bartorch_noir_dims(const bartorch_noir* h, int which, int N, int64_t* dims); BARTORCH_API void bartorch_noir_free(bartorch_noir* h); /* The same model, built for a network rather than for a solve: `noir2_net_s` @@ -854,7 +855,7 @@ BARTORCH_API int bartorch_cuda_use_memcache(int enable); BARTORCH_API void bartorch_cuda_memcache_clear_all(void); BARTORCH_API int bartorch_cuda_wait_for_stream(void* stream); BARTORCH_API int bartorch_cuda_signal_stream(void* stream); -BARTORCH_API long bartorch_cuda_free_memory(void); +BARTORCH_API int64_t bartorch_cuda_free_memory(void); /* * FINUFFT under BART's own NUFFT operator, which is what makes nufft, pics, @@ -888,7 +889,7 @@ BARTORCH_API int bartorch_finufft_usable_on(int device); BARTORCH_API int bartorch_finufft_usable(void); /* Plans made and not yet destroyed: zero once every operator, point spread * function and mask that asked for one has been freed. */ -BARTORCH_API long bartorch_finufft_live_plans(void); +BARTORCH_API int64_t bartorch_finufft_live_plans(void); BARTORCH_API const char* bartorch_last_error(void); BARTORCH_API void bartorch_clear_error(void); BARTORCH_API int bartorch_nufft_decline_reason(void); @@ -896,7 +897,7 @@ BARTORCH_API const char* bartorch_nufft_decline_text(void); BARTORCH_API void bartorch_nufft_allow_fallback(int enable); BARTORCH_API int bartorch_nufft_fallback_allowed(void); /* Operators built since the last reset: 0 by FINUFFT, 1 by BART. */ -BARTORCH_API long bartorch_nufft_counter(int which); +BARTORCH_API int64_t bartorch_nufft_counter(int which); BARTORCH_API void bartorch_nufft_reset_counters(void); /* @@ -909,7 +910,7 @@ BARTORCH_API void bartorch_nufft_reset_counters(void); * Normal operators since the last reset: 0 answered by a point spread * function, 1 by the transform pair. */ -BARTORCH_API long bartorch_toeplitz_counter(int which); +BARTORCH_API int64_t bartorch_toeplitz_counter(int which); BARTORCH_API void bartorch_toeplitz_reset_counters(void); /* Whether a pointer is device memory; always false without CUDA. */ diff --git a/src/csrc/ops/coset.cuh b/src/csrc/ops/coset.cuh index 3495b0a1..3c5b8f9e 100644 --- a/src/csrc/ops/coset.cuh +++ b/src/csrc/ops/coset.cuh @@ -26,15 +26,15 @@ struct phase_conf { - long dims[3]; - long tot; - long batch; + bart_dim_t dims[3]; + bart_dim_t tot; + bart_dim_t batch; float shifts[3]; float cn; float scale; }; -static inline struct phase_conf phase_setup(int N, const long dims[], const float shift[3], float scale) +static inline struct phase_conf phase_setup(int N, const bart_dim_t dims[], const float shift[3], float scale) { struct phase_conf c; @@ -55,7 +55,7 @@ static inline struct phase_conf phase_setup(int N, const long dims[], const floa c.dims[n] = dims[n]; c.tot *= dims[n]; - long centre = dims[n] / 2; + bart_dim_t centre = dims[n] / 2; double half = (double)centre / (double)dims[n]; c.shifts[n] += 2. * M_PI * half; @@ -70,7 +70,7 @@ static inline struct phase_conf phase_setup(int N, const long dims[], const floa return c; } -__device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, long x, long y, long z, bool conj) +__device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, bart_dim_t x, bart_dim_t y, bart_dim_t z, bool conj) { float val = c.cn + x * c.shifts[0] + y * c.shifts[1] + z * c.shifts[2]; @@ -85,7 +85,7 @@ __device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, lon } /* Where grid point `i` sits in the gathered spectrum, or -1 where it is not kept. */ -__device__ static inline long kept_at(const unsigned int* mask, const int* prefix, long i) +__device__ static inline bart_dim_t kept_at(const unsigned int* mask, const int* prefix, bart_dim_t i) { unsigned int word = mask[i >> 5]; unsigned int bit = 1u << (i & 31); @@ -93,7 +93,7 @@ __device__ static inline long kept_at(const unsigned int* mask, const int* prefi if (0 == (word & bit)) return -1; - return (long)prefix[i >> 5] + __popc(word & (bit - 1)); + return (bart_dim_t)prefix[i >> 5] + __popc(word & (bit - 1)); } /* What the callbacks of one transform read: a set's phase, a coil's diff --git a/src/csrc/ops/fft_callbacks.cu b/src/csrc/ops/fft_callbacks.cu index 6c07b170..172b6eda 100644 --- a/src/csrc/ops/fft_callbacks.cu +++ b/src/csrc/ops/fft_callbacks.cu @@ -83,7 +83,7 @@ extern "C" void bartorch_cb_fft_free(struct bartorch_cb_fft* p) /* The pair for a volume of `dims`, or NULL where cuFFT cannot link the * callbacks in. */ -extern "C" struct bartorch_cb_fft* bartorch_cb_fft_create(const long dims[3]) +extern "C" struct bartorch_cb_fft* bartorch_cb_fft_create(const bart_dim_t dims[3]) { set_jit_t set_jit = set_jit_callback(); @@ -105,7 +105,7 @@ extern "C" struct bartorch_cb_fft* bartorch_cb_fft_create(const long dims[3]) return NULL; /* The callbacks index the volume in 32 bits. */ - if (dims[0] * dims[1] * dims[2] > (long)UINT32_MAX) + if (dims[0] * dims[1] * dims[2] > (bart_dim_t)UINT32_MAX) return NULL; struct bartorch_cb_fft* p = (struct bartorch_cb_fft*)xmalloc(sizeof *p); @@ -190,7 +190,7 @@ static void run(struct bartorch_cb_fft* p, int which, const struct coset_info* i CUDA_KERNEL_ERROR; - void* work = (0 < p->work) ? cuda_malloc((long)p->work) : NULL; + void* work = (0 < p->work) ? cuda_malloc((bart_dim_t)p->work) : NULL; cufftHandle plan = p->plan[which]; @@ -205,7 +205,7 @@ static void run(struct bartorch_cb_fft* p, int which, const struct coset_info* i cuda_free(work); } -static struct coset_info info_for(int N, const long dims[], const float shift[3], float scale, +static struct coset_info info_for(int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, const _Complex float* map) { struct coset_info c; @@ -223,7 +223,7 @@ static struct coset_info info_for(int N, const long dims[], const float shift[3] /* The coefficient `src`, multiplied by the set's phase and the coil's * sensitivity, transformed, and gathered into `bank`. `volume` is where the * transform works; `src` is left as it is. */ -extern "C" void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const long dims[], +extern "C" void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, _Complex float* bank, _Complex float* volume, const _Complex float* src, const _Complex float* map) { @@ -236,7 +236,7 @@ extern "C" void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const /* The gathered spectrum `bank`, scattered, transformed back, multiplied by the * conjugates of the phase and the sensitivity, and added to `dst`. */ -extern "C" void bartorch_cb_fft_inverse(struct bartorch_cb_fft* p, int N, const long dims[], +extern "C" void bartorch_cb_fft_inverse(struct bartorch_cb_fft* p, int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, _Complex float* dst, _Complex float* volume, const _Complex float* bank, const _Complex float* map) { @@ -291,7 +291,7 @@ extern "C" void bartorch_cb_grid_free(struct bartorch_cb_grid* p) * and stay the caller's. `unitary` scales each direction by one over the * square root of the plane; without it the forward is cuFFT's unnormalized * transform and the inverse its adjoint, as BART's uncentred `fft` is. */ -extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], unsigned long flags, long kept, +extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const bart_dim_t dims[3], bart_flags_t flags, bart_dim_t kept, const unsigned int* mask, const int* prefix, const _Complex float* mod[3], int unitary) { set_jit_t set_jit = set_jit_callback(); @@ -299,7 +299,7 @@ extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], if (NULL == set_jit) return NULL; - if (dims[0] * dims[1] * dims[2] > (long)UINT32_MAX) + if (dims[0] * dims[1] * dims[2] > (bart_dim_t)UINT32_MAX) return NULL; struct grid_info g; @@ -309,7 +309,7 @@ extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], int rank = 0; for (int i = 2; i >= 0; i--) - if ((0 != ((flags >> i) & 1UL)) && (1 < dims[i])) + if ((0 != ((flags >> i) & 1)) && (1 < dims[i])) n[rank++] = (int)dims[i]; if (0 == rank) @@ -321,7 +321,7 @@ extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], for (int a = 0; a < 3; a++) { - bool t = (0 != ((flags >> a) & 1UL)) && (1 < dims[a]); + bool t = (0 != ((flags >> a) & 1)) && (1 < dims[a]); g.n[a] = (unsigned int)dims[a]; g.mstr[a] = ms; @@ -421,7 +421,7 @@ static void run_grid(struct bartorch_cb_grid* p, int which, const struct grid_in CUDA_KERNEL_ERROR; - void* work = (0 < p->work) ? cuda_malloc((long)p->work) : NULL; + void* work = (0 < p->work) ? cuda_malloc((bart_dim_t)p->work) : NULL; cufftHandle plan = p->plan[which]; diff --git a/src/csrc/ops/fft_callbacks_lto.cu b/src/csrc/ops/fft_callbacks_lto.cu index d33a54f5..5e21fbb1 100644 --- a/src/csrc/ops/fft_callbacks_lto.cu +++ b/src/csrc/ops/fft_callbacks_lto.cu @@ -26,7 +26,7 @@ __device__ static inline cuFloatComplex phase_of(const struct coset_info* c, uns unsigned int ny = (unsigned int)c->phase.dims[1]; unsigned int yz = i / nx; - return phase_at(c->phase, (long)(i - yz * nx), (long)(yz % ny), (long)(yz / ny), conj); + return phase_at(c->phase, (bart_dim_t)(i - yz * nx), (bart_dim_t)(yz % ny), (bart_dim_t)(yz / ny), conj); } __device__ cufftComplex bartorch_load_in(void* in, unsigned long long off, void* info, void* shared) @@ -45,7 +45,7 @@ __device__ void bartorch_store_gather(void* out, unsigned long long off, cufftCo { const struct coset_info* c = (const struct coset_info*)info; - long j = kept_at(c->mask, c->prefix, (long)off); + bart_dim_t j = kept_at(c->mask, c->prefix, (bart_stride_t)off); if (0 <= j) c->bank[j] = val; @@ -55,7 +55,7 @@ __device__ cufftComplex bartorch_load_scatter(void* in, unsigned long long off, { const struct coset_info* c = (const struct coset_info*)info; - long j = kept_at(c->mask, c->prefix, (long)off); + bart_dim_t j = kept_at(c->mask, c->prefix, (bart_stride_t)off); return (0 <= j) ? c->bank[j] : make_cuFloatComplex(0.f, 0.f); } @@ -102,7 +102,7 @@ __device__ void bartorch_grid_store_gather(void* out, unsigned long long off, cu { const struct grid_info* g = (const struct grid_info*)info; - long j = grid_kept(g, (unsigned int)off); + bart_dim_t j = grid_kept(g, (unsigned int)off); if (0 <= j) g->bank[j] = val; @@ -112,7 +112,7 @@ __device__ cufftComplex bartorch_grid_load_scatter(void* in, unsigned long long { const struct grid_info* g = (const struct grid_info*)info; - long j = grid_kept(g, (unsigned int)off); + bart_dim_t j = grid_kept(g, (unsigned int)off); return (0 <= j) ? g->bank[j] : make_cuFloatComplex(0.f, 0.f); } diff --git a/src/csrc/ops/grid.c b/src/csrc/ops/grid.c index 6299d43a..1dfded18 100644 --- a/src/csrc/ops/grid.c +++ b/src/csrc/ops/grid.c @@ -59,18 +59,18 @@ #ifdef USE_CUDA struct bartorch_cb_grid; -extern struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], unsigned long flags, long kept, +extern struct bartorch_cb_grid* bartorch_cb_grid_create(const bart_dim_t dims[3], bart_flags_t flags, bart_dim_t kept, const unsigned int* mask, const int* prefix, const complex float* mod[3], int unitary); -extern void bartorch_cuda_pad_map(long sx, long wx, long rest, long off, +extern void bartorch_cuda_pad_map(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, complex float* dst, const complex float* src, const complex float* map); -extern void bartorch_cuda_crop_mapc_add(long sx, long wx, long rest, long off, +extern void bartorch_cuda_crop_mapc_add(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, complex float* dst, const complex float* src, const complex float* map); extern void bartorch_cb_grid_free(struct bartorch_cb_grid* p); extern void bartorch_cb_grid_forward(struct bartorch_cb_grid* p, complex float* bank, complex float* volume, const complex float* src, const complex float* map); extern void bartorch_cb_grid_inverse(struct bartorch_cb_grid* p, complex float* dst, complex float* volume, const complex float* bank, const complex float* map); -extern int bartorch_cuda_contract_grid(long L, long B, int R, complex float* bank, const complex float* K); +extern int bartorch_cuda_contract_grid(bart_dim_t L, bart_dim_t B, int R, complex float* bank, const complex float* K); /* What the kernels between the gathered spectrum and a table of samples read * of the transform's layout (kernels.cu). */ @@ -81,17 +81,17 @@ struct bartorch_grid_axes { const complex float* mod[3]; }; -extern void bartorch_cuda_bank_to_table(long E, long X, long S, int R, long L, long per, +extern void bartorch_cuda_bank_to_table(bart_dim_t E, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* entry_u, const int* u_coord, const unsigned int* mask, const int* prefix, const struct bartorch_grid_axes* ax, const complex float* B, const complex float* bank, complex float* table); -extern void bartorch_cuda_table_to_bank(long U, long X, long S, int R, long L, long per, +extern void bartorch_cuda_table_to_bank(bart_dim_t U, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* csr_start, const int* csr, const int* u_coord, const unsigned int* mask, const int* prefix, const struct bartorch_grid_axes* ax, const complex float* B, complex float* bank, const complex float* table); #endif -static long grid_fused_count = 0; +static bart_dim_t grid_fused_count = 0; -long bartorch_grid_fused(void) +int64_t bartorch_grid_fused(void) { return grid_fused_count; } @@ -100,8 +100,8 @@ struct grid_s { linop_data_t super; - long cim_dims[DIMS]; - long out_dims[DIMS]; + bart_dim_t cim_dims[DIMS]; + bart_dim_t out_dims[DIMS]; const struct linop_s* fwd; /* transform, basis, pattern */ const struct linop_s* nrm; /* the normal as BART's chain */ @@ -109,13 +109,13 @@ struct grid_s { /* The normal through cuFFT's callbacks: the axes it transforms, the * places of a plane the pattern keeps, and the kernel at each of them * (NULL where it is one), on the host. */ - unsigned long flags; - long R; - long vol; - long plane; - long batch; - long kept; - long words; + bart_flags_t flags; + bart_dim_t R; + bart_dim_t vol; + bart_dim_t plane; + bart_dim_t batch; + bart_dim_t kept; + bart_dim_t words; unsigned int* mask; int* prefix; complex float* kernel; @@ -143,9 +143,9 @@ struct grid_s { * by `psf` (over the spatial axes of `cim_dims`). Without a wave * `dom_dims` is `cim_dims` and `img_vol` is `vol`. */ bool wave; - long dom_dims[DIMS]; - long img_vol; - long wave_off; + bart_dim_t dom_dims[DIMS]; + bart_dim_t img_vol; + bart_stride_t wave_off; complex float* psf; complex float* dpsf; @@ -157,11 +157,11 @@ struct grid_s { * `kspace_readout` says the samples are in k-space along the readout. */ bool sampled; bool kspace_readout; - long T; - long S; - long X; - long E; - long U; + bart_dim_t T; + bart_dim_t S; + bart_dim_t X; + bart_dim_t E; + bart_dim_t U; int* entry_u; int* u_coord; int* csr_start; @@ -177,9 +177,9 @@ struct grid_s { static DEF_TYPEID(grid_s); /* The spatial axes along which the pattern has more than one value. */ -static unsigned long varying(const long pat_dims[DIMS]) +static bart_flags_t varying(const bart_dim_t pat_dims[DIMS]) { - unsigned long flags = 0UL; + bart_flags_t flags = 0; for (int a = 0; a < 3; a++) if (1 < pat_dims[a]) @@ -192,12 +192,12 @@ static unsigned long varying(const long pat_dims[DIMS]) * arrangement a single multiply-accumulate contracts -- and the pattern's * spatial axes, and one everywhere else. Computed on the host: it is * coefficients squared over a plane of phase encodes. */ -static complex float* grid_kernel(long kdims[DIMS], const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis) +static complex float* grid_kernel(bart_dim_t kdims[DIMS], const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis) { - long R = (NULL == basis) ? 1 : bas_dims[COEFF_DIM]; - long T = (NULL == basis) ? 1 : bas_dims[TE_DIM]; - long F = (NULL == pattern) ? 1 : pat_dims[TE_DIM]; + bart_dim_t R = (NULL == basis) ? 1 : bas_dims[COEFF_DIM]; + bart_dim_t T = (NULL == basis) ? 1 : bas_dims[TE_DIM]; + bart_dim_t F = (NULL == pattern) ? 1 : pat_dims[TE_DIM]; md_singleton_dims(DIMS, kdims); @@ -208,7 +208,7 @@ static complex float* grid_kernel(long kdims[DIMS], const long pat_dims[DIMS], c kdims[TE_DIM] = R; kdims[COEFF_DIM] = R; - long places = md_calc_size(3, kdims); + bart_dim_t places = md_calc_size(3, kdims); complex float* P = NULL; @@ -229,22 +229,22 @@ static complex float* grid_kernel(long kdims[DIMS], const long pat_dims[DIMS], c /* conj(B[k', t]) B[k, t], once per frame. */ complex float* G = xmalloc((size_t)(T * R * R) * sizeof(complex float)); - for (long t = 0; t < T; t++) - for (long kp = 0; kp < R; kp++) - for (long k = 0; k < R; k++) + for (bart_dim_t t = 0; t < T; t++) + for (bart_dim_t kp = 0; kp < R; kp++) + for (bart_dim_t k = 0; k < R; k++) G[t * R * R + kp * R + k] = (NULL == B) ? 1.f : conjf(B[t + T * kp]) * B[t + T * k]; complex float* K = md_alloc(DIMS, kdims, CFL_SIZE); #pragma omp parallel for - for (long p = 0; p < places; p++) { + for (bart_dim_t p = 0; p < places; p++) { - for (long kp = 0; kp < R; kp++) - for (long k = 0; k < R; k++) { + for (bart_dim_t kp = 0; kp < R; kp++) + for (bart_dim_t k = 0; k < R; k++) { complex double acc = 0.; - for (long t = 0; t < T; t++) { + for (bart_dim_t t = 0; t < T; t++) { double w = 1.; @@ -274,12 +274,12 @@ static complex float* grid_kernel(long kdims[DIMS], const long pat_dims[DIMS], c * samples, as a bit per place and a count before each word, and the kernel at * those places alone, `kernel[(l R + r) R + c]` -- or no kernel, where every * kept place's is one. */ -static void grid_compress(struct grid_s* d, const long kdims[DIMS], const complex float* K, - const long pat_dims[DIMS], const complex float* pattern) +static void grid_compress(struct grid_s* d, const bart_dim_t kdims[DIMS], const complex float* K, + const bart_dim_t pat_dims[DIMS], const complex float* pattern) { - long places = md_calc_size(3, kdims); - long R = d->R; - long F = pat_dims[TE_DIM]; + bart_dim_t places = md_calc_size(3, kdims); + bart_dim_t R = d->R; + bart_dim_t F = pat_dims[TE_DIM]; complex float* P = md_alloc(DIMS, pat_dims, CFL_SIZE); md_copy(DIMS, pat_dims, P, pattern, CFL_SIZE); @@ -288,17 +288,17 @@ static void grid_compress(struct grid_s* d, const long kdims[DIMS], const comple d->mask = xmalloc((size_t)d->words * sizeof(unsigned int)); d->prefix = xmalloc((size_t)d->words * sizeof(int)); - long n = 0; + bart_dim_t n = 0; - for (long w = 0; w < d->words; w++) { + for (bart_dim_t w = 0; w < d->words; w++) { unsigned int bits = 0; - for (long b = 0; (b < 32) && (w * 32 + b < places); b++) { + for (bart_dim_t b = 0; (b < 32) && (w * 32 + b < places); b++) { bool taken = false; - for (long t = 0; (t < F) && !taken; t++) + for (bart_dim_t t = 0; (t < F) && !taken; t++) taken = (0. != cabsf(P[w * 32 + b + places * t])); if (taken) @@ -318,15 +318,15 @@ static void grid_compress(struct grid_s* d, const long kdims[DIMS], const comple complex float* kernel = xmalloc((size_t)MAX(1, n * R * R) * sizeof(complex float)); bool ones = (1 == R); - long l = 0; + bart_dim_t l = 0; - for (long p = 0; p < places; p++) { + for (bart_dim_t p = 0; p < places; p++) { if (0 == (d->mask[p >> 5] & (1u << (p & 31)))) continue; - for (long r = 0; r < R; r++) - for (long c = 0; c < R; c++) { + for (bart_dim_t r = 0; r < R; r++) + for (bart_dim_t c = 0; c < R; c++) { complex float v = K[p + places * (r + R * c)]; @@ -354,13 +354,13 @@ static void grid_compress(struct grid_s* d, const long kdims[DIMS], const comple if (!MD_IS_SET(d->flags, a)) continue; - long ad[1] = { d->cim_dims[a] }; + bart_dim_t ad[1] = { d->cim_dims[a] }; d->mod[a] = md_alloc(1, ad, CFL_SIZE); md_zfill(1, ad, d->mod[a], 1.); if (d->centred) - fftmod(1, ad, 1UL, d->mod[a], d->mod[a]); + fftmod(1, ad, 1, d->mod[a], d->mod[a]); } } @@ -368,7 +368,7 @@ static void grid_compress(struct grid_s* d, const long kdims[DIMS], const comple * on, making what it needs there the first time it is asked. */ static bool grid_ready(struct grid_s* d, const void* ref) { - if (0UL == d->flags) + if (0 == d->flags) return false; #ifdef USE_CUDA @@ -379,7 +379,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) d->tried = true; - long wd[1] = { d->words }; + bart_dim_t wd[1] = { d->words }; d->dmask = md_alloc_gpu(1, wd, sizeof(unsigned int)); d->dprefix = md_alloc_gpu(1, wd, sizeof(int)); @@ -388,7 +388,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (NULL != d->kernel) { - long kd[1] = { d->kept * d->R * d->R }; + bart_dim_t kd[1] = { d->kept * d->R * d->R }; d->dkernel = md_alloc_gpu(1, kd, CFL_SIZE); md_copy(1, kd, d->dkernel, d->kernel, CFL_SIZE); @@ -399,7 +399,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (NULL == d->mod[a]) continue; - long ad[1] = { d->cim_dims[a] }; + bart_dim_t ad[1] = { d->cim_dims[a] }; d->dmod[a] = md_alloc_gpu(1, ad, CFL_SIZE); md_copy(1, ad, d->dmod[a], d->mod[a], CFL_SIZE); @@ -407,7 +407,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (NULL != d->psf) { - long pd[1] = { d->vol }; + bart_dim_t pd[1] = { d->vol }; d->dpsf = md_alloc_gpu(1, pd, CFL_SIZE); md_copy(1, pd, d->dpsf, d->psf, CFL_SIZE); @@ -415,10 +415,10 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (d->sampled) { - long ed[1] = { MAX(1, d->E) }; - long ud[1] = { MAX(1, 2 * d->U) }; - long sd[1] = { d->U + 1 }; - long cd[1] = { MAX(1, d->csr_start[d->U]) }; + bart_dim_t ed[1] = { MAX(1, d->E) }; + bart_dim_t ud[1] = { MAX(1, 2 * d->U) }; + bart_dim_t sd[1] = { d->U + 1 }; + bart_dim_t cd[1] = { MAX(1, d->csr_start[d->U]) }; d->d_entry_u = md_alloc_gpu(1, ed, sizeof(int)); d->d_u_coord = md_alloc_gpu(1, ud, sizeof(int)); @@ -431,7 +431,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (NULL != d->bt) { - long bd[1] = { d->T * d->R }; + bart_dim_t bd[1] = { d->T * d->R }; d->d_bt = md_alloc_gpu(1, bd, CFL_SIZE); md_copy(1, bd, d->d_bt, d->bt, CFL_SIZE); @@ -454,9 +454,9 @@ static bool grid_ready(struct grid_s* d, const void* ref) /* BART's transform along `flags` in place, in the grid's convention, or its * adjoint: fftuc and ifftuc centred, fft and ifft otherwise. */ -static void grid_ft(const struct grid_s* d, const long dims[DIMS], unsigned long flags, complex float* x, bool adjoint) +static void grid_ft(const struct grid_s* d, const bart_dim_t dims[DIMS], bart_flags_t flags, complex float* x, bool adjoint) { - if (0UL == flags) + if (0 == flags) return; if (d->centred) @@ -478,9 +478,9 @@ static void grid_in(struct grid_s* d, complex float* bank, complex float* volume return; } - long wx = d->cim_dims[READ_DIM]; + bart_dim_t wx = d->cim_dims[READ_DIM]; - long hdims[DIMS]; + bart_dim_t hdims[DIMS]; md_select_dims(DIMS, FFT_FLAGS, hdims, d->cim_dims); bartorch_cuda_pad_map(d->dom_dims[READ_DIM], wx, d->vol / wx, d->wave_off, hybrid, src, map); @@ -498,9 +498,9 @@ static void grid_out(struct grid_s* d, complex float* dst, complex float* volume return; } - long wx = d->cim_dims[READ_DIM]; + bart_dim_t wx = d->cim_dims[READ_DIM]; - long hdims[DIMS]; + bart_dim_t hdims[DIMS]; md_select_dims(DIMS, FFT_FLAGS, hdims, d->cim_dims); md_clear(DIMS, hdims, hybrid, CFL_SIZE); @@ -515,29 +515,29 @@ static void grid_out(struct grid_s* d, complex float* dst, complex float* volume * `coeff_step` after the one before, and a coil's sensitivity `map_step` * after the one before in `map` (or no map). */ static void grid_fused(struct grid_s* d, complex float* dst, const complex float* src, - long coils, long coil_step, long coeff_step, const complex float* map, long map_step) + bart_dim_t coils, bart_dim_t coil_step, bart_dim_t coeff_step, const complex float* map, bart_dim_t map_step) { #ifdef USE_CUDA - long vd[1] = { d->vol }; - long bd[1] = { d->R * d->batch * MAX(1, d->kept) }; + bart_dim_t vd[1] = { d->vol }; + bart_dim_t bd[1] = { d->R * d->batch * MAX(1, d->kept) }; complex float* volume = md_alloc_gpu(1, vd, CFL_SIZE); complex float* bank = md_alloc_gpu(1, bd, CFL_SIZE); complex float* hybrid = d->wave ? md_alloc_gpu(1, vd, CFL_SIZE) : NULL; - long per = d->batch * d->kept; + bart_dim_t per = d->batch * d->kept; - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_step; - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) grid_in(d, bank + r * per, volume, hybrid, src + c * coil_step + r * coeff_step, m); if (NULL != d->dkernel) bartorch_cuda_contract_grid(d->kept, d->batch, (int)d->R, bank, d->dkernel); - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) grid_out(d, dst + c * coil_step + r * coeff_step, volume, hybrid, bank + r * per, m); } @@ -559,7 +559,7 @@ static void grid_fused(struct grid_s* d, complex float* dst, const complex float * has been transformed by its front already, so the grid never transforms it * again for a table -- where cuFFT's callbacks would (a plane of one axis * takes the readout along), the table is served on the host instead. */ -static unsigned long sampled_flags(const struct grid_s* d) +static bart_flags_t sampled_flags(const struct grid_s* d) { return d->wave ? (d->flags & ~READ_FLAG) : d->flags; } @@ -574,13 +574,13 @@ static bool sampled_fusable(const struct grid_s* d) * function -- and the adjoint back. Without a wave both are copies. */ static void wave_psf_host(const struct grid_s* d, complex float* k, bool conj) { - long pdims[DIMS]; + bart_dim_t pdims[DIMS]; md_select_dims(DIMS, FFT_FLAGS, pdims, d->cim_dims); - long kstr[DIMS]; + bart_stride_t kstr[DIMS]; md_calc_strides(DIMS, kstr, d->cim_dims, CFL_SIZE); - long pstr[DIMS]; + bart_stride_t pstr[DIMS]; md_calc_strides(DIMS, pstr, pdims, CFL_SIZE); for (int i = 0; i < DIMS; i++) @@ -629,7 +629,7 @@ static void wave_back_host(const struct grid_s* d, complex float* dst, complex f * axis alone takes a second), so the table is transformed along its samples * where the two differ: into k-space forward, out of it for the adjoint, and * the other way round for samples in image space along the readout. */ -static void sampled_readout(const struct grid_s* d, const long dims[DIMS], complex float* table, bool forward) +static void sampled_readout(const struct grid_s* d, const bart_dim_t dims[DIMS], complex float* table, bool forward) { bool transformed = (0 != MD_IS_SET(sampled_flags(d), READ_DIM)); @@ -646,32 +646,32 @@ static void sampled_readout(const struct grid_s* d, const long dims[DIMS], compl * readout, contracted with the basis for its frame. */ static void sampled_gather(const struct grid_s* d, complex float* out, const complex float* k) { - long cstr[DIMS]; + bart_stride_t cstr[DIMS]; md_calc_strides(DIMS, cstr, d->cim_dims, 1); - long ostr[DIMS]; + bart_stride_t ostr[DIMS]; md_calc_strides(DIMS, ostr, d->out_dims, 1); - long C = d->cim_dims[COIL_DIM]; + bart_dim_t C = d->cim_dims[COIL_DIM]; #pragma omp parallel for collapse(2) - for (long c = 0; c < C; c++) { - for (long e = 0; e < d->E; e++) { + for (bart_dim_t c = 0; c < C; c++) { + for (bart_dim_t e = 0; e < d->E; e++) { - long t = e / d->S; - long u = d->entry_u[e]; - long base = (e - t * d->S) * ostr[PHS2_DIM] + c * ostr[COIL_DIM] + t * ostr[TE_DIM]; + bart_dim_t t = e / d->S; + bart_dim_t u = d->entry_u[e]; + bart_dim_t base = (e - t * d->S) * ostr[PHS2_DIM] + c * ostr[COIL_DIM] + t * ostr[TE_DIM]; - for (long x = 0; x < d->X; x++) { + for (bart_dim_t x = 0; x < d->X; x++) { complex float acc = 0.; if (0 <= u) { - long at = x * cstr[READ_DIM] + d->u_coord[2 * u + 1] * cstr[PHS1_DIM] + bart_dim_t at = x * cstr[READ_DIM] + d->u_coord[2 * u + 1] * cstr[PHS1_DIM] + d->u_coord[2 * u] * cstr[PHS2_DIM] + c * cstr[COIL_DIM]; - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) acc += ((NULL == d->bt) ? 1.f : d->bt[t * d->R + r]) * k[at + r * cstr[COEFF_DIM]]; } @@ -687,31 +687,31 @@ static void sampled_scatter(const struct grid_s* d, complex float* k, const comp { md_clear(DIMS, d->cim_dims, k, CFL_SIZE); - long cstr[DIMS]; + bart_stride_t cstr[DIMS]; md_calc_strides(DIMS, cstr, d->cim_dims, 1); - long ostr[DIMS]; + bart_stride_t ostr[DIMS]; md_calc_strides(DIMS, ostr, d->out_dims, 1); - long C = d->cim_dims[COIL_DIM]; + bart_dim_t C = d->cim_dims[COIL_DIM]; #pragma omp parallel for collapse(2) - for (long c = 0; c < C; c++) { - for (long u = 0; u < d->U; u++) { + for (bart_dim_t c = 0; c < C; c++) { + for (bart_dim_t u = 0; u < d->U; u++) { - for (long x = 0; x < d->X; x++) { + for (bart_dim_t x = 0; x < d->X; x++) { - long at = x * cstr[READ_DIM] + d->u_coord[2 * u + 1] * cstr[PHS1_DIM] + bart_dim_t at = x * cstr[READ_DIM] + d->u_coord[2 * u + 1] * cstr[PHS1_DIM] + d->u_coord[2 * u] * cstr[PHS2_DIM] + c * cstr[COIL_DIM]; - for (long r = 0; r < d->R; r++) { + for (bart_dim_t r = 0; r < d->R; r++) { complex float acc = 0.; - for (long q = d->csr_start[u]; q < d->csr_start[u + 1]; q++) { + for (bart_dim_t q = d->csr_start[u]; q < d->csr_start[u + 1]; q++) { - long e = d->csr[q]; - long t = e / d->S; + bart_dim_t e = d->csr[q]; + bart_dim_t t = e / d->S; complex float v = out[x * ostr[PHS1_DIM] + (e - t * d->S) * ostr[PHS2_DIM] + c * ostr[COIL_DIM] + t * ostr[TE_DIM]]; @@ -787,18 +787,18 @@ static void grid_axes(const struct grid_s* d, struct bartorch_grid_axes* ax) * centring on and whose write gathers the kept places, then the table read * off the gathered spectrum on the card. */ static void sampled_fused_forward(struct grid_s* d, complex float* dst, const complex float* src, - long coils, long coil_step, long coeff_step, const complex float* map, long map_step) + bart_dim_t coils, bart_dim_t coil_step, bart_dim_t coeff_step, const complex float* map, bart_dim_t map_step) { - long vd[1] = { d->vol }; - long bd[1] = { d->R * d->batch * MAX(1, d->kept) }; + bart_dim_t vd[1] = { d->vol }; + bart_dim_t bd[1] = { d->R * d->batch * MAX(1, d->kept) }; - long td[DIMS]; + bart_dim_t td[DIMS]; md_select_dims(DIMS, ~COIL_FLAG, td, d->out_dims); - long tstr[DIMS]; + bart_stride_t tstr[DIMS]; md_calc_strides(DIMS, tstr, td, CFL_SIZE); - long ostr[DIMS]; + bart_stride_t ostr[DIMS]; md_calc_strides(DIMS, ostr, d->out_dims, CFL_SIZE); complex float* volume = md_alloc_gpu(1, vd, CFL_SIZE); @@ -809,13 +809,13 @@ static void sampled_fused_forward(struct grid_s* d, complex float* dst, const co struct bartorch_grid_axes ax; grid_axes(d, &ax); - long per = d->batch * d->kept; + bart_dim_t per = d->batch * d->kept; - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_step; - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) grid_in(d, bank + r * per, volume, hybrid, src + c * coil_step + r * coeff_step, m); bartorch_cuda_bank_to_table(d->E, d->X, d->S, (int)d->R, d->kept, per, d->d_entry_u, d->d_u_coord, @@ -823,7 +823,7 @@ static void sampled_fused_forward(struct grid_s* d, complex float* dst, const co sampled_readout(d, td, table, true); - md_copy2(DIMS, td, ostr, dst + c * (ostr[COIL_DIM] / (long)CFL_SIZE), tstr, table, CFL_SIZE); + md_copy2(DIMS, td, ostr, dst + c * (ostr[COIL_DIM] / (bart_stride_t)CFL_SIZE), tstr, table, CFL_SIZE); } md_free(table); @@ -841,18 +841,18 @@ static void sampled_fused_forward(struct grid_s* d, complex float* dst, const co * spectrum on the card, and per coefficient one transform back whose read * scatters it and whose write takes the conjugates off into the image. */ static void sampled_fused_adjoint(struct grid_s* d, complex float* dst, const complex float* src, - long coils, long coil_step, long coeff_step, const complex float* map, long map_step) + bart_dim_t coils, bart_dim_t coil_step, bart_dim_t coeff_step, const complex float* map, bart_dim_t map_step) { - long vd[1] = { d->vol }; - long bd[1] = { d->R * d->batch * MAX(1, d->kept) }; + bart_dim_t vd[1] = { d->vol }; + bart_dim_t bd[1] = { d->R * d->batch * MAX(1, d->kept) }; - long td[DIMS]; + bart_dim_t td[DIMS]; md_select_dims(DIMS, ~COIL_FLAG, td, d->out_dims); - long tstr[DIMS]; + bart_stride_t tstr[DIMS]; md_calc_strides(DIMS, tstr, td, CFL_SIZE); - long ostr[DIMS]; + bart_stride_t ostr[DIMS]; md_calc_strides(DIMS, ostr, d->out_dims, CFL_SIZE); complex float* volume = md_alloc_gpu(1, vd, CFL_SIZE); @@ -863,13 +863,13 @@ static void sampled_fused_adjoint(struct grid_s* d, complex float* dst, const co struct bartorch_grid_axes ax; grid_axes(d, &ax); - long per = d->batch * d->kept; + bart_dim_t per = d->batch * d->kept; - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_step; - md_copy2(DIMS, td, tstr, table, ostr, src + c * (ostr[COIL_DIM] / (long)CFL_SIZE), CFL_SIZE); + md_copy2(DIMS, td, tstr, table, ostr, src + c * (ostr[COIL_DIM] / (bart_stride_t)CFL_SIZE), CFL_SIZE); sampled_readout(d, td, table, false); @@ -878,7 +878,7 @@ static void sampled_fused_adjoint(struct grid_s* d, complex float* dst, const co bartorch_cuda_table_to_bank(d->U, d->X, d->S, (int)d->R, d->kept, per, d->d_csr_start, d->d_csr, d->d_u_coord, d->dmask, d->dprefix, &ax, d->d_bt, bank, table); - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) grid_out(d, dst + c * coil_step + r * coeff_step, volume, hybrid, bank + r * per, m); } @@ -903,7 +903,7 @@ static void grid_forward(const linop_data_t* _d, complex float* dst, const compl #ifdef USE_CUDA if (grid_ready(d, dst) && cuda_ondevice(src) && sampled_fusable(d)) { - long coils = d->cim_dims[COIL_DIM]; + bart_dim_t coils = d->cim_dims[COIL_DIM]; sampled_fused_forward(d, dst, src, coils, d->img_vol, d->img_vol * coils, NULL, 0); return; @@ -925,7 +925,7 @@ static void grid_adjoint(const linop_data_t* _d, complex float* dst, const compl #ifdef USE_CUDA if (grid_ready(d, dst) && cuda_ondevice(src) && sampled_fusable(d)) { - long coils = d->cim_dims[COIL_DIM]; + bart_dim_t coils = d->cim_dims[COIL_DIM]; md_clear(DIMS, d->dom_dims, dst, CFL_SIZE); sampled_fused_adjoint(d, dst, src, coils, d->img_vol, d->img_vol * coils, NULL, 0); @@ -945,7 +945,7 @@ static void grid_normal(const linop_data_t* _d, complex float* dst, const comple if (grid_ready(d, dst) && cuda_ondevice(src)) { - long coils = d->cim_dims[COIL_DIM]; + bart_dim_t coils = d->cim_dims[COIL_DIM]; md_clear(DIMS, d->dom_dims, dst, CFL_SIZE); grid_fused(d, dst, src, coils, d->img_vol, d->img_vol * coils, NULL, 0); @@ -1008,11 +1008,11 @@ int bartorch_grid_folds(const struct linop_s* op, const void* ref) /* The normal of a slab of coils, each with its sensitivity from `map`, * against the image `src`, added to the image `dst`. */ void bartorch_grid_normal_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map) + const int64_t map_strs[DIMS], const complex float* map) { struct grid_s* d = CAST_DOWN(grid_s, linop_get_data(op)); - grid_fused(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (long)CFL_SIZE); + grid_fused(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE); } /* Whether `op` is a sampled-only Cartesian transform whose forward and adjoint @@ -1027,12 +1027,12 @@ int bartorch_grid_folds_samples(const struct linop_s* op, const void* ref) /* The samples of a slab of coils, each with its sensitivity from `map`, of the * image `src`, into the slab's samples `dst`. */ void bartorch_grid_forward_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map) + const int64_t map_strs[DIMS], const complex float* map) { struct grid_s* d = CAST_DOWN(grid_s, linop_get_data(op)); #ifdef USE_CUDA - sampled_fused_forward(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (long)CFL_SIZE); + sampled_fused_forward(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE); #else (void)d; (void)dst; (void)src; (void)map_strs; (void)map; error("bartorch: the Cartesian callbacks run on a card\n"); @@ -1041,12 +1041,12 @@ void bartorch_grid_forward_sense(const struct linop_s* op, complex float* dst, c /* The adjoint of the slab's samples `src`, added to the image `dst`. */ void bartorch_grid_adjoint_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map) + const int64_t map_strs[DIMS], const complex float* map) { struct grid_s* d = CAST_DOWN(grid_s, linop_get_data(op)); #ifdef USE_CUDA - sampled_fused_adjoint(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (long)CFL_SIZE); + sampled_fused_adjoint(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE); #else (void)d; (void)dst; (void)src; (void)map_strs; (void)map; error("bartorch: the Cartesian callbacks run on a card\n"); @@ -1057,7 +1057,7 @@ void bartorch_grid_adjoint_sense(const struct linop_s* op, complex float* dst, c * dense forward, the normal as BART's chain, and what the callbacks need. * See grid_transform_create. */ /* BART's transform as a linop in the convention asked for, or its adjoint. */ -static struct linop_s* grid_ft_create(const long dims[DIMS], unsigned long flags, bool centred, bool adjoint) +static struct linop_s* grid_ft_create(const bart_dim_t dims[DIMS], bart_flags_t flags, bool centred, bool adjoint) { if (centred) return adjoint ? linop_ifftc_create(DIMS, dims, flags) : linop_fftc_create(DIMS, dims, flags); @@ -1065,29 +1065,29 @@ static struct linop_s* grid_ft_create(const long dims[DIMS], unsigned long flags return adjoint ? linop_ifft_create(DIMS, dims, flags) : linop_fft_create(DIMS, dims, flags); } -static struct grid_s* grid_state(const long cim_dims[DIMS], - const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz, bool centred) +static struct grid_s* grid_state(const bart_dim_t cim_dims[DIMS], + const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz, bool centred) { - unsigned long fft_flags = FFT_FLAGS & md_nontriv_dims(DIMS, cim_dims); + bart_flags_t fft_flags = FFT_FLAGS & md_nontriv_dims(DIMS, cim_dims); if (NULL != pattern) { /* The kernel is laid out over the spatial axes and the frames and * nothing else, so a pattern that differs between coils or sets * of maps is not one this can collapse. */ - if (0UL != (md_nontriv_dims(DIMS, pat_dims) & ~(FFT_FLAGS | TE_FLAG))) + if (0 != (md_nontriv_dims(DIMS, pat_dims) & ~(FFT_FLAGS | TE_FLAG))) error("bartorch: a Cartesian pattern varies along the spatial axes and the frames only\n"); for (int a = 0; a < 3; a++) if ((1 != pat_dims[a]) && (pat_dims[a] != cim_dims[a])) - error("bartorch: the pattern is %ld along axis %d, where the images are %ld\n", + error("bartorch: the pattern is %" PRId64 " along axis %d, where the images are %" PRId64 "\n", pat_dims[a], a, cim_dims[a]); - long frames = (NULL == basis) ? 1 : bas_dims[TE_DIM]; + bart_dim_t frames = (NULL == basis) ? 1 : bas_dims[TE_DIM]; if ((1 != pat_dims[TE_DIM]) && (frames != pat_dims[TE_DIM])) - error("bartorch: the pattern has %ld frames and the basis %ld\n", pat_dims[TE_DIM], frames); + error("bartorch: the pattern has %" PRId64 " frames and the basis %" PRId64 "\n", pat_dims[TE_DIM], frames); } PTR_ALLOC(struct grid_s, d); @@ -1099,14 +1099,14 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], struct linop_s* fwd = grid_ft_create(cim_dims, fft_flags, centred, false); - long R = 1; + bart_dim_t R = 1; if (NULL != basis) { R = bas_dims[COEFF_DIM]; if (R != cim_dims[COEFF_DIM]) - error("bartorch: the basis has %ld coefficients and the image %ld\n", R, cim_dims[COEFF_DIM]); + error("bartorch: the basis has %" PRId64 " coefficients and the image %" PRId64 "\n", R, cim_dims[COEFF_DIM]); d->out_dims[COEFF_DIM] = 1; d->out_dims[TE_DIM] = bas_dims[TE_DIM]; @@ -1117,7 +1117,7 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], if (NULL != pattern) fwd = linop_chain_FF(fwd, linop_sampling_create(d->out_dims, pat_dims, pattern)); - unsigned long flags = (NULL == pattern) ? 0UL : (varying(pat_dims) & fft_flags); + bart_flags_t flags = (NULL == pattern) ? 0 : (varying(pat_dims) & fft_flags); /* cuFFT links no callbacks into a transform along a single axis -- its * plan answers CUFFT_INTERNAL_ERROR -- so a pattern that varies along one @@ -1125,9 +1125,9 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], * Along that axis the transform and its inverse cancel, and so do the * centring and its conjugate, so the normal is the same one; the pattern * the kernel and the kept places are read from is repeated along it. */ - long kpat_dims[DIMS]; + bart_dim_t kpat_dims[DIMS]; complex float* repeated = NULL; - long promoted = 1; + bart_dim_t promoted = 1; if (NULL != pattern) md_copy_dims(DIMS, kpat_dims, pat_dims); @@ -1144,11 +1144,11 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], kpat_dims[extra] = cim_dims[extra]; - long istrs[DIMS]; + bart_stride_t istrs[DIMS]; md_calc_strides(DIMS, istrs, pat_dims, CFL_SIZE); istrs[extra] = 0; - long ostrs[DIMS]; + bart_stride_t ostrs[DIMS]; md_calc_strides(DIMS, ostrs, kpat_dims, CFL_SIZE); complex float* host = md_alloc(DIMS, pat_dims, CFL_SIZE); @@ -1166,7 +1166,7 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], const complex float* kpattern = (NULL != repeated) ? repeated : pattern; - long kdims[DIMS]; + bart_dim_t kdims[DIMS]; complex float* K = grid_kernel(kdims, kpat_dims, kpattern, bas_dims, basis); /* The uncentred transform is unnormalized, so along the axis taken in @@ -1186,7 +1186,7 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], /* The coefficients go out on TE, where the multiply-accumulate * can put them, and a reshape reads them back on COEFF; the two * layouts are the same memory. */ - long mixed_dims[DIMS]; + bart_dim_t mixed_dims[DIMS]; md_copy_dims(DIMS, mixed_dims, cim_dims); mixed_dims[COEFF_DIM] = 1; mixed_dims[TE_DIM] = R; @@ -1218,11 +1218,11 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], } if (0 == toeplitz) - d->flags = 0UL; + d->flags = 0; struct linop_s* nrm = core; - if (0UL != d->flags) { + if (0 != d->flags) { nrm = linop_chain_FF(linop_chain_FF(grid_ft_create(cim_dims, d->flags, centred, false), core), grid_ft_create(cim_dims, d->flags, centred, true)); @@ -1263,7 +1263,7 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], d->d_csr = NULL; d->d_bt = NULL; - debug_printf(DP_DEBUG1, "Cartesian slab: %ld coefficients, normal transforms axes %lx of %lx, %ld of %ld places kept\n", + debug_printf(DP_DEBUG1, "Cartesian slab: %" PRId64 " coefficients, normal transforms axes %" PRIx64 " of %" PRIx64 ", %" PRId64 " of %" PRId64 " places kept\n", R, d->flags, fft_flags, d->kept, d->plane); return PTR_PASS(d); @@ -1272,13 +1272,13 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], /* The slab transform over coil images of `cim_dims`, or over every coil when * the loop does not run. `pattern` and `basis` may each be NULL. Without * `toeplitz` the normal is the two applications, as BART derives it. */ -const struct linop_s* grid_transform_create(const long cim_dims[DIMS], - const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz) +const struct linop_s* grid_transform_create(const bart_dim_t cim_dims[DIMS], + const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz) { struct grid_s* d = grid_state(cim_dims, pat_dims, pattern, bas_dims, basis, toeplitz, true); - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; md_copy_dims(DIMS, out_dims, d->out_dims); return linop_create(DIMS, out_dims, DIMS, cim_dims, CAST_UP(d), @@ -1297,38 +1297,38 @@ const struct linop_s* grid_transform_create(const long cim_dims[DIMS], * does in the two applications. Forward, the table is read off the gathered * spectrum and transformed along its readout where that is asked for; that * is a transform of the table, not of the volume. */ -static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, int components, - const long* positions, const long bas_dims[DIMS], const complex float* basis, bool centred) +static struct grid_s* sampled_state(const bart_dim_t cim_dims[DIMS], bart_dim_t T, bart_dim_t S, int components, + const bart_dim_t* positions, const bart_dim_t bas_dims[DIMS], const complex float* basis, bool centred) { - long ny = cim_dims[PHS1_DIM]; - long nz = cim_dims[PHS2_DIM]; + bart_dim_t ny = cim_dims[PHS1_DIM]; + bart_dim_t nz = cim_dims[PHS2_DIM]; if (((1 < nz) ? 2 : 1) != components) error("bartorch: positions of %d indices for an image of %d phase-encode axes\n", components, (1 < nz) ? 2 : 1); if ((NULL == basis) ? (1 != T) : (bas_dims[TE_DIM] != T)) - error("bartorch: positions over %ld frames, and a basis of %ld\n", T, (NULL == basis) ? 1L : bas_dims[TE_DIM]); + error("bartorch: positions over %" PRId64 " frames, and a basis of %" PRId64 "\n", T, (NULL == basis) ? 1 : bas_dims[TE_DIM]); - long plane = ny * nz; - long E = T * S; + bart_dim_t plane = ny * nz; + bart_dim_t E = T * S; int* place_u = xmalloc((size_t)plane * sizeof(int)); float* counts = xmalloc((size_t)(T * plane) * sizeof(float)); int* entry_u = xmalloc((size_t)MAX(1, E) * sizeof(int)); - for (long p = 0; p < plane; p++) + for (bart_dim_t p = 0; p < plane; p++) place_u[p] = -1; - for (long i = 0; i < T * plane; i++) + for (bart_dim_t i = 0; i < T * plane; i++) counts[i] = 0.f; - long U = 0; - long valid = 0; + bart_dim_t U = 0; + bart_dim_t valid = 0; - for (long e = 0; e < E; e++) { + for (bart_dim_t e = 0; e < E; e++) { - long zc = (2 == components) ? positions[e * components] : 0; - long yc = positions[e * components + components - 1]; + bart_dim_t zc = (2 == components) ? positions[e * components] : 0; + bart_dim_t yc = positions[e * components + components - 1]; if ((-1 == zc) || (-1 == yc)) { @@ -1337,9 +1337,9 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i } if ((zc < 0) || (zc >= nz) || (yc < 0) || (yc >= ny)) - error("bartorch: a phase encode lies outside the %ld x %ld plane\n", nz, ny); + error("bartorch: a phase encode lies outside the %" PRId64 " x %" PRId64 " plane\n", nz, ny); - long p = yc + ny * zc; + bart_dim_t p = yc + ny * zc; if (-1 == place_u[p]) place_u[p] = (int)U++; @@ -1354,7 +1354,7 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i int* csr = xmalloc((size_t)MAX(1, valid) * sizeof(int)); int* fill = xmalloc((size_t)MAX(1, U) * sizeof(int)); - for (long p = 0; p < plane; p++) { + for (bart_dim_t p = 0; p < plane; p++) { if (0 > place_u[p]) continue; @@ -1363,27 +1363,27 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i u_coord[2 * place_u[p] + 1] = (int)(p % ny); } - for (long u = 0; u <= U; u++) + for (bart_dim_t u = 0; u <= U; u++) csr_start[u] = 0; - for (long e = 0; e < E; e++) + for (bart_dim_t e = 0; e < E; e++) if (0 <= entry_u[e]) csr_start[entry_u[e] + 1]++; - for (long u = 0; u < U; u++) { + for (bart_dim_t u = 0; u < U; u++) { csr_start[u + 1] += csr_start[u]; fill[u] = csr_start[u]; } - for (long e = 0; e < E; e++) + for (bart_dim_t e = 0; e < E; e++) if (0 <= entry_u[e]) csr[fill[entry_u[e]]++] = (int)e; xfree(fill); xfree(place_u); - long pat_dims[DIMS]; + bart_dim_t pat_dims[DIMS]; md_singleton_dims(DIMS, pat_dims); pat_dims[PHS1_DIM] = ny; pat_dims[PHS2_DIM] = nz; @@ -1391,7 +1391,7 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i complex float* pattern = md_alloc(DIMS, pat_dims, CFL_SIZE); - for (long i = 0; i < T * plane; i++) + for (bart_dim_t i = 0; i < T * plane; i++) pattern[i] = sqrtf(counts[i]); xfree(counts); @@ -1424,8 +1424,8 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i d->bt = xmalloc((size_t)(T * d->R) * sizeof(complex float)); - for (long t = 0; t < T; t++) - for (long r = 0; r < d->R; r++) + for (bart_dim_t t = 0; t < T; t++) + for (bart_dim_t r = 0; r < d->R; r++) d->bt[t * d->R + r] = B[t + T * r]; md_free(B); @@ -1436,14 +1436,14 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i d->out_dims[PHS2_DIM] = S; d->out_dims[TE_DIM] = T; - debug_printf(DP_DEBUG1, "Cartesian samples: %ld frames x %ld shots x %ld readout at %ld places\n", + debug_printf(DP_DEBUG1, "Cartesian samples: %" PRId64 " frames x %" PRId64 " shots x %" PRId64 " readout at %" PRId64 " places\n", T, S, d->X, U); return d; } -const struct linop_s* grid_sampled_create(const long cim_dims[DIMS], long T, long S, int components, - const long* positions, const long bas_dims[DIMS], const complex float* basis, +const struct linop_s* grid_sampled_create(const bart_dim_t cim_dims[DIMS], bart_dim_t T, bart_dim_t S, int components, + const bart_dim_t* positions, const bart_dim_t bas_dims[DIMS], const complex float* basis, int kspace_readout, int toeplitz) { struct grid_s* d = sampled_state(cim_dims, T, S, components, positions, bas_dims, basis, true); @@ -1451,7 +1451,7 @@ const struct linop_s* grid_sampled_create(const long cim_dims[DIMS], long T, lon d->toeplitz = (0 != toeplitz); d->kspace_readout = (0 != kspace_readout); - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; md_copy_dims(DIMS, out_dims, d->out_dims); return linop_create(DIMS, out_dims, DIMS, cim_dims, CAST_UP(d), @@ -1460,17 +1460,17 @@ const struct linop_s* grid_sampled_create(const long cim_dims[DIMS], long T, lon /* A wave over the grid `d` holds: the domain, the offset of the zero-fill, * and the point spread function over the grid's spatial axes. */ -static void wave_setup(struct grid_s* d, const long dom_dims[DIMS], const complex float* psf) +static void wave_setup(struct grid_s* d, const bart_dim_t dom_dims[DIMS], const complex float* psf) { - long wx = d->cim_dims[READ_DIM]; - long sx = dom_dims[READ_DIM]; + bart_dim_t wx = d->cim_dims[READ_DIM]; + bart_dim_t sx = dom_dims[READ_DIM]; d->wave = true; md_copy_dims(DIMS, d->dom_dims, dom_dims); d->img_vol = md_calc_size(3, dom_dims); d->wave_off = (wx / 2 > sx / 2) ? (wx / 2 - sx / 2) : (sx / 2 - wx / 2); - long pdims[DIMS]; + bart_dim_t pdims[DIMS]; md_select_dims(DIMS, FFT_FLAGS, pdims, d->cim_dims); d->psf = md_alloc(DIMS, pdims, CFL_SIZE); @@ -1505,11 +1505,11 @@ static void wave_normal_chain(struct grid_s* d) * closed-form normal puts its kernel between the phase-encode transforms, so * it takes a pattern the same all along the readout -- the readout is not * the transform's to cancel -- and any other normal is the two applications. */ -const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, const complex float* psf, int centred, - const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz) +const struct linop_s* wave_transform_create(const bart_dim_t dom_dims[DIMS], bart_dim_t wx, const complex float* psf, int centred, + const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz) { - long cim_dims[DIMS]; + bart_dim_t cim_dims[DIMS]; md_copy_dims(DIMS, cim_dims, dom_dims); cim_dims[READ_DIM] = wx; @@ -1519,7 +1519,7 @@ const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, wave_setup(d, dom_dims, psf); - unsigned long pe = FFT_FLAGS & ~READ_FLAG & md_nontriv_dims(DIMS, cim_dims); + bart_flags_t pe = FFT_FLAGS & ~READ_FLAG & md_nontriv_dims(DIMS, cim_dims); struct linop_s* fwd = linop_chain_FF(wave_front(d), grid_ft_create(cim_dims, pe, d->centred, false)); @@ -1534,7 +1534,7 @@ const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, wave_normal_chain(d); - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; md_copy_dims(DIMS, out_dims, d->out_dims); return linop_create(DIMS, out_dims, DIMS, d->dom_dims, CAST_UP(d), @@ -1543,11 +1543,11 @@ const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, /* The same over sampled-only k-space: a table of phase encodes per frame with * the oversampled readout along each, as the front leaves it. */ -const struct linop_s* wave_sampled_create(const long dom_dims[DIMS], long wx, const complex float* psf, int centred, - long T, long S, int components, const long* positions, - const long bas_dims[DIMS], const complex float* basis, int toeplitz) +const struct linop_s* wave_sampled_create(const bart_dim_t dom_dims[DIMS], bart_dim_t wx, const complex float* psf, int centred, + bart_dim_t T, bart_dim_t S, int components, const bart_dim_t* positions, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz) { - long cim_dims[DIMS]; + bart_dim_t cim_dims[DIMS]; md_copy_dims(DIMS, cim_dims, dom_dims); cim_dims[READ_DIM] = wx; @@ -1560,7 +1560,7 @@ const struct linop_s* wave_sampled_create(const long dom_dims[DIMS], long wx, co wave_normal_chain(d); - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; md_copy_dims(DIMS, out_dims, d->out_dims); return linop_create(DIMS, out_dims, DIMS, d->dom_dims, CAST_UP(d), diff --git a/src/csrc/ops/grid.cuh b/src/csrc/ops/grid.cuh index 4b9f3e38..4eea19ae 100644 --- a/src/csrc/ops/grid.cuh +++ b/src/csrc/ops/grid.cuh @@ -67,14 +67,14 @@ __device__ static inline unsigned int grid_index(const struct grid_info* g, unsi /* Where cuFFT offset `off` sits in the gathered spectrum, or -1 where the * pattern does not keep its place. */ -__device__ static inline long grid_kept(const struct grid_info* g, unsigned int off) +__device__ static inline bart_dim_t grid_kept(const struct grid_info* g, unsigned int off) { unsigned int batch = off / g->plane; unsigned int place = off - batch * g->plane; - long j = kept_at(g->mask, g->prefix, (long)place); + bart_dim_t j = kept_at(g->mask, g->prefix, (bart_dim_t)place); - return (0 > j) ? -1 : (long)batch * g->L + j; + return (0 > j) ? -1 : (bart_dim_t)batch * g->L + j; } #endif diff --git a/src/csrc/ops/iter.c b/src/csrc/ops/iter.c index 799e8ac6..bd437dd6 100644 --- a/src/csrc/ops/iter.c +++ b/src/csrc/ops/iter.c @@ -100,9 +100,9 @@ struct bartorch_prox_s { int xform; }; -int bartorch_prox_create(const char* kind, long xflags, long jflags, float lambda, int k, +int bartorch_prox_create(const char* kind, uint64_t xflags, uint64_t jflags, float lambda, int k, int llr_blk, const char* wavelet, int shift_mode, - const long* img_dims, bartorch_prox** out) + const int64_t* img_dims, bartorch_prox** out) { int xform; @@ -113,8 +113,8 @@ int bartorch_prox_create(const char* kind, long xflags, long jflags, float lambd (void)opt_reg_init(&ropts); ropts.regs[0].xform = xform; - ropts.regs[0].xflags = (unsigned long)xflags; - ropts.regs[0].jflags = (unsigned long)jflags; + ropts.regs[0].xflags = xflags; + ropts.regs[0].jflags = jflags; ropts.regs[0].lambda = lambda; ropts.regs[0].k = k; ropts.regs[0].graph_file = NULL; @@ -123,9 +123,9 @@ int bartorch_prox_create(const char* kind, long xflags, long jflags, float lambd const struct operator_p_s* prox_ops[NUM_REGS] = { NULL }; const struct linop_s* trafos[NUM_REGS] = { NULL }; - const long (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; + const bart_dim_t (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; - long dims[DIMS]; + bart_dim_t dims[DIMS]; md_copy_dims(DIMS, dims, img_dims); opt_reg_configure(DIMS, dims, &ropts, prox_ops, trafos, sdims, @@ -150,10 +150,10 @@ int bartorch_prox_create(const char* kind, long xflags, long jflags, float lambd } int bartorch_prox_set_create(int n, const char* const* kinds, - const long* xflags, const long* jflags, const float* lambda, const int* k, + const uint64_t* xflags, const uint64_t* jflags, const float* lambda, const int* k, int llr_blk, const char* wavelet, int shift_mode, - const float* alpha, const float* gamma, const long* img_dims, - int max_out, bartorch_prox** out, int* count, long* svars) + const float* alpha, const float* gamma, const int64_t* img_dims, + int max_out, bartorch_prox** out, int* count, int64_t* svars) { if ((0 > n) || (NUM_REGS < n) || (NULL == out) || (NULL == count) || (NULL == svars)) return -1; @@ -169,8 +169,8 @@ int bartorch_prox_set_create(int n, const char* const* kinds, return -4; ropts.regs[i].xform = xform; - ropts.regs[i].xflags = (unsigned long)xflags[i]; - ropts.regs[i].jflags = (unsigned long)jflags[i]; + ropts.regs[i].xflags = xflags[i]; + ropts.regs[i].jflags = jflags[i]; ropts.regs[i].lambda = lambda[i]; ropts.regs[i].k = k[i]; ropts.regs[i].graph_file = NULL; @@ -189,9 +189,9 @@ int bartorch_prox_set_create(int n, const char* const* kinds, const struct operator_p_s* prox_ops[NUM_REGS] = { NULL }; const struct linop_s* trafos[NUM_REGS] = { NULL }; - const long (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; + const bart_dim_t (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; - long dims[DIMS]; + bart_dim_t dims[DIMS]; md_copy_dims(DIMS, dims, img_dims); opt_reg_configure(DIMS, dims, &ropts, prox_ops, trafos, sdims, @@ -232,7 +232,7 @@ int bartorch_prox_set_create(int n, const char* const* kinds, * Returns the rank, or a negative code. `dims` holds `N` entries and is * filled with ones past the rank. */ -int bartorch_prox_domain(const bartorch_prox* h, int N, long* dims) +int bartorch_prox_domain(const bartorch_prox* h, int N, int64_t* dims) { if ((NULL == h) || (NULL == h->op) || (NULL == dims) || (1 > N)) return -1; @@ -383,7 +383,7 @@ void bartorch_prox_free(bartorch_prox* h) struct counting_monitor { struct iter_monitor_s super; - long count; + bart_dim_t count; }; static void counting_monitor_fun(struct iter_monitor_s* monitor, const struct vec_iter_s* /*ops*/, const float* /*x*/) @@ -403,7 +403,7 @@ struct maxeigen_data { operator_data_t super; float lambda; - long size; + bart_dim_t size; const struct linop_s* model_op; }; @@ -510,7 +510,7 @@ int bartorch_maxeigen(const bartorch_linop* handle, const bartorch_linop* precon int bartorch_solve(const bartorch_linop* handle, const char* algorithm, - const char* const* reg_kinds, const long* reg_xflags, const long* reg_jflags, + const char* const* reg_kinds, const uint64_t* reg_xflags, const uint64_t* reg_jflags, const float* reg_lambda, const int* reg_k, const bartorch_prox* const* reg_ops, int n_reg, float cclambda, int maxiter, float step, int eigen, int hogwild, @@ -522,7 +522,7 @@ int bartorch_solve(const bartorch_linop* handle, const bartorch_linop* precond, int llr_blk, const char* wavelet, int shift_mode, const float* alpha, const float* gamma, - void* x, const void* y, long* iterations) + void* x, const void* y, int64_t* iterations) { const struct linop_s* model_op = bartorch_linop_unwrap(handle); @@ -582,8 +582,8 @@ int bartorch_solve(const bartorch_linop* handle, return -4; ropts.regs[i].xform = xform; - ropts.regs[i].xflags = (unsigned long)reg_xflags[i]; - ropts.regs[i].jflags = (unsigned long)reg_jflags[i]; + ropts.regs[i].xflags = reg_xflags[i]; + ropts.regs[i].jflags = reg_jflags[i]; ropts.regs[i].lambda = reg_lambda[i]; ropts.regs[i].k = reg_k[i]; ropts.regs[i].graph_file = NULL; @@ -595,10 +595,10 @@ int bartorch_solve(const bartorch_linop* handle, const struct operator_p_s* thresh_ops[NUM_REGS] = { NULL }; const struct linop_s* trafos[NUM_REGS] = { NULL }; - long img_dims[DIMS]; + bart_dim_t img_dims[DIMS]; md_copy_dims(DIMS, img_dims, linop_domain(model_op)->dims); - long ksp_dims[DIMS]; + bart_dim_t ksp_dims[DIMS]; md_copy_dims(DIMS, ksp_dims, linop_codomain(model_op)->dims); /* Three terms are not one proximal operator on the image: total @@ -619,7 +619,7 @@ int bartorch_solve(const bartorch_linop* handle, if (extending) { - const long (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; + const bart_dim_t (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; /* `--alpha` and `--gamma` are one pair each for the whole set, and * only these three terms read them; `opt_reg_init` has already put @@ -720,8 +720,8 @@ int bartorch_solve(const bartorch_linop* handle, * of that vector out and the encoding is chained onto it, which is what * `pics.c` does. The solve then walks the long vector and the image is * the front of the answer. */ - long total = md_calc_size(DIMS, img_dims); - long ext = total + ropts.svars; + bart_dim_t total = md_calc_size(DIMS, img_dims); + bart_dim_t ext = total + ropts.svars; complex float* xbig = NULL; @@ -785,7 +785,7 @@ int bartorch_solve(const bartorch_linop* handle, return 0; } -float bartorch_scaling_norm(long size, const void* image, float rescale, int compat, float p) +float bartorch_scaling_norm(int64_t size, const void* image, float rescale, int compat, float p) { /* On the host, and on a copy: the estimate is read off order statistics * and BART sorts the array it is handed to get them. */ diff --git a/src/csrc/ops/kernels.cu b/src/csrc/ops/kernels.cu index b587c8e4..1f004e21 100644 --- a/src/csrc/ops/kernels.cu +++ b/src/csrc/ops/kernels.cu @@ -38,15 +38,15 @@ __global__ static void kern_phase_map_in(struct phase_conf c, cuFloatComplex* ds int startZ = threadIdx.z + blockDim.z * blockIdx.z; int strideZ = blockDim.z * gridDim.z; - for (long z = startZ; z < c.dims[2]; z += strideZ) - for (long y = startY; y < c.dims[1]; y += strideY) - for (long x = startX; x < c.dims[0]; x += strideX) { + for (bart_dim_t z = startZ; z < c.dims[2]; z += strideZ) + for (bart_dim_t y = startY; y < c.dims[1]; y += strideY) + for (bart_dim_t x = startX; x < c.dims[0]; x += strideX) { - long idx = x + c.dims[0] * (y + c.dims[1] * z); + bart_dim_t idx = x + c.dims[0] * (y + c.dims[1] * z); cuFloatComplex w = cuCmulf(map[idx], phase_at(c, x, y, z, false)); - for (long i = 0; i < c.batch; i++) + for (bart_dim_t i = 0; i < c.batch; i++) dst[idx + i * c.tot] = cuCmulf(src[idx + i * c.tot], w); } } @@ -62,20 +62,20 @@ __global__ static void kern_phase_map_out(struct phase_conf c, cuFloatComplex* d int startZ = threadIdx.z + blockDim.z * blockIdx.z; int strideZ = blockDim.z * gridDim.z; - for (long z = startZ; z < c.dims[2]; z += strideZ) - for (long y = startY; y < c.dims[1]; y += strideY) - for (long x = startX; x < c.dims[0]; x += strideX) { + for (bart_dim_t z = startZ; z < c.dims[2]; z += strideZ) + for (bart_dim_t y = startY; y < c.dims[1]; y += strideY) + for (bart_dim_t x = startX; x < c.dims[0]; x += strideX) { - long idx = x + c.dims[0] * (y + c.dims[1] * z); + bart_dim_t idx = x + c.dims[0] * (y + c.dims[1] * z); cuFloatComplex w = cuCmulf(cuConjf(map[idx]), phase_at(c, x, y, z, true)); - for (long i = 0; i < c.batch; i++) + for (bart_dim_t i = 0; i < c.batch; i++) dst[idx + i * c.tot] = cuCaddf(dst[idx + i * c.tot], cuCmulf(src[idx + i * c.tot], w)); } } -extern "C" void bartorch_cuda_phase_map_in(int N, const long dims[], const float shift[3], float scale, +extern "C" void bartorch_cuda_phase_map_in(int N, const bart_dim_t dims[], const float shift[3], float scale, _Complex float* dst, const _Complex float* src, const _Complex float* map) { struct phase_conf c = phase_setup(N, dims, shift, scale); @@ -88,7 +88,7 @@ extern "C" void bartorch_cuda_phase_map_in(int N, const long dims[], const float CUDA_KERNEL_ERROR; } -extern "C" void bartorch_cuda_phase_map_out(int N, const long dims[], const float shift[3], float scale, +extern "C" void bartorch_cuda_phase_map_out(int N, const bart_dim_t dims[], const float shift[3], float scale, _Complex float* dst, const _Complex float* src, const _Complex float* map) { struct phase_conf c = phase_setup(N, dims, shift, scale); @@ -104,43 +104,43 @@ extern "C" void bartorch_cuda_phase_map_out(int N, const long dims[], const floa /* Gather and scatter over the places the samples reach, found through the * mask and counts of coset.cuh. A scatter writes zeros everywhere else, which * is what the transform that follows it needs to see there. */ -__global__ static void kern_gather(long V, const unsigned int* mask, const int* prefix, +__global__ static void kern_gather(bart_dim_t V, const unsigned int* mask, const int* prefix, cuFloatComplex* dst, const cuFloatComplex* src) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; - for (long i = start; i < V; i += stride) { + for (bart_dim_t i = start; i < V; i += stride) { - long j = kept_at(mask, prefix, i); + bart_dim_t j = kept_at(mask, prefix, i); if (0 <= j) dst[j] = src[i]; } } -__global__ static void kern_scatter(long V, const unsigned int* mask, const int* prefix, +__global__ static void kern_scatter(bart_dim_t V, const unsigned int* mask, const int* prefix, cuFloatComplex* dst, const cuFloatComplex* src) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; - for (long i = start; i < V; i += stride) { + for (bart_dim_t i = start; i < V; i += stride) { - long j = kept_at(mask, prefix, i); + bart_dim_t j = kept_at(mask, prefix, i); dst[i] = (0 <= j) ? src[j] : make_cuFloatComplex(0.f, 0.f); } } -static dim3 grid_for(long n) +static dim3 grid_for(bart_dim_t n) { - long blocks = (n + 255) / 256; + bart_dim_t blocks = (n + 255) / 256; return dim3((unsigned int)((blocks < 65535) ? blocks : 65535)); } -extern "C" void bartorch_cuda_gather(long V, const unsigned int* mask, const int* prefix, +extern "C" void bartorch_cuda_gather(bart_dim_t V, const unsigned int* mask, const int* prefix, _Complex float* dst, const _Complex float* src) { kern_gather<<>>(V, mask, prefix, (cuFloatComplex*)dst, (const cuFloatComplex*)src); @@ -148,7 +148,7 @@ extern "C" void bartorch_cuda_gather(long V, const unsigned int* mask, const int CUDA_KERNEL_ERROR; } -extern "C" void bartorch_cuda_scatter(long V, const unsigned int* mask, const int* prefix, +extern "C" void bartorch_cuda_scatter(bart_dim_t V, const unsigned int* mask, const int* prefix, _Complex float* dst, const _Complex float* src) { kern_scatter<<>>(V, mask, prefix, (cuFloatComplex*)dst, (const cuFloatComplex*)src); @@ -188,7 +188,7 @@ __device__ static inline cuFloatComplex ifftmod_at(int n, int j) * what the centred transforms of a padded kernel put on around their * transforms. Every axis after the third is a batch the same factor * multiplies. */ -__global__ static void kern_modulate(unsigned int d0, unsigned int d1, unsigned int d2, long rest, +__global__ static void kern_modulate(unsigned int d0, unsigned int d1, unsigned int d2, bart_dim_t rest, int g0, int g1, int g2, int o0, int o1, int o2, float scale, cuFloatComplex* x) { unsigned int vol = d0 * d1 * d2; @@ -204,15 +204,15 @@ __global__ static void kern_modulate(unsigned int d0, unsigned int d1, unsigned f = make_cuFloatComplex(scale * f.x, scale * f.y); - for (long r = 0; r < rest; r++) + for (bart_dim_t r = 0; r < rest; r++) x[i + r * vol] = cuCmulf(x[i + r * vol], f); } } -extern "C" void bartorch_cuda_modulate(const long dims[3], long rest, const long grid[3], const long off[3], +extern "C" void bartorch_cuda_modulate(const bart_dim_t dims[3], bart_dim_t rest, const bart_dim_t grid[3], const bart_stride_t off[3], float scale, _Complex float* x) { - long vol = dims[0] * dims[1] * dims[2]; + bart_dim_t vol = dims[0] * dims[1] * dims[2]; kern_modulate<<>>((unsigned int)dims[0], (unsigned int)dims[1], (unsigned int)dims[2], rest, (int)grid[0], (int)grid[1], (int)grid[2], @@ -232,12 +232,12 @@ extern "C" void bartorch_cuda_modulate(const long dims[3], long rest, const long enum { CONTRACT_MAX = 16 }; template -__global__ static void kern_contract_upper_real(long L, int R, cuFloatComplex* bank, const P* mat) +__global__ static void kern_contract_upper_real(bart_dim_t L, int R, cuFloatComplex* bank, const P* mat) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; - for (long l = start; l < L; l += stride) { + for (bart_dim_t l = start; l < L; l += stride) { cuFloatComplex in[CONTRACT_MAX]; @@ -253,7 +253,7 @@ __global__ static void kern_contract_upper_real(long L, int R, cuFloatComplex* b int lo = (r < c) ? r : c; int hi = (r < c) ? c : r; - float m = widen(mat[(long)(lo + hi * (hi + 1) / 2) * L + l]); + float m = widen(mat[(bart_dim_t)(lo + hi * (hi + 1) / 2) * L + l]); re += m * in[c].x; im += m * in[c].y; @@ -264,7 +264,7 @@ __global__ static void kern_contract_upper_real(long L, int R, cuFloatComplex* b } } -extern "C" int bartorch_cuda_contract_upper_real(long L, int R, _Complex float* bank, const float* mat) +extern "C" int bartorch_cuda_contract_upper_real(bart_dim_t L, int R, _Complex float* bank, const float* mat) { if ((R < 1) || (R > CONTRACT_MAX)) return -1; @@ -277,7 +277,7 @@ extern "C" int bartorch_cuda_contract_upper_real(long L, int R, _Complex float* } /* The same, against a function kept in bfloat16. */ -extern "C" int bartorch_cuda_contract_upper_real_bf16(long L, int R, _Complex float* bank, const void* mat) +extern "C" int bartorch_cuda_contract_upper_real_bf16(bart_dim_t L, int R, _Complex float* bank, const void* mat) { if ((R < 1) || (R > CONTRACT_MAX)) return -1; @@ -296,15 +296,15 @@ extern "C" int bartorch_cuda_contract_upper_real_bf16(long L, int R, _Complex fl * kernel and written back over itself. Entry (r, c) of the kernel at place l * is `K[(l R + r) R + c]`: the kernel varies over the transformed plane only, * so every batch reads the same one. */ -__global__ static void kern_contract_grid(long L, long B, int R, cuFloatComplex* bank, const cuFloatComplex* K) +__global__ static void kern_contract_grid(bart_dim_t L, bart_dim_t B, int R, cuFloatComplex* bank, const cuFloatComplex* K) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = L * B; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = L * B; - for (long i = start; i < n; i += stride) { + for (bart_dim_t i = start; i < n; i += stride) { - long l = i % L; + bart_dim_t l = i % L; cuFloatComplex in[CONTRACT_MAX]; @@ -323,7 +323,7 @@ __global__ static void kern_contract_grid(long L, long B, int R, cuFloatComplex* } } -extern "C" int bartorch_cuda_contract_grid(long L, long B, int R, _Complex float* bank, const _Complex float* K) +extern "C" int bartorch_cuda_contract_grid(bart_dim_t L, bart_dim_t B, int R, _Complex float* bank, const _Complex float* K) { if ((R < 1) || (R > CONTRACT_MAX)) return -1; @@ -350,60 +350,60 @@ struct bartorch_grid_axes { /* Where coordinates `c` (x, y, z) sit in the spectrum, or -1, and the * centring there. */ -__device__ static inline long grid_spectrum_at(const struct bartorch_grid_axes* ax, long L, - const unsigned int* mask, const int* prefix, const long c[3], cuFloatComplex* mod) +__device__ static inline bart_dim_t grid_spectrum_at(const struct bartorch_grid_axes* ax, bart_dim_t L, + const unsigned int* mask, const int* prefix, const bart_dim_t c[3], cuFloatComplex* mod) { - long place = 0; - long batch = 0; + bart_dim_t place = 0; + bart_dim_t batch = 0; cuFloatComplex m = make_cuFloatComplex(1.f, 0.f); for (int a = 0; a < 3; a++) { if (0 != ax->pstr[a]) { - place += c[a] * (long)ax->pstr[a]; + place += c[a] * (bart_dim_t)ax->pstr[a]; m = cuCmulf(m, ax->mod[a][c[a]]); } else if (0 != ax->bstr[a]) { - batch += c[a] * (long)ax->bstr[a]; + batch += c[a] * (bart_dim_t)ax->bstr[a]; } } *mod = m; - long j = kept_at(mask, prefix, place); + bart_dim_t j = kept_at(mask, prefix, place); return (0 > j) ? -1 : batch * L + j; } /* table[e X + x] = centring * sum_r B[t R + r] bank[r per + at(x, place of e)], * with t = e / S, zero for a padding entry. B NULL is one coefficient. */ -__global__ static void kern_bank_to_table(long E, long X, long S, int R, long L, long per, +__global__ static void kern_bank_to_table(bart_dim_t E, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* entry_u, const int* u_coord, const unsigned int* mask, const int* prefix, struct bartorch_grid_axes ax, const cuFloatComplex* B, const cuFloatComplex* bank, cuFloatComplex* table) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = E * X; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = E * X; - for (long i = start; i < n; i += stride) { + for (bart_dim_t i = start; i < n; i += stride) { - long e = i / X; - long u = entry_u[e]; + bart_dim_t e = i / X; + bart_dim_t u = entry_u[e]; cuFloatComplex acc = make_cuFloatComplex(0.f, 0.f); if (0 <= u) { - long c[3] = { i - e * X, u_coord[2 * u + 1], u_coord[2 * u] }; + bart_dim_t c[3] = { i - e * X, u_coord[2 * u + 1], u_coord[2 * u] }; cuFloatComplex m; - long at = grid_spectrum_at(&ax, L, mask, prefix, c, &m); + bart_dim_t at = grid_spectrum_at(&ax, L, mask, prefix, c, &m); if (0 <= at) { - long t = e / S; + bart_dim_t t = e / S; for (int r = 0; r < R; r++) { @@ -423,21 +423,21 @@ __global__ static void kern_bank_to_table(long E, long X, long S, int R, long L, /* The adjoint: bank[r per + at(x, u)] = conj(centring) * sum over the entries * e of place u of conj(B[t R + r]) table[e X + x]. Each (u, x) is its own * place of the spectrum, so no two threads write the same one. */ -__global__ static void kern_table_to_bank(long U, long X, long S, int R, long L, long per, +__global__ static void kern_table_to_bank(bart_dim_t U, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* csr_start, const int* csr, const int* u_coord, const unsigned int* mask, const int* prefix, struct bartorch_grid_axes ax, const cuFloatComplex* B, cuFloatComplex* bank, const cuFloatComplex* table) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = U * X; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = U * X; - for (long i = start; i < n; i += stride) { + for (bart_dim_t i = start; i < n; i += stride) { - long u = i / X; - long c[3] = { i - u * X, u_coord[2 * u + 1], u_coord[2 * u] }; + bart_dim_t u = i / X; + bart_dim_t c[3] = { i - u * X, u_coord[2 * u + 1], u_coord[2 * u] }; cuFloatComplex m; - long at = grid_spectrum_at(&ax, L, mask, prefix, c, &m); + bart_dim_t at = grid_spectrum_at(&ax, L, mask, prefix, c, &m); if (0 > at) continue; @@ -448,9 +448,9 @@ __global__ static void kern_table_to_bank(long U, long X, long S, int R, long L, cuFloatComplex acc = make_cuFloatComplex(0.f, 0.f); - for (long k = csr_start[u]; k < csr_start[u + 1]; k++) { + for (bart_dim_t k = csr_start[u]; k < csr_start[u + 1]; k++) { - long e = csr[k]; + bart_dim_t e = csr[k]; cuFloatComplex v = table[e * X + c[0]]; acc = cuCaddf(acc, (NULL == B) ? v : cuCmulf(cuConjf(B[(e / S) * R + r]), v)); @@ -461,7 +461,7 @@ __global__ static void kern_table_to_bank(long U, long X, long S, int R, long L, } } -extern "C" void bartorch_cuda_bank_to_table(long E, long X, long S, int R, long L, long per, +extern "C" void bartorch_cuda_bank_to_table(bart_dim_t E, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* entry_u, const int* u_coord, const unsigned int* mask, const int* prefix, const struct bartorch_grid_axes* ax, const _Complex float* B, const _Complex float* bank, _Complex float* table) { @@ -471,7 +471,7 @@ extern "C" void bartorch_cuda_bank_to_table(long E, long X, long S, int R, long CUDA_KERNEL_ERROR; } -extern "C" void bartorch_cuda_table_to_bank(long U, long X, long S, int R, long L, long per, +extern "C" void bartorch_cuda_table_to_bank(bart_dim_t U, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* csr_start, const int* csr, const int* u_coord, const unsigned int* mask, const int* prefix, const struct bartorch_grid_axes* ax, const _Complex float* B, _Complex float* bank, const _Complex float* table) { @@ -485,17 +485,17 @@ extern "C" void bartorch_cuda_table_to_bank(long U, long X, long S, int R, long * sensitivity, zero-filled along the readout from `sx` to `wx` about the * centre, and its adjoint. The image is laid out as BART lays it, readout * fastest; `map` NULL is one. */ -__global__ static void kern_pad_map(long sx, long wx, long rest, long off, +__global__ static void kern_pad_map(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, cuFloatComplex* dst, const cuFloatComplex* src, const cuFloatComplex* map) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = wx * rest; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = wx * rest; - for (long i = start; i < n; i += stride) { + for (bart_dim_t i = start; i < n; i += stride) { - long r = i / wx; - long x = i - r * wx - off; + bart_dim_t r = i / wx; + bart_dim_t x = i - r * wx - off; if ((0 > x) || (x >= sx)) { @@ -503,22 +503,22 @@ __global__ static void kern_pad_map(long sx, long wx, long rest, long off, continue; } - long j = x + r * sx; + bart_dim_t j = x + r * sx; dst[i] = (NULL == map) ? src[j] : cuCmulf(src[j], map[j]); } } -__global__ static void kern_crop_mapc_add(long sx, long wx, long rest, long off, +__global__ static void kern_crop_mapc_add(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, cuFloatComplex* dst, const cuFloatComplex* src, const cuFloatComplex* map) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = sx * rest; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = sx * rest; - for (long j = start; j < n; j += stride) { + for (bart_dim_t j = start; j < n; j += stride) { - long r = j / sx; + bart_dim_t r = j / sx; cuFloatComplex v = src[(j - r * sx) + off + r * wx]; if (NULL != map) @@ -528,7 +528,7 @@ __global__ static void kern_crop_mapc_add(long sx, long wx, long rest, long off, } } -extern "C" void bartorch_cuda_pad_map(long sx, long wx, long rest, long off, +extern "C" void bartorch_cuda_pad_map(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, _Complex float* dst, const _Complex float* src, const _Complex float* map) { kern_pad_map<<>>(sx, wx, rest, off, @@ -537,7 +537,7 @@ extern "C" void bartorch_cuda_pad_map(long sx, long wx, long rest, long off, CUDA_KERNEL_ERROR; } -extern "C" void bartorch_cuda_crop_mapc_add(long sx, long wx, long rest, long off, +extern "C" void bartorch_cuda_crop_mapc_add(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, _Complex float* dst, const _Complex float* src, const _Complex float* map) { kern_crop_mapc_add<<>>(sx, wx, rest, off, diff --git a/src/csrc/ops/ops.c b/src/csrc/ops/ops.c index 37a5aef5..dee19392 100644 --- a/src/csrc/ops/ops.c +++ b/src/csrc/ops/ops.c @@ -129,7 +129,7 @@ static bartorch_nlop* wrap_nlop(const struct nlop_s* op) return h; } -static void copy_dims(const struct iovec_s* iov, int N, long* dims) +static void copy_dims(const struct iovec_s* iov, int N, bart_dim_t* dims) { for (int i = 0; i < N; i++) dims[i] = (i < iov->N) ? iov->dims[i] : 1; @@ -186,7 +186,7 @@ static void cb_lin_del(const linop_data_t* _d) struct linop_callback_args { - int ON; const long* odims; int IN; const long* idims; + int ON; const bart_dim_t* odims; int IN; const bart_dim_t* idims; bartorch_apply_fn forward; bartorch_apply_fn adjoint; bartorch_apply_fn normal; void* ctx; bartorch_release_fn release; bartorch_linop* result; @@ -209,7 +209,7 @@ static int linop_callback_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_callback(int ON, const long* odims, int IN, const long* idims, +bartorch_linop* bartorch_linop_callback(int ON, const int64_t* odims, int IN, const int64_t* idims, bartorch_apply_fn forward, bartorch_apply_fn adjoint, bartorch_apply_fn normal, void* ctx, bartorch_release_fn release) { @@ -220,7 +220,7 @@ bartorch_linop* bartorch_linop_callback(int ON, const long* odims, int IN, const /* --- BART's own linear operators ------------------------------------------- */ -struct linop_fft_args { int N; const long* dims; unsigned long flags; int inverse; int centered; bartorch_linop* result; }; +struct linop_fft_args { int N; const bart_dim_t* dims; bart_flags_t flags; int inverse; int centered; bartorch_linop* result; }; static int linop_fft_worker(void* p) { @@ -236,13 +236,13 @@ static int linop_fft_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_fft(int N, const long* dims, unsigned long flags, int inverse, int centered) +bartorch_linop* bartorch_linop_fft(int N, const int64_t* dims, uint64_t flags, int inverse, int centered) { struct linop_fft_args a = { N, dims, flags, inverse, centered, NULL }; return (0 == guarded(linop_fft_worker, &a)) ? a.result : NULL; } -struct linop_cdiag_args { int N; const long* dims; unsigned long flags; const void* diag; bartorch_linop* result; }; +struct linop_cdiag_args { int N; const bart_dim_t* dims; bart_flags_t flags; const void* diag; bartorch_linop* result; }; static int linop_cdiag_worker(void* p) { @@ -251,13 +251,13 @@ static int linop_cdiag_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_cdiag(int N, const long* dims, unsigned long flags, const void* diag) +bartorch_linop* bartorch_linop_cdiag(int N, const int64_t* dims, uint64_t flags, const void* diag) { struct linop_cdiag_args a = { N, dims, flags, diag, NULL }; return (0 == guarded(linop_cdiag_worker, &a)) ? a.result : NULL; } -struct linop_fmac_args { int N; const long* odims; const long* idims; const long* tdims; const void* tensor; bartorch_linop* result; }; +struct linop_fmac_args { int N; const bart_dim_t* odims; const bart_dim_t* idims; const bart_dim_t* tdims; const void* tensor; bartorch_linop* result; }; static int linop_fmac_worker(void* p) { @@ -266,13 +266,13 @@ static int linop_fmac_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_fmac(int N, const long* odims, const long* idims, const long* tdims, const void* tensor) +bartorch_linop* bartorch_linop_fmac(int N, const int64_t* odims, const int64_t* idims, const int64_t* tdims, const void* tensor) { struct linop_fmac_args a = { N, odims, idims, tdims, tensor, NULL }; return (0 == guarded(linop_fmac_worker, &a)) ? a.result : NULL; } -struct linop_sampling_args { const long* dims; const long* pat_dims; const void* pattern; bartorch_linop* result; }; +struct linop_sampling_args { const bart_dim_t* dims; const bart_dim_t* pat_dims; const void* pattern; bartorch_linop* result; }; static int linop_sampling_worker(void* p) { @@ -281,13 +281,13 @@ static int linop_sampling_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_sampling(const long* dims, const long* pat_dims, const void* pattern) +bartorch_linop* bartorch_linop_sampling(const int64_t* dims, const int64_t* pat_dims, const void* pattern) { struct linop_sampling_args a = { dims, pat_dims, pattern, NULL }; return (0 == guarded(linop_sampling_worker, &a)) ? a.result : NULL; } -struct linop_set_diag_args { const bartorch_linop* h; int N; const long* ddims; const void* diag; }; +struct linop_set_diag_args { const bartorch_linop* h; int N; const bart_dim_t* ddims; const void* diag; }; static int linop_set_diag_worker(void* p) { @@ -303,7 +303,7 @@ static int linop_set_diag_worker(void* p) return 0; } -int bartorch_linop_set_diagonal(const bartorch_linop* h, int N, const long* ddims, const void* diag) +int bartorch_linop_set_diagonal(const bartorch_linop* h, int N, const int64_t* ddims, const void* diag) { struct linop_set_diag_args a = { h, N, ddims, diag }; return guarded(linop_set_diag_worker, &a); @@ -311,9 +311,9 @@ int bartorch_linop_set_diagonal(const bartorch_linop* h, int N, const long* ddim struct linop_nufft_args { - int N; const long* ksp_dims; const long* cim_dims; const long* traj_dims; const void* traj; - const long* wgh_dims; const void* weights; - const long* bas_dims; const void* basis; + int N; const bart_dim_t* ksp_dims; const bart_dim_t* cim_dims; const bart_dim_t* traj_dims; const void* traj; + const bart_dim_t* wgh_dims; const void* weights; + const bart_dim_t* bas_dims; const void* basis; int toeplitz; float os; float width; bartorch_linop* result; }; @@ -334,7 +334,7 @@ static int linop_nufft_worker(void* p) /* Weights and a subspace basis belong to the operator rather than to * something chained onto it: the normal is a point spread function over * both, which a chain could not be. */ - long wgh_dims[a->N]; + bart_dim_t wgh_dims[a->N]; if (NULL == a->weights) md_singleton_dims(a->N, wgh_dims); @@ -346,9 +346,9 @@ static int linop_nufft_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_nufft(int N, const long* ksp_dims, const long* cim_dims, const long* traj_dims, - const void* traj, const long* wgh_dims, const void* weights, - const long* bas_dims, const void* basis, int toeplitz, float os, float width) +bartorch_linop* bartorch_linop_nufft(int N, const int64_t* ksp_dims, const int64_t* cim_dims, const int64_t* traj_dims, + const void* traj, const int64_t* wgh_dims, const void* weights, + const int64_t* bas_dims, const void* basis, int toeplitz, float os, float width) { struct linop_nufft_args a = { N, ksp_dims, cim_dims, traj_dims, traj, wgh_dims, weights, bas_dims, basis, toeplitz, os, width, NULL }; @@ -369,9 +369,9 @@ struct blocks_s { linop_data_t super; const struct linop_s* op; - long n; - long isize; - long osize; + bart_dim_t n; + bart_dim_t isize; + bart_dim_t osize; }; static DEF_TYPEID(blocks_s); @@ -380,7 +380,7 @@ static void blocks_forward(const linop_data_t* _d, complex float* dst, const com { const auto d = CAST_DOWN(blocks_s, _d); - for (long k = 0; k < d->n; k++) + for (bart_dim_t k = 0; k < d->n; k++) linop_forward_unchecked(d->op, dst + k * d->osize, src + k * d->isize); } @@ -388,7 +388,7 @@ static void blocks_adjoint(const linop_data_t* _d, complex float* dst, const com { const auto d = CAST_DOWN(blocks_s, _d); - for (long k = 0; k < d->n; k++) + for (bart_dim_t k = 0; k < d->n; k++) linop_adjoint_unchecked(d->op, dst + k * d->isize, src + k * d->osize); } @@ -396,7 +396,7 @@ static void blocks_normal(const linop_data_t* _d, complex float* dst, const comp { const auto d = CAST_DOWN(blocks_s, _d); - for (long k = 0; k < d->n; k++) + for (bart_dim_t k = 0; k < d->n; k++) linop_normal_unchecked(d->op, dst + k * d->isize, src + k * d->isize); } @@ -410,7 +410,7 @@ static void blocks_del(const linop_data_t* _d) struct linop_blocks_args { - const bartorch_linop* block; int N; const long* odims; const long* idims; long n; + const bartorch_linop* block; int N; const bart_dim_t* odims; const bart_dim_t* idims; bart_dim_t n; bartorch_linop* result; }; @@ -421,11 +421,11 @@ static int linop_blocks_worker(void* p) auto dom = linop_domain(a->block->op); auto cod = linop_codomain(a->block->op); - long isize = md_calc_size(dom->N, dom->dims); - long osize = md_calc_size(cod->N, cod->dims); + bart_dim_t isize = md_calc_size(dom->N, dom->dims); + bart_dim_t osize = md_calc_size(cod->N, cod->dims); if ((a->n < 1) || (md_calc_size(a->N, a->idims) != a->n * isize) || (md_calc_size(a->N, a->odims) != a->n * osize)) - error("bartorch: %ld blocks of an operator from %ld to %ld elements are not %ld to %ld elements\n", + error("bartorch: %" PRId64 " blocks of an operator from %" PRId64 " to %" PRId64 " elements are not %" PRId64 " to %" PRId64 " elements\n", a->n, isize, osize, md_calc_size(a->N, a->idims), md_calc_size(a->N, a->odims)); PTR_ALLOC(struct blocks_s, d); @@ -441,7 +441,7 @@ static int linop_blocks_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_blocks(const bartorch_linop* block, int N, const long* odims, const long* idims, long n) +bartorch_linop* bartorch_linop_blocks(const bartorch_linop* block, int N, const int64_t* odims, const int64_t* idims, int64_t n) { struct linop_blocks_args a = { block, N, odims, idims, n, NULL }; return (0 == guarded(linop_blocks_worker, &a)) ? a.result : NULL; @@ -452,7 +452,7 @@ bartorch_linop* bartorch_linop_blocks(const bartorch_linop* block, int N, const * hands the caller's buffers straight through. An operator built on BART's * roles for a torch layout is given the plain reversal of the torch shapes * this way, so it chains with every other operator built on that reversal. */ -struct linop_reshaped_args { const bartorch_linop* op; int N; const long* odims; const long* idims; bartorch_linop* result; }; +struct linop_reshaped_args { const bartorch_linop* op; int N; const bart_dim_t* odims; const bart_dim_t* idims; bartorch_linop* result; }; static int linop_reshaped_worker(void* p) { @@ -469,7 +469,7 @@ static int linop_reshaped_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_reshaped(const bartorch_linop* op, int N, const long* odims, const long* idims) +bartorch_linop* bartorch_linop_reshaped(const bartorch_linop* op, int N, const int64_t* odims, const int64_t* idims) { struct linop_reshaped_args a = { op, N, odims, idims, NULL }; return (0 == guarded(linop_reshaped_worker, &a)) ? a.result : NULL; @@ -520,8 +520,8 @@ static int linop_with_normal_worker(void* p) /* `linop_from_ops` asserts this, and an assertion here would take the * process rather than the call. */ - if (!md_check_equal_dims(dom->N, dom->dims, nrm_dom->dims, ~0UL) - || !md_check_equal_dims(dom->N, dom->dims, nrm_cod->dims, ~0UL)) + if (!md_check_equal_dims(dom->N, dom->dims, nrm_dom->dims, ~UINT64_C(0)) + || !md_check_equal_dims(dom->N, dom->dims, nrm_cod->dims, ~UINT64_C(0))) error("bartorch: a normal operator maps the domain to itself\n"); /* `linop_from_ops` takes its own reference to each of these. */ @@ -639,7 +639,7 @@ bartorch_linop* bartorch_linop_normal_op(const bartorch_linop* a) return (0 == guarded(linop_normal_op_worker, &args)) ? args.result : NULL; } -struct linop_scale_args { int N; const long* dims; float re; float im; bartorch_linop* result; }; +struct linop_scale_args { int N; const bart_dim_t* dims; float re; float im; bartorch_linop* result; }; static int linop_scale_worker(void* p) { @@ -649,13 +649,13 @@ static int linop_scale_worker(void* p) } /* The scale is passed as two floats: a complex float is not in the ABI. */ -bartorch_linop* bartorch_linop_scale(int N, const long* dims, float re, float im) +bartorch_linop* bartorch_linop_scale(int N, const int64_t* dims, float re, float im) { struct linop_scale_args args = { N, dims, re, im, NULL }; return (0 == guarded(linop_scale_worker, &args)) ? args.result : NULL; } -struct linop_dims_args { int N; const long* dims; bartorch_linop* result; }; +struct linop_dims_args { int N; const bart_dim_t* dims; bartorch_linop* result; }; static int linop_zconj_worker(void* p) { @@ -664,7 +664,7 @@ static int linop_zconj_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_zconj(int N, const long* dims) +bartorch_linop* bartorch_linop_zconj(int N, const int64_t* dims) { struct linop_dims_args args = { N, dims, NULL }; return (0 == guarded(linop_zconj_worker, &args)) ? args.result : NULL; @@ -677,13 +677,13 @@ static int linop_identity_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_identity(int N, const long* dims) +bartorch_linop* bartorch_linop_identity(int N, const int64_t* dims) { struct linop_dims_args args = { N, dims, NULL }; return (0 == guarded(linop_identity_worker, &args)) ? args.result : NULL; } -struct linop_null_args { int NO; const long* odims; int NI; const long* idims; bartorch_linop* result; }; +struct linop_null_args { int NO; const bart_dim_t* odims; int NI; const bart_dim_t* idims; bartorch_linop* result; }; static int linop_null_worker(void* p) { @@ -692,7 +692,7 @@ static int linop_null_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_null(int NO, const long* odims, int NI, const long* idims) +bartorch_linop* bartorch_linop_null(int NO, const int64_t* odims, int NI, const int64_t* idims) { struct linop_null_args args = { NO, odims, NI, idims, NULL }; return (0 == guarded(linop_null_worker, &args)) ? args.result : NULL; @@ -741,7 +741,7 @@ double bartorch_linop_maxeigen(const bartorch_linop* a) * reversed and padded to DIMS by the host, so nothing here reorders anything. */ -struct linop_flagged_args { int N; const long* dims; unsigned long flags; const void* data; bartorch_linop* result; }; +struct linop_flagged_args { int N; const bart_dim_t* dims; bart_flags_t flags; const void* data; bartorch_linop* result; }; static int linop_rdiag_worker(void* p) { @@ -752,13 +752,13 @@ static int linop_rdiag_worker(void* p) /* md_zrmul: real parts by real parts and imaginary by imaginary, which is a * diagonal on each of the two components and not a real-valued diagonal. */ -bartorch_linop* bartorch_linop_rdiag(int N, const long* dims, unsigned long flags, const void* diag) +bartorch_linop* bartorch_linop_rdiag(int N, const int64_t* dims, uint64_t flags, const void* diag) { struct linop_flagged_args a = { N, dims, flags, diag, NULL }; return (0 == guarded(linop_rdiag_worker, &a)) ? a.result : NULL; } -struct linop_matrix_args { int N; const long* odims; const long* idims; const long* mdims; const void* matrix; bartorch_linop* result; }; +struct linop_matrix_args { int N; const bart_dim_t* odims; const bart_dim_t* idims; const bart_dim_t* mdims; const void* matrix; bartorch_linop* result; }; static int linop_matrix_worker(void* p) { @@ -767,13 +767,13 @@ static int linop_matrix_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_matrix(int N, const long* odims, const long* idims, const long* mdims, const void* matrix) +bartorch_linop* bartorch_linop_matrix(int N, const int64_t* odims, const int64_t* idims, const int64_t* mdims, const void* matrix) { struct linop_matrix_args a = { N, odims, idims, mdims, matrix, NULL }; return (0 == guarded(linop_matrix_worker, &a)) ? a.result : NULL; } -struct linop_conv_args { int N; unsigned long flags; int ctype; int cmode; const long* odims; const long* idims; const long* kdims; const void* kernel; bartorch_linop* result; }; +struct linop_conv_args { int N; bart_flags_t flags; int ctype; int cmode; const bart_dim_t* odims; const bart_dim_t* idims; const bart_dim_t* kdims; const void* kernel; bartorch_linop* result; }; static int linop_conv_worker(void* p) { @@ -783,14 +783,14 @@ static int linop_conv_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_conv(int N, unsigned long flags, int ctype, int cmode, - const long* odims, const long* idims, const long* kdims, const void* kernel) +bartorch_linop* bartorch_linop_conv(int N, uint64_t flags, int ctype, int cmode, + const int64_t* odims, const int64_t* idims, const int64_t* kdims, const void* kernel) { struct linop_conv_args a = { N, flags, ctype, cmode, odims, idims, kdims, kernel, NULL }; return (0 == guarded(linop_conv_worker, &a)) ? a.result : NULL; } -struct linop_grad_args { int N; const long* dims; int d; unsigned long flags; bartorch_linop* result; }; +struct linop_grad_args { int N; const bart_dim_t* dims; int d; bart_flags_t flags; bartorch_linop* result; }; static int linop_grad_worker(void* p) { @@ -801,7 +801,7 @@ static int linop_grad_worker(void* p) /* The finite differences along the flagged axes, stacked along d, which must * be a dimension the input has only one of. */ -bartorch_linop* bartorch_linop_grad(int N, const long* dims, int d, unsigned long flags) +bartorch_linop* bartorch_linop_grad(int N, const int64_t* dims, int d, uint64_t flags) { struct linop_grad_args a = { N, dims, d, flags, NULL }; return (0 == guarded(linop_grad_worker, &a)) ? a.result : NULL; @@ -814,9 +814,9 @@ static int linop_zreal_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_zreal(int N, const long* dims) +bartorch_linop* bartorch_linop_zreal(int N, const int64_t* dims) { - struct linop_flagged_args a = { N, dims, 0UL, NULL, NULL }; + struct linop_flagged_args a = { N, dims, 0, NULL, NULL }; return (0 == guarded(linop_zreal_worker, &a)) ? a.result : NULL; } @@ -828,7 +828,7 @@ static int linop_sum_worker(void* p) } /* The one BART operator with a closed-form pseudo-inverse. */ -bartorch_linop* bartorch_linop_sum(int N, const long* dims, unsigned long flags) +bartorch_linop* bartorch_linop_sum(int N, const int64_t* dims, uint64_t flags) { struct linop_flagged_args a = { N, dims, flags, NULL, NULL }; return (0 == guarded(linop_sum_worker, &a)) ? a.result : NULL; @@ -844,7 +844,7 @@ static int linop_scaled_sum_worker(void* p) /* The sum divided by the square root of how many were summed, so that its * normal is an orthogonal projection. That is the operator BART's closed-form * pseudo-inverse is written for. */ -bartorch_linop* bartorch_linop_scaled_sum(int N, const long* dims, unsigned long flags) +bartorch_linop* bartorch_linop_scaled_sum(int N, const int64_t* dims, uint64_t flags) { struct linop_flagged_args a = { N, dims, flags, NULL, NULL }; return (0 == guarded(linop_scaled_sum_worker, &a)) ? a.result : NULL; @@ -857,7 +857,7 @@ static int linop_avg_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_avg(int N, const long* dims, unsigned long flags) +bartorch_linop* bartorch_linop_avg(int N, const int64_t* dims, uint64_t flags) { struct linop_flagged_args a = { N, dims, flags, NULL, NULL }; return (0 == guarded(linop_avg_worker, &a)) ? a.result : NULL; @@ -871,13 +871,13 @@ static int linop_repmat_worker(void* p) } /* dims is the codomain: the domain is it with the flagged axes set to one. */ -bartorch_linop* bartorch_linop_repmat(int N, const long* odims, unsigned long flags) +bartorch_linop* bartorch_linop_repmat(int N, const int64_t* odims, uint64_t flags) { struct linop_flagged_args a = { N, odims, flags, NULL, NULL }; return (0 == guarded(linop_repmat_worker, &a)) ? a.result : NULL; } -struct linop_hankel_args { int N; const long* dims; int dim; int window_dim; int window; bartorch_linop* result; }; +struct linop_hankel_args { int N; const bart_dim_t* dims; int dim; int window_dim; int window; bartorch_linop* result; }; static int linop_hankel_worker(void* p) { @@ -889,7 +889,7 @@ static int linop_hankel_worker(void* p) /* A sliding window along dim, laid out along window_dim -- which the input * must have only one of. BART builds it as a strided view, so nothing is * copied to make the windows overlap. */ -bartorch_linop* bartorch_linop_hankel(int N, const long* dims, int dim, int window_dim, int window) +bartorch_linop* bartorch_linop_hankel(int N, const int64_t* dims, int dim, int window_dim, int window) { struct linop_hankel_args a = { N, dims, dim, window_dim, window, NULL }; return (0 == guarded(linop_hankel_worker, &a)) ? a.result : NULL; @@ -902,13 +902,13 @@ static int linop_flip_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_flip(int N, const long* dims, unsigned long flags) +bartorch_linop* bartorch_linop_flip(int N, const int64_t* dims, uint64_t flags) { struct linop_flagged_args a = { N, dims, flags, NULL, NULL }; return (0 == guarded(linop_flip_worker, &a)) ? a.result : NULL; } -struct linop_two_shapes_args { int NO; const long* odims; int NI; const long* idims; const long* pos; bartorch_linop* result; }; +struct linop_two_shapes_args { int NO; const bart_dim_t* odims; int NI; const bart_dim_t* idims; const bart_dim_t* pos; bartorch_linop* result; }; static int linop_reshape_worker(void* p) { @@ -917,7 +917,7 @@ static int linop_reshape_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_reshape(int NO, const long* odims, int NI, const long* idims) +bartorch_linop* bartorch_linop_reshape(int NO, const int64_t* odims, int NI, const int64_t* idims) { struct linop_two_shapes_args a = { NO, odims, NI, idims, NULL, NULL }; return (0 == guarded(linop_reshape_worker, &a)) ? a.result : NULL; @@ -932,7 +932,7 @@ static int linop_resize_worker(void* p) /* Centred: what BART's `resize -c` does, cropping or zero-filling about the * middle of each axis rather than the corner. */ -bartorch_linop* bartorch_linop_resize(int N, const long* odims, const long* idims) +bartorch_linop* bartorch_linop_resize(int N, const int64_t* odims, const int64_t* idims) { struct linop_two_shapes_args a = { N, odims, N, idims, NULL, NULL }; return (0 == guarded(linop_resize_worker, &a)) ? a.result : NULL; @@ -945,13 +945,13 @@ static int linop_extract_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_extract(int N, const long* pos, const long* odims, const long* idims) +bartorch_linop* bartorch_linop_extract(int N, const int64_t* pos, const int64_t* odims, const int64_t* idims) { struct linop_two_shapes_args a = { N, odims, N, idims, pos, NULL }; return (0 == guarded(linop_extract_worker, &a)) ? a.result : NULL; } -struct linop_transpose_args { int N; int a; int b; const long* dims; bartorch_linop* result; }; +struct linop_transpose_args { int N; int a; int b; const bart_dim_t* dims; bartorch_linop* result; }; static int linop_transpose_worker(void* p) { @@ -960,13 +960,13 @@ static int linop_transpose_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_transpose(int N, int a, int b, const long* dims) +bartorch_linop* bartorch_linop_transpose(int N, int a, int b, const int64_t* dims) { struct linop_transpose_args t = { N, a, b, dims, NULL }; return (0 == guarded(linop_transpose_worker, &t)) ? t.result : NULL; } -struct linop_permute_args { int N; const int* order; const long* idims; bartorch_linop* result; }; +struct linop_permute_args { int N; const int* order; const bart_dim_t* idims; bartorch_linop* result; }; static int linop_permute_worker(void* p) { @@ -976,13 +976,13 @@ static int linop_permute_worker(void* p) } /* order is BART's: the output's dimension i is the input's order[i]. */ -bartorch_linop* bartorch_linop_permute(int N, const int* order, const long* idims) +bartorch_linop* bartorch_linop_permute(int N, const int* order, const int64_t* idims) { struct linop_permute_args a = { N, order, idims, NULL }; return (0 == guarded(linop_permute_worker, &a)) ? a.result : NULL; } -struct linop_shift_args { int N; const long* dims; int dim; long shift; int pad; bartorch_linop* result; }; +struct linop_shift_args { int N; const bart_dim_t* dims; int dim; bart_dim_t shift; int pad; bartorch_linop* result; }; static int linop_shift_worker(void* p) { @@ -991,21 +991,21 @@ static int linop_shift_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_shift(int N, const long* dims, int dim, long shift, int pad) +bartorch_linop* bartorch_linop_shift(int N, const int64_t* dims, int dim, int64_t shift, int pad) { struct linop_shift_args a = { N, dims, dim, shift, pad, NULL }; return (0 == guarded(linop_shift_worker, &a)) ? a.result : NULL; } -struct linop_padding_args { int N; const long* dims; int pad; const long* before; const long* after; bartorch_linop* result; }; +struct linop_padding_args { int N; const bart_dim_t* dims; int pad; const bart_dim_t* before; const bart_dim_t* after; bartorch_linop* result; }; static int linop_padding_worker(void* p) { struct linop_padding_args* a = p; /* BART takes these non-const and does not write through them. */ - long before[DIMS]; - long after[DIMS]; + bart_dim_t before[DIMS]; + bart_dim_t after[DIMS]; md_copy_dims(a->N, before, a->before); md_copy_dims(a->N, after, a->after); @@ -1013,7 +1013,7 @@ static int linop_padding_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_padding(int N, const long* dims, int pad, const long* before, const long* after) +bartorch_linop* bartorch_linop_padding(int N, const int64_t* dims, int pad, const int64_t* before, const int64_t* after) { struct linop_padding_args a = { N, dims, pad, before, after, NULL }; return (0 == guarded(linop_padding_worker, &a)) ? a.result : NULL; @@ -1052,14 +1052,14 @@ int bartorch_linop_pseudo_inv(const bartorch_linop* h, float lambda, void* dst, return guarded(linop_pinv_worker, &args); } -int bartorch_linop_domain(const bartorch_linop* h, int N, long* dims) +int bartorch_linop_domain(const bartorch_linop* h, int N, int64_t* dims) { const struct iovec_s* iov = linop_domain(h->op); copy_dims(iov, N, dims); return iov->N; } -int bartorch_linop_codomain(const bartorch_linop* h, int N, long* dims) +int bartorch_linop_codomain(const bartorch_linop* h, int N, int64_t* dims) { const struct iovec_s* iov = linop_codomain(h->op); copy_dims(iov, N, dims); @@ -1161,7 +1161,7 @@ static void cb_nl_del(const nlop_data_t* _d) struct nlop_callback_args { - int ON; const long* odims; int IN; const long* idims; + int ON; const bart_dim_t* odims; int IN; const bart_dim_t* idims; bartorch_apply_fn forward; bartorch_apply_fn derivative; bartorch_apply_fn adjoint; void* ctx; bartorch_release_fn release; bartorch_nlop* result; @@ -1236,8 +1236,8 @@ static void cbg_del(const nlop_data_t* _d) struct nlop_callback_generic_args { - int OO; int ON; const long* odims; - int II; int IN; const long* idims; + int OO; int ON; const bart_dim_t* odims; + int II; int IN; const bart_dim_t* idims; bartorch_generic_apply_fn forward; bartorch_pair_apply_fn derivative; bartorch_pair_apply_fn adjoint; @@ -1261,8 +1261,8 @@ static int nlop_callback_generic_worker(void* p) /* The shapes arrive flat, one argument after another; BART wants them * as arrays of arrays, which is the same memory read differently. */ - const long (*od)[a->ON] = (const long (*)[a->ON])a->odims; - const long (*id)[a->IN] = (const long (*)[a->IN])a->idims; + const bart_dim_t (*od)[a->ON] = (const bart_dim_t (*)[a->ON])a->odims; + const bart_dim_t (*id)[a->IN] = (const bart_dim_t (*)[a->IN])a->idims; /* One derivative and one adjoint for every (input, output) pair. The * two shims below dispatch on the pair themselves, so every entry is @@ -1285,8 +1285,8 @@ static int nlop_callback_generic_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const long* odims, - int II, int IN, const long* idims, +bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const int64_t* odims, + int II, int IN, const int64_t* idims, bartorch_generic_apply_fn forward, bartorch_pair_apply_fn derivative, bartorch_pair_apply_fn adjoint, @@ -1304,7 +1304,7 @@ bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const long* odims, return (0 == guarded(nlop_callback_generic_worker, &args)) ? args.result : NULL; } -bartorch_nlop* bartorch_nlop_callback(int ON, const long* odims, int IN, const long* idims, +bartorch_nlop* bartorch_nlop_callback(int ON, const int64_t* odims, int IN, const int64_t* idims, bartorch_apply_fn forward, bartorch_apply_fn derivative, bartorch_apply_fn adjoint, void* ctx, bartorch_release_fn release) { @@ -1343,14 +1343,14 @@ bartorch_nlop* bartorch_nlop_chain(const bartorch_nlop* a, const bartorch_nlop* return (0 == guarded(nlop_chain_worker, &args)) ? args.result : NULL; } -int bartorch_nlop_domain(const bartorch_nlop* h, int N, long* dims) +int bartorch_nlop_domain(const bartorch_nlop* h, int N, int64_t* dims) { const struct iovec_s* iov = nlop_domain(h->op); copy_dims(iov, N, dims); return iov->N; } -int bartorch_nlop_codomain(const bartorch_nlop* h, int N, long* dims) +int bartorch_nlop_codomain(const bartorch_nlop* h, int N, int64_t* dims) { const struct iovec_s* iov = nlop_codomain(h->op); copy_dims(iov, N, dims); @@ -1366,7 +1366,7 @@ static int nlop_apply_worker(void* p) const struct iovec_s* cod = nlop_codomain(a->h->op); switch (a->mode) { - case 0: nlop_generic_apply_select_derivative_unchecked(a->h->op, 2, (void*[]){ a->dst, (void*)a->src }, ~0UL, ~0UL); break; + case 0: nlop_generic_apply_select_derivative_unchecked(a->h->op, 2, (void*[]){ a->dst, (void*)a->src }, ~UINT64_C(0), ~UINT64_C(0)); break; case 1: nlop_derivative(a->h->op, cod->N, cod->dims, a->dst, dom->N, dom->dims, a->src); break; default: nlop_adjoint(a->h->op, dom->N, dom->dims, a->dst, cod->N, cod->dims, a->src); break; } @@ -1412,7 +1412,7 @@ int bartorch_nlop_outputs(const bartorch_nlop* h) return (NULL == h) ? -1 : nlop_get_nr_out_args(h->op); } -int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, long* dims) +int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, int64_t* dims) { if ((NULL == h) || (NULL == dims) || (0 > i) || (i >= nlop_get_nr_in_args(h->op))) return -1; @@ -1427,7 +1427,7 @@ int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, long* dims) return iov->N; } -int bartorch_nlop_output_codomain(const bartorch_nlop* h, int o, int N, long* dims) +int bartorch_nlop_output_codomain(const bartorch_nlop* h, int o, int N, int64_t* dims) { if ((NULL == h) || (NULL == dims) || (0 > o) || (o >= nlop_get_nr_out_args(h->op))) return -1; @@ -1463,7 +1463,7 @@ struct nlop_generic_args { const bartorch_nlop* h; int nargs; void** args; }; static int nlop_generic_worker(void* p) { struct nlop_generic_args* a = p; - nlop_generic_apply_select_derivative_unchecked(a->h->op, a->nargs, a->args, ~0UL, ~0UL); + nlop_generic_apply_select_derivative_unchecked(a->h->op, a->nargs, a->args, ~UINT64_C(0), ~UINT64_C(0)); return 0; } @@ -1517,7 +1517,7 @@ bartorch_linop* bartorch_nlop_derivative_linop(const bartorch_nlop* h, int o, in * operators side by side, an output tied back to an input, two inputs made * one, and the reorderings that make those usable. */ -struct nlop_reshape_args { const bartorch_nlop* a; int at; int N; const long* dims; int output; bartorch_nlop* result; }; +struct nlop_reshape_args { const bartorch_nlop* a; int at; int N; const bart_dim_t* dims; int output; bartorch_nlop* result; }; static int nlop_reshape_worker(void* p) { @@ -1532,7 +1532,7 @@ static int nlop_reshape_worker(void* p) return 0; } -static bartorch_nlop* nlop_reshape(const bartorch_nlop* a, int at, int N, const long* dims, int output) +static bartorch_nlop* nlop_reshape(const bartorch_nlop* a, int at, int N, const bart_dim_t* dims, int output) { if ((NULL == a) || (NULL == dims) || (0 >= N)) return NULL; @@ -1542,12 +1542,12 @@ static bartorch_nlop* nlop_reshape(const bartorch_nlop* a, int at, int N, const return (0 == guarded(nlop_reshape_worker, &args)) ? args.result : NULL; } -bartorch_nlop* bartorch_nlop_reshape_in(const bartorch_nlop* a, int i, int N, const long* dims) +bartorch_nlop* bartorch_nlop_reshape_in(const bartorch_nlop* a, int i, int N, const int64_t* dims) { return nlop_reshape(a, i, N, dims, 0); } -bartorch_nlop* bartorch_nlop_reshape_out(const bartorch_nlop* a, int o, int N, const long* dims) +bartorch_nlop* bartorch_nlop_reshape_out(const bartorch_nlop* a, int o, int N, const int64_t* dims) { return nlop_reshape(a, o, N, dims, 1); } @@ -1700,7 +1700,7 @@ static int nlop_flatten_worker(void* p) * checks the rank it was given against the operator's own -- so a * flattened model handed to a solver asserts unless the vector is * restated the way the rest of the library states it. */ - long dims[DIMS]; + bart_dim_t dims[DIMS]; md_singleton_dims(DIMS, dims); dims[0] = md_calc_size(nlop_generic_domain(op, 0)->N, nlop_generic_domain(op, 0)->dims); @@ -1757,12 +1757,12 @@ bartorch_nlop* bartorch_nlop_del_out(const bartorch_nlop* x, int o) struct nlop_dims_args { int N; - const long* dims; - const long* dims2; - const long* dims3; + const bart_dim_t* dims; + const bart_dim_t* dims2; + const bart_dim_t* dims3; float a; float b; - unsigned long flags; + bart_flags_t flags; bartorch_nlop* result; }; @@ -1773,12 +1773,12 @@ static int nlop_tenmul_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_tenmul(int N, const long* odims, const long* idims1, const long* idims2) +bartorch_nlop* bartorch_nlop_tenmul(int N, const int64_t* odims, const int64_t* idims1, const int64_t* idims2) { if ((NULL == odims) || (NULL == idims1) || (NULL == idims2)) return NULL; - struct nlop_dims_args v = { N, odims, idims1, idims2, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, odims, idims1, idims2, 0., 0., 0, NULL }; return (0 == guarded(nlop_tenmul_worker, &v)) ? v.result : NULL; } @@ -1792,12 +1792,12 @@ static int nlop_zdiv_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zdiv(int N, const long* dims, float eps) +bartorch_nlop* bartorch_nlop_zdiv(int N, const int64_t* dims, float eps) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0, NULL }; return (0 == guarded(nlop_zdiv_worker, &v)) ? v.result : NULL; } @@ -1809,12 +1809,12 @@ static int nlop_zaxpbz_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zaxpbz(int N, const long* dims, float a, float b) +bartorch_nlop* bartorch_nlop_zaxpbz(int N, const int64_t* dims, float a, float b) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, a, b, ~0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, a, b, ~UINT64_C(0), NULL }; return (0 == guarded(nlop_zaxpbz_worker, &v)) ? v.result : NULL; } @@ -1826,12 +1826,12 @@ static int nlop_zexp_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zexp(int N, const long* dims) +bartorch_nlop* bartorch_nlop_zexp(int N, const int64_t* dims) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0, NULL }; return (0 == guarded(nlop_zexp_worker, &v)) ? v.result : NULL; } @@ -1843,12 +1843,12 @@ static int nlop_zlog_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zlog(int N, const long* dims) +bartorch_nlop* bartorch_nlop_zlog(int N, const int64_t* dims) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0, NULL }; return (0 == guarded(nlop_zlog_worker, &v)) ? v.result : NULL; } @@ -1862,12 +1862,12 @@ static int nlop_zinv_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zinv(int N, const long* dims, float eps) +bartorch_nlop* bartorch_nlop_zinv(int N, const int64_t* dims, float eps) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0, NULL }; return (0 == guarded(nlop_zinv_worker, &v)) ? v.result : NULL; } @@ -1879,12 +1879,12 @@ static int nlop_zsqrt_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zsqrt(int N, const long* dims) +bartorch_nlop* bartorch_nlop_zsqrt(int N, const int64_t* dims) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0, NULL }; return (0 == guarded(nlop_zsqrt_worker, &v)) ? v.result : NULL; } @@ -1896,12 +1896,12 @@ static int nlop_zspow_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zspow(int N, const long* dims, float re, float im) +bartorch_nlop* bartorch_nlop_zspow(int N, const int64_t* dims, float re, float im) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, re, im, 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, re, im, 0, NULL }; return (0 == guarded(nlop_zspow_worker, &v)) ? v.result : NULL; } @@ -1913,12 +1913,12 @@ static int nlop_zsadd_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zsadd(int N, const long* dims, float re, float im) +bartorch_nlop* bartorch_nlop_zsadd(int N, const int64_t* dims, float re, float im) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, re, im, 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, re, im, 0, NULL }; return (0 == guarded(nlop_zsadd_worker, &v)) ? v.result : NULL; } @@ -1930,12 +1930,12 @@ static int nlop_zabs_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zabs(int N, const long* dims) +bartorch_nlop* bartorch_nlop_zabs(int N, const int64_t* dims) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0, NULL }; return (0 == guarded(nlop_zabs_worker, &v)) ? v.result : NULL; } @@ -1947,12 +1947,12 @@ static int nlop_smo_abs_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_smo_abs(int N, const long* dims, float eps) +bartorch_nlop* bartorch_nlop_smo_abs(int N, const int64_t* dims, float eps) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0, NULL }; return (0 == guarded(nlop_smo_abs_worker, &v)) ? v.result : NULL; } @@ -1966,7 +1966,7 @@ static int nlop_zrss_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zrss(int N, const long* dims, unsigned long flags, float eps) +bartorch_nlop* bartorch_nlop_zrss(int N, const int64_t* dims, uint64_t flags, float eps) { if (NULL == dims) return NULL; @@ -1983,7 +1983,7 @@ static int nlop_zss_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zss(int N, const long* dims, unsigned long flags) +bartorch_nlop* bartorch_nlop_zss(int N, const int64_t* dims, uint64_t flags) { if (NULL == dims) return NULL; @@ -1998,7 +1998,7 @@ bartorch_nlop* bartorch_nlop_zss(int N, const long* dims, unsigned long flags) * `nlop_const_create` copies, so the caller's buffer is free after the call. * Combined and linked, this is how an input is pinned to a value. */ -struct nlop_const_args { int N; const long* dims; const void* val; bartorch_nlop* result; }; +struct nlop_const_args { int N; const bart_dim_t* dims; const void* val; bartorch_nlop* result; }; static int nlop_const_worker(void* p) { @@ -2007,7 +2007,7 @@ static int nlop_const_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_const(int N, const long* dims, const void* val) +bartorch_nlop* bartorch_nlop_const(int N, const int64_t* dims, const void* val) { if ((NULL == dims) || (NULL == val)) return NULL; @@ -2017,7 +2017,7 @@ bartorch_nlop* bartorch_nlop_const(int N, const long* dims, const void* val) return (0 == guarded(nlop_const_worker, &v)) ? v.result : NULL; } -struct nlop_set_const_args { const bartorch_nlop* a; int i; int N; const long* dims; const void* val; bartorch_nlop* result; }; +struct nlop_set_const_args { const bartorch_nlop* a; int i; int N; const bart_dim_t* dims; const void* val; bartorch_nlop* result; }; static int nlop_set_input_const_worker(void* p) { @@ -2026,7 +2026,7 @@ static int nlop_set_input_const_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int N, const long* dims, const void* val) +bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int N, const int64_t* dims, const void* val) { if ((NULL == a) || (NULL == dims) || (NULL == val) || (0 > i) || (i >= nlop_get_nr_in_args(a->op))) @@ -2038,7 +2038,7 @@ bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int } -struct nlop_norm_inv_args { const bartorch_nlop* normal; int maxiter; float tol; float l2lambda; long batch; bartorch_nlop* result; }; +struct nlop_norm_inv_args { const bartorch_nlop* normal; int maxiter; float tol; float l2lambda; bart_dim_t batch; bartorch_nlop* result; }; static int nlop_norm_inv_worker(void* p) { @@ -2062,12 +2062,12 @@ static int nlop_norm_inv_worker(void* p) /* Every axis of the vector takes its own weight, which is what * `model_net.c` asks for and what lets `lambda` arrive as a tensor. */ - v->result = wrap_nlop(norm_inv_lambda_create(&conf, v->normal->op, ~0UL)); + v->result = wrap_nlop(norm_inv_lambda_create(&conf, v->normal->op, ~UINT64_C(0))); return 0; } -bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* normal, int maxiter, float tol, float l2lambda, long batch) +bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* normal, int maxiter, float tol, float l2lambda, int64_t batch) { if ((NULL == normal) || (1 > batch)) return NULL; @@ -2114,20 +2114,20 @@ void bartorch_nlop_free(bartorch_nlop* h) struct noir_create_args { int N; - const long* ksp_dims; - const long* cim_dims; - const long* img_dims; - const long* kco_dims; - const long* col_dims; - const long* pat_dims; + const bart_dim_t* ksp_dims; + const bart_dim_t* cim_dims; + const bart_dim_t* img_dims; + const bart_dim_t* kco_dims; + const bart_dim_t* col_dims; + const bart_dim_t* pat_dims; const void* pattern; - const long* trj_dims; + const bart_dim_t* trj_dims; const void* traj; - const long* wgh_dims; + const bart_dim_t* wgh_dims; const void* weights; - const long* bas_dims; + const bart_dim_t* bas_dims; const void* basis; - const long* msk_dims; + const bart_dim_t* msk_dims; const void* mask; int noncart; int optimized; @@ -2174,15 +2174,15 @@ static int noir_create_worker(void* p) } bartorch_noir* bartorch_noir_create(int N, - const long* ksp_dims, const long* cim_dims, const long* img_dims, - const long* kco_dims, const long* col_dims, - const long* pat_dims, const void* pattern, - const long* trj_dims, const void* traj, - const long* wgh_dims, const void* weights, - const long* bas_dims, const void* basis, - const long* msk_dims, const void* mask, + const int64_t* ksp_dims, const int64_t* cim_dims, const int64_t* img_dims, + const int64_t* kco_dims, const int64_t* col_dims, + const int64_t* pat_dims, const void* pattern, + const int64_t* trj_dims, const void* traj, + const int64_t* wgh_dims, const void* weights, + const int64_t* bas_dims, const void* basis, + const int64_t* msk_dims, const void* mask, int noncart, int optimized, int toeplitz, - unsigned long fft_flags, unsigned long wght_flags, + uint64_t fft_flags, uint64_t wght_flags, int rvc, int sos, float a, float b, float c, float oversampling_coils, int ret_os_coils) { @@ -2288,12 +2288,12 @@ bartorch_linop* bartorch_noir_transform(const bartorch_noir* h) return noir_linop(h, 3); } -int bartorch_noir_dims(const bartorch_noir* h, int which, int N, long* dims) +int bartorch_noir_dims(const bartorch_noir* h, int which, int N, int64_t* dims) { if ((NULL == h) || (NULL == dims) || (N < h->model.N)) return -1; - const long* src = NULL; + const bart_dim_t* src = NULL; switch (which) { case 0: src = h->model.ksp_dims; break; @@ -2357,8 +2357,8 @@ static int irgnm_worker(void* p) const struct iovec_s* dom = nlop_domain(a->F->op); const struct iovec_s* cod = nlop_codomain(a->F->op); - long N = 2 * md_calc_size(dom->N, dom->dims); - long M = 2 * md_calc_size(cod->N, cod->dims); + bart_dim_t N = 2 * md_calc_size(dom->N, dom->dims); + bart_dim_t M = 2 * md_calc_size(cod->N, cod->dims); iter4_irgnm(CAST_UP(&conf), a->F->op, N, a->x, a->xref, M, a->y, NULL, (struct iter_op_s){ NULL, NULL }); return 0; @@ -2404,8 +2404,8 @@ static int irgnm2_worker(void* p) const struct iovec_s* dom = nlop_domain(a->F->op); const struct iovec_s* cod = nlop_codomain(a->F->op); - long N = 2 * md_calc_size(dom->N, dom->dims); - long M = 2 * md_calc_size(cod->N, cod->dims); + bart_dim_t N = 2 * md_calc_size(dom->N, dom->dims); + bart_dim_t M = 2 * md_calc_size(cod->N, cod->dims); iter4_irgnm2(CAST_UP(&conf), a->F->op, N, a->x, a->xref, M, a->y, NULL, (struct iter_op_s){ NULL, NULL }); diff --git a/src/csrc/ops/paired.cu b/src/csrc/ops/paired.cu index 2fcb277d..4c98b5f1 100644 --- a/src/csrc/ops/paired.cu +++ b/src/csrc/ops/paired.cu @@ -196,7 +196,7 @@ struct XArgs { const unsigned* mask; const int* prefix; const cplx* tx; /* 2 x N: each set's phase along x */ - long L; + bart_dim_t L; }; /* The pass along x of both sets of the pair: per line of four coefficients, @@ -307,7 +307,7 @@ __global__ void __launch_bounds__(Shape::XF::max_threads_per_block) fused(XAr if (word & bit) { - const long j = a.prefix[g >> 5] + __popc(word & (bit - 1)); + const bart_dim_t j = a.prefix[g >> 5] + __popc(word & (bit - 1)); #pragma unroll for (unsigned r = 0; r < R; r++) @@ -316,7 +316,7 @@ __global__ void __launch_bounds__(Shape::XF::max_threads_per_block) fused(XAr const unsigned lo = (r < c) ? r : c; const unsigned hi = (r < c) ? c : r; - const float m = widen(static_cast(a.psf[sx])[(long)(lo + hi * (hi + 1) / 2) * a.L + j]); + const float m = widen(static_cast(a.psf[sx])[(bart_dim_t)(lo + hi * (hi + 1) / 2) * a.L + j]); o[r].x += m * v[c].x; o[r].y += m * v[c].y; @@ -377,7 +377,7 @@ struct Call { int bf16; const unsigned* mask; const int* prefix; - long L; + bart_dim_t L; const cplx* tab; /* the pair's tables: x of each set, y, z */ float scale; }; @@ -496,7 +496,7 @@ const Entry table[] = { BARTORCH_PAIRED_SIZES(BARTORCH_PAIRED_ENTRY) }; /* A set's phase along one axis, as `phase_setup` in coset.cuh builds it -- * the shift, the centring, the fftmod folded in -- in double precision. */ -void axis_phase(long d, float shift, cplx* out) +void axis_phase(bart_dim_t d, float shift, cplx* out) { double s = shift; @@ -506,13 +506,13 @@ void axis_phase(long d, float shift, cplx* out) double slope = 2. * M_PI * s / d; double offset = -slope * d / 2.; - long centre = d / 2; + bart_dim_t centre = d / 2; double half = (double)centre / d; slope += 2. * M_PI * half; offset -= 2. * M_PI * half * centre / 2.; - for (long i = 0; i < d; i++) { + for (bart_dim_t i = 0; i < d; i++) { double v = offset + i * slope; out[i].x = (float)cos(v); @@ -533,7 +533,7 @@ struct bartorch_paired { /* The pair kernels for a grid of `dims`, with the sets' shifts as * `bartorch_psf_shift` gives them (set i and i + 1, i even, differ only along * x); NULL where there are none or the card cannot run them. */ -extern "C" struct bartorch_paired* bartorch_paired_create(const long dims[3], int coeffs, int sets, const float (*shifts)[3]) +extern "C" struct bartorch_paired* bartorch_paired_create(const bart_dim_t dims[3], int coeffs, int sets, const float (*shifts)[3]) { if ((R != (unsigned)coeffs) || (8 != sets) || (dims[0] != dims[1]) || (dims[0] != dims[2])) return NULL; @@ -541,7 +541,7 @@ extern "C" struct bartorch_paired* bartorch_paired_create(const long dims[3], in const Entry* entry = NULL; for (const Entry& e : table) - if ((long)e.n == dims[0]) + if ((bart_dim_t)e.n == dims[0]) entry = &e; if ((NULL == entry) || (0 != entry->prepare())) @@ -551,7 +551,7 @@ extern "C" struct bartorch_paired* bartorch_paired_create(const long dims[3], in if ((shifts[i][1] != shifts[i + 1][1]) || (shifts[i][2] != shifts[i + 1][2])) return NULL; - const long n = dims[0]; + const bart_dim_t n = dims[0]; const int pairs = sets / 2; cplx* host = (cplx*)xmalloc(sizeof(cplx) * 4 * n * pairs); @@ -583,7 +583,7 @@ extern "C" struct bartorch_paired* bartorch_paired_create(const long dims[3], in cudaMemcpy(p->tables, host, sizeof(cplx) * 4 * n * pairs, cudaMemcpyHostToDevice); xfree(host); - debug_printf(DP_DEBUG1, "bartorch: paired kernels for %ld^3\n", n); + debug_printf(DP_DEBUG1, "bartorch: paired kernels for %" PRId64 "^3\n", n); return p; } @@ -623,7 +623,7 @@ extern "C" void bartorch_paired_in(const struct bartorch_paired* p, int k, } extern "C" void bartorch_paired_fused(const struct bartorch_paired* p, int k, _Complex float* scratch, - const void* psf0, const void* psf1, int bf16, const unsigned int* mask, const int* prefix, long L) + const void* psf0, const void* psf1, int bf16, const unsigned int* mask, const int* prefix, bart_dim_t L) { Call c; diff --git a/src/csrc/ops/sense.c b/src/csrc/ops/sense.c index d5e91f43..58c16322 100644 --- a/src/csrc/ops/sense.c +++ b/src/csrc/ops/sense.c @@ -67,7 +67,7 @@ extern void bartorch_nufft_coset_normal(const struct linop_s* op, complex float* extern int bartorch_nufft_coset_folds(const struct linop_s* op); extern void bartorch_nufft_coset_normal_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, int last); + const bart_stride_t map_strs[], const complex float* map, int last); extern void bartorch_nufft_coset_end(const struct linop_s* op); /* Provided by grid.c: a Cartesian transform's normal, run through cuFFT's @@ -75,27 +75,27 @@ extern void bartorch_nufft_coset_end(const struct linop_s* op); extern int bartorch_grid_folds(const struct linop_s* op, const void* ref); extern int bartorch_grid_folds_samples(const struct linop_s* op, const void* ref); extern void bartorch_grid_forward_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map); + const bart_stride_t map_strs[DIMS], const complex float* map); extern void bartorch_grid_adjoint_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map); + const bart_stride_t map_strs[DIMS], const complex float* map); extern void bartorch_grid_normal_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map); + const bart_stride_t map_strs[DIMS], const complex float* map); #ifdef USE_CUDA /* csrc/kernels.cu: a volume times BART's inverse fftmod along its first three axes. */ -extern void bartorch_cuda_modulate(const long dims[3], long rest, const long grid[3], const long off[3], +extern void bartorch_cuda_modulate(const bart_dim_t dims[3], bart_dim_t rest, const bart_dim_t grid[3], const bart_stride_t off[3], float scale, complex float* x); #endif -extern struct linop_s* bart_sense_init(unsigned long shared_img_flags, const long max_dims[DIMS], - unsigned long sens_flags, const complex float* sens); +extern struct linop_s* bart_sense_init(bart_flags_t shared_img_flags, const bart_dim_t max_dims[DIMS], + bart_flags_t sens_flags, const complex float* sens); -extern const struct linop_s* bart_sense_nc_init(const long max_dims[DIMS], const long map_dims[DIMS], const complex float* maps, - const long ksp_dims[DIMS], - const long traj_dims[DIMS], const complex float* traj, const struct nufft_conf_s* conf, - const long wgs_dims[DIMS], const complex float* weights, - const long basis_dims[DIMS], const complex float* basis, - const struct linop_s** fft_opp, unsigned long shared_img_dims); +extern const struct linop_s* bart_sense_nc_init(const bart_dim_t max_dims[DIMS], const bart_dim_t map_dims[DIMS], const complex float* maps, + const bart_dim_t ksp_dims[DIMS], + const bart_dim_t traj_dims[DIMS], const complex float* traj, const struct nufft_conf_s* conf, + const bart_dim_t wgs_dims[DIMS], const complex float* weights, + const bart_dim_t basis_dims[DIMS], const complex float* basis, + const struct linop_s** fft_opp, bart_flags_t shared_img_dims); /* How many coils a slab holds. Zero leaves the operators as BART builds * them, every coil at once, which is the fastest and the largest. */ @@ -119,7 +119,7 @@ int bartorch_sense_fold_maps(void) * (bartorch_encoding_count). bartorch_sense_counter is the first three of * them under the slab loop's own name. */ enum { SN_COUNTS = BARTORCH_ENCODING_ITEMS + 1 }; -static long sense_counters[SN_COUNTS]; +static bart_dim_t sense_counters[SN_COUNTS]; static void counted(int which) { @@ -137,7 +137,7 @@ int bartorch_sense_coil_batch(void) return coil_batch; } -long bartorch_encoding_counter(int which) +int64_t bartorch_encoding_counter(int which) { return ((0 <= which) && (which < SN_COUNTS)) ? sense_counters[which] : 0; } @@ -148,7 +148,7 @@ void bartorch_encoding_reset_counters(void) sense_counters[i] = 0; } -long bartorch_sense_counter(int which) +int64_t bartorch_sense_counter(int which) { return bartorch_encoding_counter(((BARTORCH_ENCODING_BUILT == which) || (BARTORCH_ENCODING_FOLDED == which)) ? which : BARTORCH_ENCODING_CHAINED); @@ -163,37 +163,37 @@ struct sense_s { linop_data_t super; - long batch; /* coils in a slab */ + bart_dim_t batch; /* coils in a slab */ bool fold; /* apply the maps inside the transform of the normal */ - long coils; /* coils in all */ + bart_dim_t coils; /* coils in all */ /* One slab: the dimensions the sensitivities contract over, the coil * images they produce, and what the transform answers. */ - long slab_dims[DIMS]; - long cim_dims[DIMS]; - long out_dims[DIMS]; - long img_dims[DIMS]; - - long map_dims[DIMS]; - long slab_map_strs[DIMS]; /* a slab of maps, densely */ - long cim_strs[DIMS]; - long out_strs[DIMS]; - long img_strs[DIMS]; - long map_strs[DIMS]; + bart_dim_t slab_dims[DIMS]; + bart_dim_t cim_dims[DIMS]; + bart_dim_t out_dims[DIMS]; + bart_dim_t img_dims[DIMS]; + + bart_dim_t map_dims[DIMS]; + bart_stride_t slab_map_strs[DIMS]; /* a slab of maps, densely */ + bart_stride_t cim_strs[DIMS]; + bart_stride_t out_strs[DIMS]; + bart_stride_t img_strs[DIMS]; + bart_stride_t map_strs[DIMS]; /* The whole of what the caller sees, and the stride that steps a slab * along the coil axis of it and of the sensitivities. */ - long full_out_dims[DIMS]; - long full_out_strs[DIMS]; - long out_slab_offset; - long map_slab_offset; + bart_dim_t full_out_dims[DIMS]; + bart_stride_t full_out_strs[DIMS]; + bart_stride_t out_slab_offset; + bart_stride_t map_slab_offset; /* Where the coils lie in the samples. BART's tools keep them on * COIL_DIM, so a slab is a stride along it; the torch layout puts them * slowest, so the whole is one coil's samples, contiguous, one block * after another. */ bool coils_slowest; - long block_dims[DIMS]; + bart_dim_t block_dims[DIMS]; /* The sensitivities, and whether they are ours to free: the Cartesian * operator scales and modulates a copy, the non-Cartesian one reads @@ -208,9 +208,9 @@ struct sense_s { * unitary spectrum back to the grid it was taken on needs no scaling, * so what comes back is the map band-limited and nothing else. */ const complex float* kernels; - long kern_dims[DIMS]; - long kern_strs[DIMS]; - long kern_slab_offset; + bart_dim_t kern_dims[DIMS]; + bart_stride_t kern_strs[DIMS]; + bart_stride_t kern_slab_offset; /* The transform for one slab: a Fourier transform on a grid, a NUFFT * off one. */ @@ -226,43 +226,43 @@ struct sense_s { * its blocks, and the planes the slab's transform works on are gathered * out of a coil image transformed along z and put back into one. The * normal then takes the transform along z too. NULL is every position. */ - long stack_count; - long* stack_positions; - long plane_elems; - long sub_dims[DIMS]; + bart_dim_t stack_count; + bart_dim_t* stack_positions; + bart_dim_t plane_elems; + bart_dim_t sub_dims[DIMS]; /* A contraction whose image weight differs between sets: each term's * image weight goes on before the sensitivities contract the sets and its * sample weight after the transform, all inside the slab. Zero terms * leaves any contraction around the slab instead. */ - long terms; + bart_dim_t terms; struct multiplace_array_s* term_image; struct multiplace_array_s* term_sample; - long term_image_strs[DIMS]; - long term_sample_strs[DIMS]; - long term_image_step; - long term_sample_step; - long term_image_item_step; /* how far an item steps into a term's weights, or 0 */ - long term_sample_item_step; + bart_stride_t term_image_strs[DIMS]; + bart_stride_t term_sample_strs[DIMS]; + bart_dim_t term_image_step; + bart_dim_t term_sample_step; + bart_dim_t term_image_item_step; /* how far an item steps into a term's weights, or 0 */ + bart_dim_t term_sample_item_step; /* Items that each have a trajectory of their own: `slab` is the first of * `item_slabs`, which is NULL with one item. `img_dims` and a coil's * samples are one item's, and the whole is the items one after another, * so an item is an offset into both. */ - long items; + bart_dim_t items; const struct linop_s** item_slabs; - long item_image_step; - long item_sample_step; - long whole_img_dims[DIMS]; + bart_dim_t item_image_step; + bart_dim_t item_sample_step; + bart_dim_t whole_img_dims[DIMS]; }; static DEF_TYPEID(sense_s); /* Where a slab starts, as an index into an array laid out over every coil. * Strides are in bytes and this indexes complex floats. */ -static long slab_at(long stride, long coil) +static bart_dim_t slab_at(bart_stride_t stride, bart_dim_t coil) { - return coil * stride / (long)CFL_SIZE; + return coil * stride / (bart_stride_t)CFL_SIZE; } /* Whether a slab has to be brought to where the arithmetic is. @@ -288,9 +288,9 @@ static bool assembled(const struct sense_s* d, const void* ref) * Kernels are laid out, padded back on to the image grid and transformed, * which is the map they were taken from with everything above the kernel's * own band removed. A bank on the host is copied across as it stands. */ -static void fetch_slab(const struct sense_s* d, long coil, complex float* into) +static void fetch_slab(const struct sense_s* d, bart_dim_t coil, complex float* into) { - long mdims[DIMS]; + bart_dim_t mdims[DIMS]; md_copy_dims(DIMS, mdims, d->map_dims); mdims[COIL_DIM] = d->batch; @@ -301,11 +301,11 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) return; } - long kdims[DIMS]; + bart_dim_t kdims[DIMS]; md_copy_dims(DIMS, kdims, d->kern_dims); kdims[COIL_DIM] = d->batch; - long kstrs[DIMS]; + bart_stride_t kstrs[DIMS]; md_calc_strides(DIMS, kstrs, kdims, CFL_SIZE); complex float* k = md_alloc_sameplace(DIMS, kdims, CFL_SIZE, into); @@ -320,7 +320,7 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) * planes through it, and only the third is the whole grid. A centred * unitary transform is one transform per axis whichever way it is taken, * so this is the same map. */ - long sdims[DIMS]; + bart_dim_t sdims[DIMS]; md_copy_dims(DIMS, sdims, kdims); /* Each axis's centred unitary transform is a modulation, a transform, the @@ -331,19 +331,19 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) * map at the end -- where the last axis alone would otherwise make three * passes over the whole grid. */ bool modulated = false; - long grid[3]; - long off[3]; + bart_dim_t grid[3]; + bart_stride_t off[3]; float scale = 1.f; #ifdef USE_CUDA - modulated = cuda_ondevice(into) && (md_calc_size(3, mdims) < (1L << 31)); + modulated = cuda_ondevice(into) && (md_calc_size(3, mdims) < (INT64_C(1) << 31)); for (int a = 0; a < 3; a++) { bool fft = MD_IS_SET(FFT_FLAGS, a) && (1 < mdims[a]); grid[a] = fft ? mdims[a] : 1; - off[a] = labs(mdims[a] / 2 - kdims[a] / 2); + off[a] = llabs(mdims[a] / 2 - kdims[a] / 2); if (fft) scale /= sqrtf((float)mdims[a]); @@ -359,7 +359,7 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) for (int a = 0; a < 3; a++) { - long ndims[DIMS]; + bart_dim_t ndims[DIMS]; md_copy_dims(DIMS, ndims, sdims); ndims[a] = mdims[a]; @@ -380,12 +380,12 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) md_copy_dims(DIMS, sdims, ndims); } - assert(md_check_equal_dims(DIMS, sdims, mdims, ~0UL)); + assert(md_check_equal_dims(DIMS, sdims, mdims, ~UINT64_C(0))); #ifdef USE_CUDA if (modulated) { - long zero[3] = { 0, 0, 0 }; + bart_dim_t zero[3] = { 0, 0, 0 }; bartorch_cuda_modulate(mdims, md_calc_size(DIMS - 3, mdims + 3), grid, zero, scale, into); } @@ -399,7 +399,7 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) /* Somewhere to put a slab, when one has to be put together. */ static complex float* slab_buffer(const struct sense_s* d, const void* ref) { - long mdims[DIMS]; + bart_dim_t mdims[DIMS]; md_copy_dims(DIMS, mdims, d->map_dims); mdims[COIL_DIM] = d->batch; @@ -418,8 +418,8 @@ static void stream_wait(void) { } #endif /* What one slab does, whichever way the operator is being applied. */ -typedef void (*slab_fn)(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* ctx); +typedef void (*slab_fn)(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* ctx); /* Slab buffers kept from one walk over the coils to the next. * @@ -430,7 +430,7 @@ typedef void (*slab_fn)(const struct sense_s* d, long coil, const complex float* struct slab_walk { complex float* buf[2]; - long holds[2]; /* the first coil each buffer holds, or -1 */ + bart_dim_t holds[2]; /* the first coil each buffer holds, or -1 */ bool reverse; }; @@ -450,7 +450,7 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo { if (!assembled(d, ref)) { - for (long c = 0; c < d->coils; c += d->batch) + for (bart_dim_t c = 0; c < d->coils; c += d->batch) fn(d, c, d->maps + slab_at(d->map_slab_offset, c), d->map_strs, c + d->batch >= d->coils, ctx); return; @@ -462,7 +462,7 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo if (NULL == w->buf[0]) w->buf[0] = slab_buffer(d, ref); - const long* mstrs = d->slab_map_strs; + const bart_stride_t* mstrs = d->slab_map_strs; bool overlap = (1 < stream_count()) && (d->batch < d->coils); @@ -471,10 +471,10 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo overlap = overlap && (NULL != w->buf[1]); - long slabs = (d->coils + d->batch - 1) / d->batch; - long order[slabs]; + bart_dim_t slabs = (d->coils + d->batch - 1) / d->batch; + bart_dim_t order[slabs]; - for (long t = 0; t < slabs; t++) + for (bart_dim_t t = 0; t < slabs; t++) order[t] = (w->reverse ? slabs - 1 - t : t) * d->batch; /* The buffer the first slab is in, if the walk before left it there. */ @@ -487,9 +487,9 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo w->holds[0] = order[0]; } - for (long t = 0; t < slabs; t++) { + for (bart_dim_t t = 0; t < slabs; t++) { - long c = order[t]; + bart_dim_t c = order[t]; bool last = (t + 1 == slabs); if (!overlap) { @@ -504,7 +504,7 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo continue; } - long next = last ? -1 : order[t + 1]; + bart_dim_t next = last ? -1 : order[t + 1]; /* Armed here rather than once: BART forgets which level owns * the streams as soon as anything asks for one from below it, @@ -575,14 +575,14 @@ static bool crosses(const void* ptr) return !bartorch_on_device(ptr) && (0 <= bartorch_cuda_device()); } -static complex float* onto_card(const long dims[DIMS], const complex float* ptr, bool filled) +static complex float* onto_card(const bart_dim_t dims[DIMS], const complex float* ptr, bool filled) { #ifdef USE_CUDA if (crosses(ptr)) { complex float* on = md_alloc_gpu(DIMS, dims, CFL_SIZE); - if (filled && (0 != bartorch_cuda_copy_pageable(on, ptr, md_calc_size(DIMS, dims) * (long)CFL_SIZE))) + if (filled && (0 != bartorch_cuda_copy_pageable(on, ptr, md_calc_size(DIMS, dims) * (bart_stride_t)CFL_SIZE))) md_copy(DIMS, dims, on, ptr, CFL_SIZE); return on; @@ -593,12 +593,12 @@ static complex float* onto_card(const long dims[DIMS], const complex float* ptr, return (complex float*)ptr; } -static void off_card(const long dims[DIMS], complex float* ptr, complex float* on, bool filled) +static void off_card(const bart_dim_t dims[DIMS], complex float* ptr, complex float* on, bool filled) { if (on == ptr) return; - if (filled && (0 != bartorch_cuda_copy_pageable(ptr, on, md_calc_size(DIMS, dims) * (long)CFL_SIZE))) + if (filled && (0 != bartorch_cuda_copy_pageable(ptr, on, md_calc_size(DIMS, dims) * (bart_stride_t)CFL_SIZE))) md_copy(DIMS, dims, ptr, on, CFL_SIZE); md_free(on); @@ -619,10 +619,10 @@ struct slab_ctx { /* Where there are several items: the one a slab function works on, what * each does, and how far an item steps the source and the destination. */ - long item; + bart_dim_t item; slab_fn fn; - long src_step; - long dst_step; + bart_dim_t src_step; + bart_dim_t dst_step; }; /* The transform an item's slab function applies. */ @@ -636,38 +636,38 @@ static const struct linop_s* slab_of(const struct sense_s* d, const struct slab_ * coefficients -- steps over the planes, a position after another. */ static void planes_take(const struct sense_s* d, complex float* sub, const complex float* cim) { - long z = d->cim_dims[PHS2_DIM]; - long outer = md_calc_size(DIMS, d->cim_dims) / (z * d->plane_elems); + bart_dim_t z = d->cim_dims[PHS2_DIM]; + bart_dim_t outer = md_calc_size(DIMS, d->cim_dims) / (z * d->plane_elems); - for (long o = 0; o < outer; o++) - for (long j = 0; j < d->stack_count; j++) + for (bart_dim_t o = 0; o < outer; o++) + for (bart_dim_t j = 0; j < d->stack_count; j++) md_copy(1, MD_DIMS(d->plane_elems), sub + (o * d->stack_count + j) * d->plane_elems, cim + (o * z + d->stack_positions[j]) * d->plane_elems, CFL_SIZE); } static void planes_put(const struct sense_s* d, complex float* cim, const complex float* sub) { - long z = d->cim_dims[PHS2_DIM]; - long outer = md_calc_size(DIMS, d->cim_dims) / (z * d->plane_elems); + bart_dim_t z = d->cim_dims[PHS2_DIM]; + bart_dim_t outer = md_calc_size(DIMS, d->cim_dims) / (z * d->plane_elems); md_clear(DIMS, d->cim_dims, cim, CFL_SIZE); - for (long o = 0; o < outer; o++) - for (long j = 0; j < d->stack_count; j++) + for (bart_dim_t o = 0; o < outer; o++) + for (bart_dim_t j = 0; j < d->stack_count; j++) md_copy(1, MD_DIMS(d->plane_elems), cim + (o * z + d->stack_positions[j]) * d->plane_elems, sub + (o * d->stack_count + j) * d->plane_elems, CFL_SIZE); } /* Each item in turn under one slab of coils. */ -static void each_item(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void each_item(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { struct slab_ctx* c = _c; complex float* dst = c->dst; const complex float* src = c->src; - for (long t = 0; t < d->items; t++) { + for (bart_dim_t t = 0; t < d->items; t++) { c->item = t; c->dst = dst + t * c->dst_step; @@ -682,7 +682,7 @@ static void each_item(const struct sense_s* d, long coil, const complex float* m } /* A slab's samples into their place in the whole, and back out of it. */ -static void put_samples(const struct sense_s* d, long coil, complex float* dst, const complex float* out) +static void put_samples(const struct sense_s* d, bart_dim_t coil, complex float* dst, const complex float* out) { if (!d->coils_slowest) { @@ -695,15 +695,15 @@ static void put_samples(const struct sense_s* d, long coil, complex float* dst, * The destination is walked by the whole's own strides rather than by a * block's, so an axis the whole carries above the coils lands where it * belongs instead of inside the block. */ - long coil_step = d->out_strs[COIL_DIM] / (long)CFL_SIZE; + bart_dim_t coil_step = d->out_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE; - for (long b = 0; b < d->batch; b++) + for (bart_dim_t b = 0; b < d->batch; b++) md_copy2(DIMS, d->block_dims, d->full_out_strs, dst + slab_at(d->out_slab_offset, coil + b), d->out_strs, out + b * coil_step, CFL_SIZE); } -static void take_samples(const struct sense_s* d, long coil, complex float* out, const complex float* src) +static void take_samples(const struct sense_s* d, bart_dim_t coil, complex float* out, const complex float* src) { if (!d->coils_slowest) { @@ -712,15 +712,15 @@ static void take_samples(const struct sense_s* d, long coil, complex float* out, return; } - long coil_step = d->out_strs[COIL_DIM] / (long)CFL_SIZE; + bart_dim_t coil_step = d->out_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE; - for (long b = 0; b < d->batch; b++) + for (bart_dim_t b = 0; b < d->batch; b++) md_copy2(DIMS, d->block_dims, d->out_strs, out + b * coil_step, d->full_out_strs, src + slab_at(d->out_slab_offset, coil + b), CFL_SIZE); } -static void forward_slab(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void forward_slab(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -743,8 +743,8 @@ static void forward_slab(const struct sense_s* d, long coil, const complex float put_samples(d, coil, c->dst, c->out); } -static void adjoint_slab(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void adjoint_slab(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -767,8 +767,8 @@ static void adjoint_slab(const struct sense_s* d, long coil, const complex float /* The same for a sampled-only Cartesian transform on a card: the sensitivity * goes on as a coefficient is read into the transform and comes off as the * adjoint writes into the image, so no coil image is made. */ -static void forward_slab_gridded(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void forward_slab_gridded(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -778,8 +778,8 @@ static void forward_slab_gridded(const struct sense_s* d, long coil, const compl put_samples(d, coil, c->dst, c->out); } -static void adjoint_slab_gridded(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void adjoint_slab_gridded(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -789,8 +789,8 @@ static void adjoint_slab_gridded(const struct sense_s* d, long coil, const compl bartorch_grid_adjoint_sense(d->slab, c->dst, c->out, mstrs, map); } -static void normal_slab(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; (void)last; @@ -821,8 +821,8 @@ static void normal_slab(const struct sense_s* d, long coil, const complex float* * off into the answer. Clearing what the convolution accumulates into is a * pass over the coil images, which is what walking the sets outside costs -- * against the whole function crossing again, which is what it saves. */ -static void normal_slab_coset(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab_coset(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; struct slab_ctx* c = _c; @@ -854,8 +854,8 @@ static void normal_slab_coset(const struct sense_s* d, long coil, const complex * image the map would have been multiplied into nor the one the answer would * have landed in is ever made. At 256^3 over four coefficients each of those * is half a gigabyte. */ -static void normal_slab_folded(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab_folded(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; struct slab_ctx* c = _c; @@ -866,8 +866,8 @@ static void normal_slab_folded(const struct sense_s* d, long coil, const complex /* The same for a Cartesian transform: the sensitivity goes on as a * coefficient is read into the transform and comes off as it is written * into the answer, so here too no coil image is made. */ -static void normal_slab_gridded(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab_gridded(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; (void)last; @@ -879,14 +879,14 @@ static void normal_slab_gridded(const struct sense_s* d, long coil, const comple /* The terms of a contraction the slab loop applies: the image weight before the * sensitivities, the sample weight after the transform, the terms' samples * added up in `acc`. */ -static void terms_forward(const struct sense_s* d, const complex float* map, const long* mstrs, struct slab_ctx* c) +static void terms_forward(const struct sense_s* d, const complex float* map, const bart_stride_t* mstrs, struct slab_ctx* c) { const complex float* image = multiplace_read(d->term_image, c->src); const complex float* sample = multiplace_read(d->term_sample, c->src); md_clear(DIMS, d->out_dims, c->acc, CFL_SIZE); - for (long l = 0; l < d->terms; l++) { + for (bart_dim_t l = 0; l < d->terms; l++) { md_zmul2(DIMS, d->img_dims, d->img_strs, c->img, d->img_strs, c->src, d->term_image_strs, image + l * d->term_image_step + c->item * d->term_image_item_step); @@ -902,13 +902,13 @@ static void terms_forward(const struct sense_s* d, const complex float* map, con } } -static void terms_adjoint(const struct sense_s* d, const complex float* map, const long* mstrs, +static void terms_adjoint(const struct sense_s* d, const complex float* map, const bart_stride_t* mstrs, struct slab_ctx* c, const complex float* samples) { const complex float* image = multiplace_read(d->term_image, c->dst); const complex float* sample = multiplace_read(d->term_sample, c->dst); - for (long l = 0; l < d->terms; l++) { + for (bart_dim_t l = 0; l < d->terms; l++) { md_zmulc2(DIMS, d->out_dims, d->out_strs, c->out, d->out_strs, samples, d->term_sample_strs, sample + l * d->term_sample_step + c->item * d->term_sample_item_step); @@ -923,8 +923,8 @@ static void terms_adjoint(const struct sense_s* d, const complex float* map, con } } -static void forward_slab_terms(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void forward_slab_terms(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -933,8 +933,8 @@ static void forward_slab_terms(const struct sense_s* d, long coil, const complex put_samples(d, coil, c->dst, c->acc); } -static void adjoint_slab_terms(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void adjoint_slab_terms(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -943,8 +943,8 @@ static void adjoint_slab_terms(const struct sense_s* d, long coil, const complex terms_adjoint(d, map, mstrs, c, c->acc); } -static void normal_slab_terms(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab_terms(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; (void)last; @@ -1010,7 +1010,7 @@ static void sense_adjoint(const linop_data_t* _d, complex float* dst, const comp complex float* dst_on = onto_card(d->whole_img_dims, dst, false); /* The caller's pages are faulted in while the card works (cuda.c). */ - void* faulting = (dst_on != dst) ? bartorch_host_prefault_begin(dst, md_calc_size(DIMS, d->whole_img_dims) * (long)CFL_SIZE) : NULL; + void* faulting = (dst_on != dst) ? bartorch_host_prefault_begin(dst, md_calc_size(DIMS, d->whole_img_dims) * (bart_stride_t)CFL_SIZE) : NULL; bool terms = (0 != d->terms); bool gridded = !terms && d->fold && (1 == d->slab_dims[MAPS_DIM]) @@ -1088,7 +1088,7 @@ static void sense_normal(const linop_data_t* _d, complex float* dst, const compl && (0 != bartorch_grid_folds(d->slab, dst_on)); /* The caller's pages are faulted in while the card works (cuda.c). */ - void* faulting = (dst_on != dst) ? bartorch_host_prefault_begin(dst, md_calc_size(DIMS, d->whole_img_dims) * (long)CFL_SIZE) : NULL; + void* faulting = (dst_on != dst) ? bartorch_host_prefault_begin(dst, md_calc_size(DIMS, d->whole_img_dims) * (bart_stride_t)CFL_SIZE) : NULL; struct slab_ctx c = { @@ -1134,7 +1134,7 @@ static void sense_normal(const linop_data_t* _d, complex float* dst, const compl * the item rather than once for every coil. */ struct slab_walk walk = { { NULL, NULL }, { -1, -1 }, false }; - for (long t = 0; t < d->items; t++) { + for (bart_dim_t t = 0; t < d->items; t++) { const struct linop_s* slab = (NULL == d->item_slabs) ? d->slab : d->item_slabs[t]; @@ -1195,7 +1195,7 @@ static void sense_del(const linop_data_t* _d) } else { - for (long t = 0; t < d->items; t++) + for (bart_dim_t t = 0; t < d->items; t++) linop_free(d->item_slabs[t]); xfree(d->item_slabs); @@ -1213,10 +1213,10 @@ static void sense_del(const linop_data_t* _d) * They can when nothing else is laid out along them: a pattern or a basis * that varies across coils would have to be sliced with them, and the * sensitivities have to carry the coils the images do. */ -static bool sliceable(long batch, const long max_dims[DIMS], const long map_dims[DIMS], const long out_dims[DIMS], - unsigned long shared_img_flags) +static bool sliceable(bart_dim_t batch, const bart_dim_t max_dims[DIMS], const bart_dim_t map_dims[DIMS], const bart_dim_t out_dims[DIMS], + bart_flags_t shared_img_flags) { - long coils = max_dims[COIL_DIM]; + bart_dim_t coils = max_dims[COIL_DIM]; if ((0 == batch) || (coils < 2)) return false; @@ -1241,9 +1241,9 @@ static bool sliceable(long batch, const long max_dims[DIMS], const long map_dims * one asked for -- the one asked for whenever it divides the coils, and one * when nothing else does. */ -static long slab_size(long coils, long want) +static bart_dim_t slab_size(bart_dim_t coils, bart_dim_t want) { - for (long n = MIN(want, coils); n > 1; n--) + for (bart_dim_t n = MIN(want, coils); n > 1; n--) if (0 == coils % n) return n; @@ -1253,8 +1253,8 @@ static long slab_size(long coils, long want) /* The parts of the operator that do not depend on which transform it is. * `batch` and `fold` are the form's, so that nothing about one build is read * from the settings BART's own tools leave behind. */ -static struct sense_s* sense_slabs(long batch, bool fold, const long max_dims[DIMS], const long map_dims[DIMS], - const long out_dims[DIMS], unsigned long shared_img_flags, bool keep_sets) +static struct sense_s* sense_slabs(bart_dim_t batch, bool fold, const bart_dim_t max_dims[DIMS], const bart_dim_t map_dims[DIMS], + const bart_dim_t out_dims[DIMS], bart_flags_t shared_img_flags, bool keep_sets) { PTR_ALLOC(struct sense_s, d); SET_TYPEID(sense_s, d); @@ -1286,7 +1286,7 @@ static struct sense_s* sense_slabs(long batch, bool fold, const long max_dims[DI * something after the transform needs them: a k-space factor that * differs between sets is summed over on the far side instead, so the * coil images keep them and the transform runs once per set. */ - md_select_dims(DIMS, keep_sets ? ~0UL : ~MAPS_FLAG, d->cim_dims, d->slab_dims); + md_select_dims(DIMS, keep_sets ? ~UINT64_C(0) : ~MAPS_FLAG, d->cim_dims, d->slab_dims); md_select_dims(DIMS, ~COIL_FLAG & ~shared_img_flags, d->img_dims, max_dims); md_calc_strides(DIMS, d->cim_strs, d->cim_dims, CFL_SIZE); @@ -1303,7 +1303,7 @@ static struct sense_s* sense_slabs(long batch, bool fold, const long max_dims[DI md_calc_strides(DIMS, d->map_strs, map_dims, CFL_SIZE); d->map_slab_offset = d->map_strs[COIL_DIM]; - long slab_map_dims[DIMS]; + bart_dim_t slab_map_dims[DIMS]; md_copy_dims(DIMS, slab_map_dims, map_dims); slab_map_dims[COIL_DIM] = d->batch; md_calc_strides(DIMS, d->slab_map_strs, slab_map_dims, CFL_SIZE); @@ -1322,7 +1322,7 @@ static struct sense_s* sense_slabs(long batch, bool fold, const long max_dims[DI * than around it because one bank cannot serve every item. They are still * part of what a slab answers, so they stay in the block; what they must not * do is push the coils above them. */ -static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned long outer, const long* item_dims) +static void sense_output_from(struct sense_s* d, bool coils_slowest, bart_flags_t outer, const bart_dim_t* item_dims) { auto cod = linop_codomain(d->slab); @@ -1344,11 +1344,11 @@ static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned lo d->block_dims[COIL_DIM] = 1; /* A block is one item's; a coil holds the items one after another. */ - long whole_dims[DIMS]; + bart_dim_t whole_dims[DIMS]; md_copy_dims(DIMS, whole_dims, d->block_dims); if (NULL != item_dims) - md_max_dims(DIMS, ~0UL, whole_dims, whole_dims, item_dims); + md_max_dims(DIMS, ~UINT64_C(0), whole_dims, whole_dims, item_dims); d->item_sample_step = md_calc_size(DIMS, d->block_dims); @@ -1357,7 +1357,7 @@ static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned lo * where a block has nothing beyond it, a later one where encoding axes * lie there, and never above an outer axis, which the torch layout puts * slower than the coils. */ - long inner_dims[DIMS]; + bart_dim_t inner_dims[DIMS]; md_select_dims(DIMS, ~outer, inner_dims, whole_dims); int last = DIMS - 1; @@ -1372,7 +1372,7 @@ static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned lo /* The coils were placed above every inner axis; an outer one that is not * above them too would be read in the wrong order rather than refused. */ - unsigned long at_or_below = (1UL << (coil_axis + 1)) - 1UL; + bart_flags_t at_or_below = (UINT64_C(1) << (coil_axis + 1)) - 1; if (0 != (outer & md_nontriv_dims(DIMS, whole_dims) & at_or_below)) error("bartorch: a batch the sensitivities vary along lies slower than the coils, " @@ -1386,7 +1386,7 @@ static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned lo static struct linop_s* sense_operator(struct sense_s* d) { - debug_printf(DP_DEBUG1, "SENSE over %ld coils, %ld at a time\n", d->coils, d->batch); + debug_printf(DP_DEBUG1, "SENSE over %" PRId64 " coils, %" PRId64 " at a time\n", d->coils, d->batch); counted(BARTORCH_ENCODING_BUILT); @@ -1400,22 +1400,22 @@ static void chained(void) } /* y = F S x, on a grid. */ -struct linop_s* sense_init(unsigned long shared_img_flags, const long max_dims[DIMS], - unsigned long sens_flags, const complex float* sens) +struct linop_s* sense_init(bart_flags_t shared_img_flags, const bart_dim_t max_dims[DIMS], + bart_flags_t sens_flags, const complex float* sens) { - long map_dims[DIMS]; - long ksp_dims[DIMS]; + bart_dim_t map_dims[DIMS]; + bart_dim_t ksp_dims[DIMS]; md_select_dims(DIMS, sens_flags, map_dims, max_dims); md_select_dims(DIMS, ~MAPS_FLAG, ksp_dims, max_dims); - if (!sliceable((long)coil_batch, max_dims, map_dims, ksp_dims, shared_img_flags)) { + if (!sliceable((bart_dim_t)coil_batch, max_dims, map_dims, ksp_dims, shared_img_flags)) { chained(); return bart_sense_init(shared_img_flags, max_dims, sens_flags, sens); } - struct sense_s* d = sense_slabs((long)coil_batch, 0 != fold_maps, max_dims, map_dims, ksp_dims, shared_img_flags, false); + struct sense_s* d = sense_slabs((bart_dim_t)coil_batch, 0 != fold_maps, max_dims, map_dims, ksp_dims, shared_img_flags, false); /* The scaling and the modulation `maps_create` folds into the * sensitivities, kept here because the loop reads them many times. */ @@ -1424,29 +1424,29 @@ struct linop_s* sense_init(unsigned long shared_img_flags, const long max_dims[D fftmod(DIMS, map_dims, FFT_FLAGS, d->owned, d->owned); d->maps = d->owned; - long slab_ksp_dims[DIMS]; + bart_dim_t slab_ksp_dims[DIMS]; md_copy_dims(DIMS, slab_ksp_dims, ksp_dims); slab_ksp_dims[COIL_DIM] = d->batch; d->slab = linop_fft_create(DIMS, slab_ksp_dims, FFT_FLAGS); - sense_output_from(d, false, 0UL, NULL); + sense_output_from(d, false, 0, NULL); return sense_operator(d); } /* y = A S x, off one. */ -const struct linop_s* sense_nc_init(const long max_dims[DIMS], const long map_dims[DIMS], const complex float* maps, - const long ksp_dims[DIMS], - const long traj_dims[DIMS], const complex float* traj, const struct nufft_conf_s* _conf, - const long wgs_dims[DIMS], const complex float* weights, - const long basis_dims[DIMS], const complex float* basis, - const struct linop_s** fft_opp, unsigned long shared_img_dims) +const struct linop_s* sense_nc_init(const bart_dim_t max_dims[DIMS], const bart_dim_t map_dims[DIMS], const complex float* maps, + const bart_dim_t ksp_dims[DIMS], + const bart_dim_t traj_dims[DIMS], const complex float* traj, const struct nufft_conf_s* _conf, + const bart_dim_t wgs_dims[DIMS], const complex float* weights, + const bart_dim_t basis_dims[DIMS], const complex float* basis, + const struct linop_s** fft_opp, bart_flags_t shared_img_dims) { - long ksp_dims2[DIMS]; + bart_dim_t ksp_dims2[DIMS]; md_copy_dims(DIMS, ksp_dims2, ksp_dims); ksp_dims2[COEFF_DIM] = max_dims[COEFF_DIM]; - bool sliced = sliceable((long)coil_batch, max_dims, map_dims, ksp_dims2, shared_img_dims) + bool sliced = sliceable((bart_dim_t)coil_batch, max_dims, map_dims, ksp_dims2, shared_img_dims) && ((NULL == weights) || (1 == wgs_dims[COIL_DIM])) && ((NULL == basis) || (1 == basis_dims[COIL_DIM])); @@ -1457,9 +1457,9 @@ const struct linop_s* sense_nc_init(const long max_dims[DIMS], const long map_di wgs_dims, weights, basis_dims, basis, fft_opp, shared_img_dims); } - struct sense_s* d = sense_slabs((long)coil_batch, 0 != fold_maps, max_dims, map_dims, ksp_dims2, shared_img_dims, false); + struct sense_s* d = sense_slabs((bart_dim_t)coil_batch, 0 != fold_maps, max_dims, map_dims, ksp_dims2, shared_img_dims, false); - long slab_ksp_dims[DIMS]; + bart_dim_t slab_ksp_dims[DIMS]; md_copy_dims(DIMS, slab_ksp_dims, ksp_dims2); slab_ksp_dims[COIL_DIM] = d->batch; @@ -1468,7 +1468,7 @@ const struct linop_s* sense_nc_init(const long max_dims[DIMS], const long map_di (weights ? wgs_dims : NULL), weights, (basis ? basis_dims : NULL), basis, *_conf); - sense_output_from(d, false, 0UL, NULL); + sense_output_from(d, false, 0, NULL); /* The caller reads the point spread function off this and imports one * into it; a slab's transform carries the same one, because a point @@ -1482,7 +1482,7 @@ const struct linop_s* sense_nc_init(const long max_dims[DIMS], const long map_di /* Hold the sensitivities the way the caller has them: as maps the loop reads * where they lie, or as the kernels it inflates a slab at a time. */ -static void sense_hold(struct sense_s* d, const long sens_dims[DIMS], const complex float* sens, int kernels) +static void sense_hold(struct sense_s* d, const bart_dim_t sens_dims[DIMS], const complex float* sens, int kernels) { if (0 == kernels) { @@ -1511,18 +1511,18 @@ static void kernels_need_the_loop(void) * its pattern and basis and the normal that transforms only the axes the * pattern varies along, the same over a table of sampled phase encodes, and * the wave in either arrangement. */ -extern const struct linop_s* grid_transform_create(const long cim_dims[DIMS], - const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz); -extern const struct linop_s* grid_sampled_create(const long cim_dims[DIMS], long T, long S, int components, - const long* positions, const long bas_dims[DIMS], const complex float* basis, +extern const struct linop_s* grid_transform_create(const bart_dim_t cim_dims[DIMS], + const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz); +extern const struct linop_s* grid_sampled_create(const bart_dim_t cim_dims[DIMS], bart_dim_t T, bart_dim_t S, int components, + const bart_dim_t* positions, const bart_dim_t bas_dims[DIMS], const complex float* basis, int kspace_readout, int toeplitz); -extern const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, const complex float* psf, - int centred, const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz); -extern const struct linop_s* wave_sampled_create(const long dom_dims[DIMS], long wx, const complex float* psf, - int centred, long T, long S, int components, const long* positions, - const long bas_dims[DIMS], const complex float* basis, int toeplitz); +extern const struct linop_s* wave_transform_create(const bart_dim_t dom_dims[DIMS], bart_dim_t wx, const complex float* psf, + int centred, const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz); +extern const struct linop_s* wave_sampled_create(const bart_dim_t dom_dims[DIMS], bart_dim_t wx, const complex float* psf, + int centred, bart_dim_t T, bart_dim_t S, int components, const bart_dim_t* positions, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz); /* `slab` with the form's contraction around it, sum_l diag(b_l) slab * diag(c_l): each term a copy of the weights on the side they multiply, all @@ -1541,15 +1541,15 @@ static const struct linop_s* contracted(const struct bartorch_encoding* f, const const complex float* sample = f->segment_sample; const complex float* image = f->segment_image; - long sample_step = md_calc_size(DIMS, f->segment_sample_dims); - long image_step = md_calc_size(DIMS, f->segment_image_dims); + bart_dim_t sample_step = md_calc_size(DIMS, f->segment_sample_dims); + bart_dim_t image_step = md_calc_size(DIMS, f->segment_image_dims); - unsigned long sample_flags = md_nontriv_dims(DIMS, f->segment_sample_dims); - unsigned long image_flags = md_nontriv_dims(DIMS, f->segment_image_dims); + bart_flags_t sample_flags = md_nontriv_dims(DIMS, f->segment_sample_dims); + bart_flags_t image_flags = md_nontriv_dims(DIMS, f->segment_image_dims); const struct linop_s* sum = NULL; - for (long l = 0; l < f->segments; l++) { + for (bart_dim_t l = 0; l < f->segments; l++) { const struct linop_s* term = linop_chain_FF(linop_chain_FF( linop_cdiag_create(DIMS, dom->dims, image_flags, image + l * image_step), @@ -1584,9 +1584,9 @@ static void hold_terms(struct sense_s* d, const struct bartorch_encoding* f) if (NULL != f->item_dims) { - unsigned long item_flags = md_nontriv_dims(DIMS, f->item_dims); - unsigned long on_image = item_flags & md_nontriv_dims(DIMS, f->segment_image_dims); - unsigned long on_sample = item_flags & md_nontriv_dims(DIMS, f->segment_sample_dims); + bart_flags_t item_flags = md_nontriv_dims(DIMS, f->item_dims); + bart_flags_t on_image = item_flags & md_nontriv_dims(DIMS, f->segment_image_dims); + bart_flags_t on_sample = item_flags & md_nontriv_dims(DIMS, f->segment_sample_dims); if (((0 != on_image) && (item_flags != on_image)) || ((0 != on_sample) && (item_flags != on_sample))) error("bartorch: weights before the sensitivities vary along every item axis or none\n"); @@ -1598,8 +1598,8 @@ static void hold_terms(struct sense_s* d, const struct bartorch_encoding* f) d->term_sample_item_step = d->term_sample_step / md_calc_size(DIMS, f->item_dims); } - long image[1] = { d->terms * d->term_image_step }; - long sample[1] = { d->terms * d->term_sample_step }; + bart_dim_t image[1] = { d->terms * d->term_image_step }; + bart_dim_t sample[1] = { d->terms * d->term_sample_step }; d->term_image = multiplace_move(1, image, CFL_SIZE, f->segment_image); d->term_sample = multiplace_move(1, sample, CFL_SIZE, f->segment_sample); @@ -1633,7 +1633,7 @@ static const struct linop_s* summed_over_sets(const struct bartorch_encoding* f, * it -- the coils, the slab loop, the streaming, the contraction -- is the * same code whichever transform it is. */ -static const struct linop_s* form_transform(const struct bartorch_encoding* f, const long cim_dims[DIMS], +static const struct linop_s* form_transform(const struct bartorch_encoding* f, const bart_dim_t cim_dims[DIMS], const struct nufft_conf_s* conf) { switch (f->transform) { @@ -1688,8 +1688,8 @@ static const struct linop_s* form_transform(const struct bartorch_encoding* f, c * the coils, or one whose coils the loop cannot slice, and the counter says * so rather than leaving it to be guessed from a timing. */ -static const struct linop_s* form_chain(const struct bartorch_encoding* f, const long max_dims[DIMS], - const long map_dims[DIMS], const long cim_dims[DIMS], const struct nufft_conf_s* conf) +static const struct linop_s* form_chain(const struct bartorch_encoding* f, const bart_dim_t max_dims[DIMS], + const bart_dim_t map_dims[DIMS], const bart_dim_t cim_dims[DIMS], const struct nufft_conf_s* conf) { if (0 != f->kernels) kernels_need_the_loop(); @@ -1698,19 +1698,19 @@ static const struct linop_s* form_chain(const struct bartorch_encoding* f, const if (BARTORCH_ENCODING_NUFFT == f->transform) return bart_sense_nc_init(max_dims, map_dims, f->sens, f->ksp_dims, f->traj_dims, f->traj, conf, - f->wgh_dims, f->weights, f->bas_dims, f->basis, NULL, 0UL); + f->wgh_dims, f->weights, f->bas_dims, f->basis, NULL, 0); if (0 != f->modulated) { /* The flags `pics` gives it, so that what comes back is the * operator the tool builds and not one like it -- which is * what a caller asking for this convention is after. */ - unsigned long map_flags = FFT_FLAGS | SENS_FLAGS | md_nontriv_dims(DIMS, f->sens_dims); + bart_flags_t map_flags = FFT_FLAGS | SENS_FLAGS | md_nontriv_dims(DIMS, f->sens_dims); - return bart_sense_init(0UL, max_dims, map_flags, f->sens); + return bart_sense_init(0, max_dims, map_flags, f->sens); } - long img_dims[DIMS]; + bart_dim_t img_dims[DIMS]; md_select_dims(DIMS, ~COIL_FLAG, img_dims, max_dims); struct linop_s* coils = linop_fmac_dims_create(DIMS, cim_dims, img_dims, @@ -1734,7 +1734,7 @@ static const struct linop_s* form_chain(const struct bartorch_encoding* f, const */ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* f) { - const long* max_dims = f->max_dims; + const bart_dim_t* max_dims = f->max_dims; if ((BARTORCH_ENCODING_FFT != f->transform) && (0 != f->modulated)) error("bartorch: the modulated convention is a grid transform's; " @@ -1759,15 +1759,15 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* * of the image's, and the samples carry it too: the operator holds every * item of it rather than being applied once per item, because one bank * does not serve them all. */ - unsigned long outer_flags = (0 <= f->batch_dim) ? MD_BIT(f->batch_dim) : 0UL; + bart_flags_t outer_flags = (0 <= f->batch_dim) ? MD_BIT(f->batch_dim) : 0; - long map_dims[DIMS]; + bart_dim_t map_dims[DIMS]; md_select_dims(DIMS, FFT_FLAGS | COIL_FLAG | MAPS_FLAG | outer_flags, map_dims, max_dims); /* What the transform is asked for. Off a grid the samples are the * form's own, with the coefficients a basis contracts still on them; * on one they are the image's axes without the sets. */ - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; if (BARTORCH_ENCODING_NUFFT == f->transform) { @@ -1776,14 +1776,14 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* } else { - md_select_dims(DIMS, (NULL != f->slice) ? ~0UL : ~MAPS_FLAG, out_dims, max_dims); + md_select_dims(DIMS, (NULL != f->slice) ? ~UINT64_C(0) : ~MAPS_FLAG, out_dims, max_dims); } /* Items with a trajectory of their own: the operator is built for one, * and each gets its own transform over its part of the trajectory. */ - long items = 1; - long item_max_dims[DIMS]; - long item_out_dims[DIMS]; + bart_dim_t items = 1; + bart_dim_t item_max_dims[DIMS]; + bart_dim_t item_out_dims[DIMS]; md_copy_dims(DIMS, item_max_dims, max_dims); md_copy_dims(DIMS, item_out_dims, out_dims); @@ -1813,7 +1813,7 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* /* A k-space factor laid out along the coils would have to be sliced * with them, which the loop cannot do. */ - bool sliced = sliceable((long)f->coil_batch, max_dims, map_dims, out_dims, 0UL) + bool sliced = sliceable((bart_dim_t)f->coil_batch, max_dims, map_dims, out_dims, 0) && ((NULL == f->weights) || (1 == f->wgh_dims[COIL_DIM])) && ((NULL == f->basis) || (1 == f->bas_dims[COIL_DIM])); @@ -1826,8 +1826,8 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* return form_chain(f, max_dims, map_dims, out_dims, &conf); } - struct sense_s* d = sense_slabs((long)f->coil_batch, 0 != f->fold_maps, - max_dims, map_dims, out_dims, 0UL, NULL != f->slice); + struct sense_s* d = sense_slabs((bart_dim_t)f->coil_batch, 0 != f->fold_maps, + max_dims, map_dims, out_dims, 0, NULL != f->slice); sense_hold(d, f->sens_dims, f->sens, f->kernels); @@ -1851,7 +1851,7 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* "done to a kernel; ask for the centred convention, or inflate the " "kernels first\n"); - if (!md_check_equal_dims(DIMS, map_dims, f->sens_dims, ~0UL)) + if (!md_check_equal_dims(DIMS, map_dims, f->sens_dims, ~UINT64_C(0))) error("bartorch: the modulation folded into the sensitivities is the grid's, so " "this convention needs a bank on the grid rather than one broadcast " "onto it\n"); @@ -1865,7 +1865,7 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* /* Off a grid the transform is asked for a slab of samples; on one it * works from the slab of coil images and says for itself what comes * back, because a basis contracts its coefficients away. */ - long slab_ksp_dims[DIMS]; + bart_dim_t slab_ksp_dims[DIMS]; md_copy_dims(DIMS, slab_ksp_dims, out_dims); slab_ksp_dims[COIL_DIM] = d->batch; @@ -1874,13 +1874,13 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* * and one position's shots where the shots were. Under a slab of one coil * and no sets that is the same memory as the layout around it, so the * relabelling copies nothing. */ - long slab_cim_dims[DIMS]; + bart_dim_t slab_cim_dims[DIMS]; md_copy_dims(DIMS, slab_cim_dims, d->cim_dims); if (d->stacked) { - long z = d->cim_dims[PHS2_DIM]; - long blocks = (NULL == f->stack_positions) ? z : f->stack_count; + bart_dim_t z = d->cim_dims[PHS2_DIM]; + bart_dim_t blocks = (NULL == f->stack_positions) ? z : f->stack_count; if ((1 != d->batch) || (1 != d->cim_dims[MAPS_DIM]) || (1 != slab_ksp_dims[MAPS_DIM]) || (blocks < 1) || (0 != slab_ksp_dims[PHS2_DIM] % blocks) @@ -1896,13 +1896,13 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* if (NULL != f->stack_positions) { d->stack_count = blocks; - d->stack_positions = xmalloc((size_t)blocks * sizeof(long)); + d->stack_positions = xmalloc((size_t)blocks * sizeof(bart_dim_t)); d->plane_elems = d->cim_dims[READ_DIM] * d->cim_dims[PHS1_DIM]; - for (long j = 0; j < blocks; j++) { + for (bart_dim_t j = 0; j < blocks; j++) { if ((f->stack_positions[j] < 0) || (f->stack_positions[j] >= z)) - error("bartorch: a stack's position %ld is off a z grid of %ld\n", + error("bartorch: a stack's position %" PRId64 " is off a z grid of %" PRId64 "\n", f->stack_positions[j], z); d->stack_positions[j] = f->stack_positions[j]; @@ -1930,13 +1930,13 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* /* An item's part of the trajectory and of the weights is contiguous: * the item axes are the slowest either array has. */ - long trj_dims[DIMS]; - long trj_strs[DIMS]; + bart_dim_t trj_dims[DIMS]; + bart_stride_t trj_strs[DIMS]; md_select_dims(DIMS, ~md_nontriv_dims(DIMS, f->item_dims), trj_dims, f->traj_dims); md_calc_strides(DIMS, trj_strs, f->traj_dims, CFL_SIZE); - long wgh_dims[DIMS]; - long wgh_strs[DIMS]; + bart_dim_t wgh_dims[DIMS]; + bart_stride_t wgh_strs[DIMS]; if (NULL != f->weights) { @@ -1946,12 +1946,12 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* /* So is an item's part of a term's segment weights, laid out after the * term the same way. */ - unsigned long item_flags = md_nontriv_dims(DIMS, f->item_dims); + bart_flags_t item_flags = md_nontriv_dims(DIMS, f->item_dims); - long seg_sample_dims[DIMS]; - long seg_image_dims[DIMS]; - long seg_sample_strs[DIMS]; - long seg_image_strs[DIMS]; + bart_dim_t seg_sample_dims[DIMS]; + bart_dim_t seg_image_dims[DIMS]; + bart_stride_t seg_sample_strs[DIMS]; + bart_stride_t seg_image_strs[DIMS]; if (0 != f->segments) { @@ -1964,10 +1964,10 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* d->item_slabs = xmalloc((size_t)items * sizeof(d->item_slabs[0])); d->item_image_step = md_calc_size(DIMS, d->img_dims); - long pos[DIMS]; + bart_dim_t pos[DIMS]; md_set_dims(DIMS, pos, 0); - for (long t = 0; t < items; t++) { + for (bart_dim_t t = 0; t < items; t++) { struct bartorch_encoding item = slab; @@ -1987,32 +1987,32 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* * them itself. */ if ((0 != f->segments) && !before_maps) { - long step_sample = md_calc_size(DIMS, f->segment_sample_dims); - long step_image = md_calc_size(DIMS, f->segment_image_dims); + bart_dim_t step_sample = md_calc_size(DIMS, f->segment_sample_dims); + bart_dim_t step_image = md_calc_size(DIMS, f->segment_image_dims); item.segment_sample_dims = seg_sample_dims; item.segment_image_dims = seg_image_dims; /* The terms stay contiguous, so an item's weights are copied * out term by term into arrays of their own. */ - long one_sample = md_calc_size(DIMS, seg_sample_dims); - long one_image = md_calc_size(DIMS, seg_image_dims); + bart_dim_t one_sample = md_calc_size(DIMS, seg_sample_dims); + bart_dim_t one_image = md_calc_size(DIMS, seg_image_dims); complex float* sample = md_alloc(1, MD_DIMS(f->segments * one_sample), CFL_SIZE); complex float* image = md_alloc(1, MD_DIMS(f->segments * one_image), CFL_SIZE); - for (long l = 0; l < f->segments; l++) { + for (bart_dim_t l = 0; l < f->segments; l++) { md_copy2(DIMS, seg_sample_dims, MD_STRIDES(DIMS, seg_sample_dims, CFL_SIZE), sample + l * one_sample, seg_sample_strs, (const complex float*)f->segment_sample + l * step_sample - + md_calc_offset(DIMS, seg_sample_strs, pos) / (long)CFL_SIZE, + + md_calc_offset(DIMS, seg_sample_strs, pos) / (bart_stride_t)CFL_SIZE, CFL_SIZE); md_copy2(DIMS, seg_image_dims, MD_STRIDES(DIMS, seg_image_dims, CFL_SIZE), image + l * one_image, seg_image_strs, (const complex float*)f->segment_image + l * step_image - + md_calc_offset(DIMS, seg_image_strs, pos) / (long)CFL_SIZE, + + md_calc_offset(DIMS, seg_image_strs, pos) / (bart_stride_t)CFL_SIZE, CFL_SIZE); } @@ -2027,7 +2027,7 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* d->item_slabs[t] = transform; - md_next(DIMS, f->item_dims, ~0UL, pos); + md_next(DIMS, f->item_dims, ~UINT64_C(0), pos); } d->slab = d->item_slabs[0]; diff --git a/src/csrc/substitute/fft.cpp b/src/csrc/substitute/fft.cpp index e3902a4a..4ab08d2c 100644 --- a/src/csrc/substitute/fft.cpp +++ b/src/csrc/substitute/fft.cpp @@ -23,8 +23,10 @@ * pocketfft's. */ #include +#include #include #include +#include #include #include #include @@ -37,11 +39,26 @@ namespace { -/* DFTI, as much of it as this needs. MKL_LONG is `long` on the platforms - * that have MKL at all, and the two calls that take a variable argument are - * declared that way so the ABI is the one MKL was compiled with. */ +/* DFTI, as much of it as this needs. MKL_LONG is `long` in MKL's LP64 + * interface, which is the one every source of DFTI here exports: 64 bits + * on Linux and macOS, 32 on Windows. So a length, a stride or a distance + * DFTI is handed has to fit it, and a plan with one that does not is + * walked here or left to pocketfft, whose sizes are size_t and ptrdiff_t. + * The two calls that take a variable argument are declared that way so + * the ABI is the one MKL was compiled with. */ typedef long mkl_long; +bool fits_mkl_long(ptrdiff_t v) +{ + return (v >= (ptrdiff_t)std::numeric_limits::min()) + && (v <= (ptrdiff_t)std::numeric_limits::max()); +} + +bool fits_mkl_long(const fftwf_iodim64& d) +{ + return fits_mkl_long(d.n) && fits_mkl_long(d.is) && fits_mkl_long(d.os); +} + enum { DFTI_COMPLEX = 32, DFTI_NUMBER_OF_TRANSFORMS = 7, @@ -73,11 +90,11 @@ bool dfti_ready() && (NULL != dfti.forward) && (NULL != dfti.backward) && (NULL != dfti.release); } -std::atomic g_planned[2]; /* by MKL, by pocketfft */ +std::atomic g_planned[2]; /* by MKL, by pocketfft */ /* One loop dimension, in elements. */ struct loop { - long n, is, os; + ptrdiff_t n, is, os; }; struct plan_s { @@ -90,7 +107,8 @@ struct plan_s { bool forward; /* What DFTI was given, when it took it: the transformed axes and at - * most one loop axis. `outer` is what is left to walk. */ + * most one loop axis. `outer` is what is left to walk. A rank of + * zero is a plan DFTI is not given. */ int rank = 0; std::vector lengths; std::vector strides_in; /* displacement first, then one per length */ @@ -190,7 +208,7 @@ void dfti_walk(const plan_s& p, void* desc, size_t level, const std::complexrank = rank; + bool fits = true; + + for (int i = 0; i < rank; i++) + fits = fits && fits_mkl_long(dims[i]); + + p->rank = fits ? rank : 0; p->strides_in.push_back(0); p->strides_out.push_back(0); - for (int i = 0; i < rank; i++) { + for (int i = 0; i < p->rank; i++) { - p->lengths.push_back(dims[i].n); - p->strides_in.push_back(dims[i].is); - p->strides_out.push_back(dims[i].os); + p->lengths.push_back((mkl_long)dims[i].n); + p->strides_in.push_back((mkl_long)dims[i].is); + p->strides_out.push_back((mkl_long)dims[i].os); } + /* The loop axis DFTI takes has to fit it as well; one that does not + * is walked with the rest. */ int longest = -1; for (int i = 0; i < howmany_rank; i++) - if ((howmany_dims[i].n > 1) && ((longest < 0) || (howmany_dims[i].n > howmany_dims[longest].n))) + if ((howmany_dims[i].n > 1) && fits_mkl_long(howmany_dims[i]) + && ((longest < 0) || (howmany_dims[i].n > howmany_dims[longest].n))) longest = i; if (longest >= 0) diff --git a/src/csrc/substitute/finufft.c b/src/csrc/substitute/finufft.c index 1cf8dd4d..987480e0 100644 --- a/src/csrc/substitute/finufft.c +++ b/src/csrc/substitute/finufft.c @@ -187,7 +187,7 @@ struct bartorch_fi_plan { * them is gone. That is a property worth testing, and one RSS cannot be read * for: FINUFFT's own multithreaded execute retains a kilobyte per thread per * call, which any measurement of the process would drown this in. */ -static long fi_live_plans; +static int64_t fi_live_plans; /* `spread_only` asks FINUFFT for the spreading alone -- no transform, no * deapodisation -- which puts the kernel's own footprint on the grid. That is @@ -253,7 +253,7 @@ int bartorch_finufft_plan(int device, int type, int dim, const int64_t n_modes[3 /* A plan keeps the points by pointer rather than copying them, so the arrays * given here have to outlive it. */ -int bartorch_finufft_setpts(void* plan, long M, float* x, float* y, float* z) +int bartorch_finufft_setpts(void* plan, int64_t M, float* x, float* y, float* z) { const struct bartorch_fi_plan* p = plan; @@ -283,7 +283,7 @@ void bartorch_finufft_free(void* plan) fi_live_plans--; } -long bartorch_finufft_live_plans(void) +int64_t bartorch_finufft_live_plans(void) { return fi_live_plans; } diff --git a/src/csrc/substitute/nufft_finufft.c b/src/csrc/substitute/nufft_finufft.c index e0c4472e..d97089ba 100644 --- a/src/csrc/substitute/nufft_finufft.c +++ b/src/csrc/substitute/nufft_finufft.c @@ -58,61 +58,61 @@ struct bartorch_paired; #include "noncart/gpu_grid.h" /* csrc/kernels.cu: the set's phase and the coil's sensitivity in one pass. */ -extern void bartorch_cuda_phase_map_in(int N, const long dims[], const float shift[3], float scale, +extern void bartorch_cuda_phase_map_in(int N, const bart_dim_t dims[], const float shift[3], float scale, complex float* dst, const complex float* src, const complex float* map); -extern void bartorch_cuda_phase_map_out(int N, const long dims[], const float shift[3], float scale, +extern void bartorch_cuda_phase_map_out(int N, const bart_dim_t dims[], const float shift[3], float scale, complex float* dst, const complex float* src, const complex float* map); -extern void bartorch_cuda_gather(long V, const unsigned int* mask, const int* prefix, complex float* dst, const complex float* src); -extern int bartorch_cuda_contract_upper_real(long L, int R, complex float* bank, const float* mat); -extern int bartorch_cuda_contract_upper_real_bf16(long L, int R, complex float* bank, const void* mat); -extern void bartorch_cuda_scatter(long V, const unsigned int* mask, const int* prefix, complex float* dst, const complex float* src); +extern void bartorch_cuda_gather(bart_dim_t V, const unsigned int* mask, const int* prefix, complex float* dst, const complex float* src); +extern int bartorch_cuda_contract_upper_real(bart_dim_t L, int R, complex float* bank, const float* mat); +extern int bartorch_cuda_contract_upper_real_bf16(bart_dim_t L, int R, complex float* bank, const void* mat); +extern void bartorch_cuda_scatter(bart_dim_t V, const unsigned int* mask, const int* prefix, complex float* dst, const complex float* src); -extern struct bartorch_cb_fft* bartorch_cb_fft_create(const long dims[3]); +extern struct bartorch_cb_fft* bartorch_cb_fft_create(const bart_dim_t dims[3]); extern void bartorch_cb_fft_free(struct bartorch_cb_fft* p); -extern void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const long dims[], +extern void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, complex float* bank, complex float* volume, const complex float* src, const complex float* map); -extern void bartorch_cb_fft_inverse(struct bartorch_cb_fft* p, int N, const long dims[], +extern void bartorch_cb_fft_inverse(struct bartorch_cb_fft* p, int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, complex float* dst, complex float* volume, const complex float* bank, const complex float* map); #endif #ifdef BARTORCH_PAIRED -extern struct bartorch_paired* bartorch_paired_create(const long dims[3], int coeffs, int sets, const float (*shifts)[3]); +extern struct bartorch_paired* bartorch_paired_create(const bart_dim_t dims[3], int coeffs, int sets, const float (*shifts)[3]); extern void bartorch_paired_free(struct bartorch_paired* p); extern void bartorch_paired_in(const struct bartorch_paired* p, int k, const complex float* src, const complex float* map, complex float* scratch); extern void bartorch_paired_fused(const struct bartorch_paired* p, int k, complex float* scratch, - const void* psf0, const void* psf1, int bf16, const unsigned int* mask, const int* prefix, long L); + const void* psf0, const void* psf1, int bf16, const unsigned int* mask, const int* prefix, bart_dim_t L); extern void bartorch_paired_back(const struct bartorch_paired* p, int k, complex float* dst, const complex float* map, complex float* scratch); #endif #include "include/bartorch.h" -extern struct linop_s* bart_nufft_create2(int N, const long ksp_dims[N], const long cim_dims[N], const long traj_dims[N], const complex float* traj, const long wgh_dims[N], const complex float* weights, const long bas_dims[N], const complex float* basis, struct nufft_conf_s conf); -extern int bart_nufft_get_psf_dims(const struct linop_s* nufft, int N, long psf_dims[N]); -extern void bart_nufft_get_psf(const struct linop_s* nufft, int N, const long psf_dims[N], complex float* psf); -extern void bart_nufft_get_psf2(const struct linop_s* nufft, int N, const long psf_dims[N], const long psf_strs[N], complex float* psf); -extern void bart_nufft_update_psf(const struct linop_s* nufft, int ND, const long psf_dims[ND], const complex float* psf); -extern void bart_nufft_update_psf2(const struct linop_s* nufft, int ND, const long psf_dims[ND], const long psf_strs[ND], const complex float* psf); -extern void bart_nufft_update_traj(const struct linop_s* nufft, int N, const long trj_dims[N], const complex float* traj, const long wgh_dims[N], const complex float* weights, const long bas_dims[N], const complex float* basis); +extern struct linop_s* bart_nufft_create2(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], const bart_dim_t traj_dims[N], const complex float* traj, const bart_dim_t wgh_dims[N], const complex float* weights, const bart_dim_t bas_dims[N], const complex float* basis, struct nufft_conf_s conf); +extern int bart_nufft_get_psf_dims(const struct linop_s* nufft, int N, bart_dim_t psf_dims[N]); +extern void bart_nufft_get_psf(const struct linop_s* nufft, int N, const bart_dim_t psf_dims[N], complex float* psf); +extern void bart_nufft_get_psf2(const struct linop_s* nufft, int N, const bart_dim_t psf_dims[N], const bart_stride_t psf_strs[N], complex float* psf); +extern void bart_nufft_update_psf(const struct linop_s* nufft, int ND, const bart_dim_t psf_dims[ND], const complex float* psf); +extern void bart_nufft_update_psf2(const struct linop_s* nufft, int ND, const bart_dim_t psf_dims[ND], const bart_stride_t psf_strs[ND], const complex float* psf); +extern void bart_nufft_update_traj(const struct linop_s* nufft, int N, const bart_dim_t trj_dims[N], const complex float* traj, const bart_dim_t wgh_dims[N], const complex float* weights, const bart_dim_t bas_dims[N], const complex float* basis); extern const struct operator_s* bart_nufft_precond_create(const struct linop_s* nufft_op); /* Provided by psf.c, which computes the function this convolves with. */ -extern void bartorch_psf_shift(int NS, float shift[NS], int N, const long factors[N], int idx); -extern complex float* bartorch_psf_to_host(int N, const long psf_dims[N + 1], unsigned long flags, - const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, - const long wgh_dims[N + 1], const complex float* weights, +extern void bartorch_psf_shift(int NS, float shift[NS], int N, const bart_dim_t factors[N], int idx); +extern complex float* bartorch_psf_to_host(int N, const bart_dim_t psf_dims[N + 1], bart_flags_t flags, + const bart_dim_t trj_dims[N + 1], const complex float* traj, + const bart_dim_t bas_dims[N + 1], const complex float* basis, + const bart_dim_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag, - const long com_dims[N + 1], const long* idx, - const long com_psf_dims[N + 1], const long com_psf_dims3[N + 1], int real); + const bart_dim_t com_dims[N + 1], const bart_dim_t* idx, + const bart_dim_t com_psf_dims[N + 1], const bart_dim_t com_psf_dims3[N + 1], int real); /* Provided by finufft.c, which owns the FINUFFT entry points. */ extern int bartorch_finufft_plan(int device, int type, int dim, const int64_t n_modes[3], int ntrans, int isign, double eps, double upsampling, int spread_only, void** plan); -extern int bartorch_finufft_setpts(void* plan, long M, float* x, float* y, float* z); +extern int bartorch_finufft_setpts(void* plan, bart_dim_t M, float* x, float* y, float* z); extern int bartorch_finufft_exec(void* plan, complex float* c, complex float* f); extern void bartorch_finufft_free(void* plan); extern double bartorch_finufft_tolerance(void); @@ -127,7 +127,7 @@ extern double bartorch_finufft_upsampling(void); * pairs of sets convolved together by the pair kernels; and the functions * kept in bfloat16. */ enum { TP_PSF, TP_PAIR, TP_COMPRESSED, TP_CALLBACKS, TP_REAL, TP_PAIRED, TP_BF16, TP_COUNTERS }; -static long toeplitz_counters[TP_COUNTERS]; +static bart_dim_t toeplitz_counters[TP_COUNTERS]; /* Building a point spread function needs a transform of its own, and that * transform is nobody's normal: it is asked for one adjoint and freed. While @@ -135,7 +135,7 @@ static long toeplitz_counters[TP_COUNTERS]; * answered a normal either way. */ static _Thread_local int making_psf; -long bartorch_toeplitz_counter(int which) +int64_t bartorch_toeplitz_counter(int which) { return ((0 <= which) && (which < TP_COUNTERS)) ? toeplitz_counters[which] : -1; } @@ -173,7 +173,7 @@ struct fi_side { * transforms rather than inside one, so there it keeps them. */ int ntrans; - long executes; + bart_dim_t executes; }; struct nufft_fi_s { @@ -200,9 +200,9 @@ struct nufft_fi_s { double eps; double upsampling; - long samples; - long batch; - long image_elements; + bart_dim_t samples; + bart_dim_t batch; + bart_dim_t image_elements; float scale; /* out_dims is k-space as the caller sees it -- what BART calls out_dims, @@ -222,22 +222,22 @@ struct nufft_fi_s { */ int N; bool needs_tmp; - long* cim_dims; - long* out_dims; - long* out_strs; - long* grd_dims; - long* trf_strs; - long* wgh_dims; - long* wgh_strs; - long* bas_dims; - long* trj_dims; - long* ksp_dims; + bart_dim_t* cim_dims; + bart_dim_t* out_dims; + bart_stride_t* out_strs; + bart_dim_t* grd_dims; + bart_stride_t* trf_strs; + bart_dim_t* wgh_dims; + bart_stride_t* wgh_strs; + bart_dim_t* bas_dims; + bart_dim_t* trj_dims; + bart_dim_t* ksp_dims; /* What the normal is built from, for a trajectory that arrives after the * operator: a point spread function is over one, so there is none to * convolve with until there is a trajectory to make it from. */ struct nufft_conf_s conf; - long* bas_strs; + bart_stride_t* bas_strs; /* The function the normal convolves with, kept where the card is not. * @@ -259,7 +259,7 @@ struct nufft_fi_s { int slot_pending; /* a slot the card has not yet been held for, or -1 */ int coset; /* the set in it */ bool psf_registered; /* the host copy is page-locked */ - long psf_coset; /* elements in one set of the function */ + bart_dim_t psf_coset; /* elements in one set of the function */ size_t psf_size; /* a real function is stored as floats */ int cosets; int unit; /* sets in a slot: one, or the two of a pair */ @@ -298,15 +298,15 @@ static DEF_TYPEID(nufft_fi_s); * A trajectory can arrive after the operator: `nlinv` and the network models * build theirs against dimensions alone and fill it in with * `nufft_update_traj` once there is one. */ -static void install_traj(struct nufft_fi_s* d, const long traj_dims[], const complex float* traj) +static void install_traj(struct nufft_fi_s* d, const bart_dim_t traj_dims[], const complex float* traj) { int N = d->N; - long one_dims[N]; - long one_strs[N]; - long trj_strs[N]; + bart_dim_t one_dims[N]; + bart_stride_t one_strs[N]; + bart_stride_t trj_strs[N]; - md_select_dims(N, ~1UL, one_dims, traj_dims); + md_select_dims(N, ~UINT64_C(1), one_dims, traj_dims); md_calc_strides(N, one_strs, one_dims, CFL_SIZE); md_calc_strides(N, trj_strs, traj_dims, CFL_SIZE); @@ -329,7 +329,7 @@ static void install_traj(struct nufft_fi_s* d, const long traj_dims[], const com /* Memory on one side of the bus. A device only exists in a CUDA build, and * `device` is never set without one: `bartorch_on_device` says no, and * `bart_use_gpu` stays false. */ -static void* alloc_on(int device, int N, const long dims[N], size_t size) +static void* alloc_on(int device, int N, const bart_dim_t dims[N], size_t size) { #ifdef USE_CUDA if (device) @@ -381,7 +381,7 @@ static int side_build(struct nufft_fi_s* d, int which) return 12; } - long one[1] = { d->samples }; + bart_dim_t one[1] = { d->samples }; for (int i = 0; i < d->dim; i++) { @@ -425,7 +425,7 @@ static int side_retarget(struct nufft_fi_s* d, int which) if (NULL == s->forward_plan) return 0; - long one[1] = { d->samples }; + bart_dim_t one[1] = { d->samples }; for (int i = 0; i < d->dim; i++) md_copy(1, one, s->coord[i], d->radians[i], FL_SIZE); @@ -477,7 +477,7 @@ static complex float* transform_buffer(const struct nufft_fi_s* d, const void* r if (!d->needs_tmp) return NULL; - long dims[1] = { d->samples * d->batch }; + bart_dim_t dims[1] = { d->samples * d->batch }; return md_alloc_sameplace(1, dims, CFL_SIZE, ref); } @@ -502,7 +502,7 @@ static void nufft_fi_forward(const linop_data_t* _d, complex float* dst, const c int ret = 0; - for (long i = 0; (0 == ret) && (i < s->executes); i++) + for (bart_dim_t i = 0; (0 == ret) && (i < s->executes); i++) ret = bartorch_finufft_exec(s->forward_plan, out + i * s->ntrans * d->samples, (complex float*)src + i * s->ntrans * d->image_elements); @@ -573,7 +573,7 @@ static void nufft_fi_adjoint(const linop_data_t* _d, complex float* dst, const c int ret = 0; - for (long i = 0; (0 == ret) && (i < s->executes); i++) + for (bart_dim_t i = 0; (0 == ret) && (i < s->executes); i++) ret = bartorch_finufft_exec(s->adjoint_plan, (complex float*)src + i * s->ntrans * d->samples, dst + i * s->ntrans * d->image_elements); @@ -726,7 +726,7 @@ static void open_slots(struct nufft_fi_s* d, const void* ref) struct nufft_data* t = d->toeplitz_data; int ND = t->N + 1; - long psf_dims[ND]; + bart_dim_t psf_dims[ND]; md_copy_dims(ND, psf_dims, t->psf_dims); psf_dims[t->N] = d->unit; @@ -756,7 +756,7 @@ static void issue_coset(struct nufft_fi_s* d, int i, int slot) struct nufft_data* t = d->toeplitz_data; int ND = t->N + 1; - long psf_bytes = d->psf_coset * (long)d->psf_size * d->unit; + bart_dim_t psf_bytes = d->psf_coset * (bart_dim_t)d->psf_size * d->unit; const char* psf_src = (const char*)d->psf_host + (size_t)i * (size_t)psf_bytes; @@ -766,7 +766,7 @@ static void issue_coset(struct nufft_fi_s* d, int i, int slot) return; } - long psf_dims[ND]; + bart_dim_t psf_dims[ND]; md_copy_dims(ND, psf_dims, t->psf_dims); psf_dims[t->N] = d->unit; @@ -821,7 +821,7 @@ static void use_coset(struct nufft_fi_s* d, int s) struct nufft_data* t = d->toeplitz_data; int ND = t->N + 1; - long psf_dims[ND]; + bart_dim_t psf_dims[ND]; md_copy_dims(ND, psf_dims, t->psf_dims); psf_dims[t->N] = 1; @@ -839,7 +839,7 @@ static void coset_shift(const struct nufft_fi_s* d, int set, float shift[3]) { const struct nufft_data* t = d->toeplitz_data; - long factors[3]; + bart_dim_t factors[3]; for (int i = 0; i < 3; i++) factors[i] = ((t->img_dims[i] > 1) && MD_IS_SET(t->flags, i)) ? 2 : 1; @@ -852,7 +852,7 @@ static void coset_shift(const struct nufft_fi_s* d, int set, float shift[3]) * rather than read from a volume of it. It is BART's own kernel, over the * shift, centring and scale its precomputed phases carry, so what it applies * is what they would have. */ -static void apply_phase(const struct nufft_data* t, const long dims[], const float shift[3], +static void apply_phase(const struct nufft_data* t, const bart_dim_t dims[], const float shift[3], complex float* dst, const complex float* src, bool out) { float scale = 1.f / sqrtf((float)md_calc_size(3, t->img_dims)); @@ -902,7 +902,7 @@ static void apply_phase_map(const struct nufft_data* t, const float shift[3], enum { CONTRACT_CHUNK = 1 << 18 }; static void contract_bank(struct nufft_data* t, const void* psf, - const long bank_dims[], const long ciT_dims[], complex float* bank) + const bart_dim_t bank_dims[], const bart_dim_t ciT_dims[], complex float* bank) { int N = t->N; @@ -912,7 +912,7 @@ static void contract_bank(struct nufft_data* t, const void* psf, * buffer that has to be cleared and copied back. */ if (contraction_kernel && t->conf.real && t->conf.upper_triag && cuda_ondevice(bank)) { - long L = bank_dims[0]; + bart_dim_t L = bank_dims[0]; int R = (int)(md_calc_size(N, bank_dims) / L); if ((md_calc_size(N, ciT_dims) == md_calc_size(N, bank_dims)) @@ -921,17 +921,17 @@ static void contract_bank(struct nufft_data* t, const void* psf, } #endif - long locations = bank_dims[0]; - long chunk = MIN(locations, (long)CONTRACT_CHUNK); + bart_dim_t locations = bank_dims[0]; + bart_dim_t chunk = MIN(locations, (bart_dim_t)CONTRACT_CHUNK); - long bank_strs[N]; - long ciT_strs[N]; + bart_stride_t bank_strs[N]; + bart_stride_t ciT_strs[N]; md_calc_strides(N, bank_strs, bank_dims, CFL_SIZE); md_calc_strides(N, ciT_strs, ciT_dims, CFL_SIZE); - long in_dims[N]; - long out_dims[N]; + bart_dim_t in_dims[N]; + bart_dim_t out_dims[N]; md_copy_dims(N, in_dims, bank_dims); md_copy_dims(N, out_dims, ciT_dims); @@ -941,18 +941,18 @@ static void contract_bank(struct nufft_data* t, const void* psf, complex float* out = md_alloc_sameplace(N, out_dims, CFL_SIZE, bank); - for (long start = 0; start < locations; start += chunk) { + for (bart_dim_t start = 0; start < locations; start += chunk) { - long here = MIN(chunk, locations - start); + bart_dim_t here = MIN(chunk, locations - start); in_dims[0] = here; out_dims[0] = here; - long out_strs[N]; + bart_stride_t out_strs[N]; md_calc_strides(N, out_strs, out_dims, CFL_SIZE); - long max_dims[N]; - md_max_dims(N, ~0UL, max_dims, out_dims, in_dims); + bart_dim_t max_dims[N]; + md_max_dims(N, ~UINT64_C(0), max_dims, out_dims, in_dims); complex float* in = bank + start; const void* mat = (const char*)psf + (size_t)start * (size_t)t->psf_strs[0]; @@ -995,13 +995,13 @@ static void contract_bank(struct nufft_data* t, const void* psf, * it is a matrix at every frequency and the contraction needs somewhere to * land, so what comes back may not be what went in. */ static complex float* multiply_transfer(struct nufft_data* t, const void* psf, - const long cim_dims[], const long ciT_dims[], complex float* grid) + const bart_dim_t cim_dims[], const bart_dim_t ciT_dims[], complex float* grid) { int N = t->N; - if (md_check_equal_dims(N, cim_dims, ciT_dims, ~0UL)) { + if (md_check_equal_dims(N, cim_dims, ciT_dims, ~UINT64_C(0))) { - long cim_strs[N]; + bart_stride_t cim_strs[N]; md_calc_strides(N, cim_strs, cim_dims, CFL_SIZE); if (t->conf.real) @@ -1015,13 +1015,13 @@ static complex float* multiply_transfer(struct nufft_data* t, const void* psf, return grid; } - long max_dims[N]; - md_max_dims(N, ~0UL, max_dims, ciT_dims, cim_dims); + bart_dim_t max_dims[N]; + md_max_dims(N, ~UINT64_C(0), max_dims, ciT_dims, cim_dims); - long ciT_strs[N]; + bart_stride_t ciT_strs[N]; md_calc_strides(N, ciT_strs, ciT_dims, CFL_SIZE); - long cim_strs[N]; + bart_stride_t cim_strs[N]; md_calc_strides(N, cim_strs, cim_dims, CFL_SIZE); complex float* out = md_alloc_sameplace(N, ciT_dims, CFL_SIZE, grid); @@ -1154,7 +1154,7 @@ static struct bartorch_cb_fft* callbacks_for(struct nufft_fi_s* d) #ifdef USE_CUDA const struct nufft_data* t = d->toeplitz_data; - unsigned long flags = t->flags | t->conf.cfft; + bart_flags_t flags = t->flags | t->conf.cfft; bool covered = true; for (int i = 0; i < t->N; i++) @@ -1188,7 +1188,7 @@ static void slot_ready(struct nufft_fi_s* d) } /* Down to the places the samples reach, and back up with zeros elsewhere. */ -static void gather(const struct nufft_fi_s* d, long grid, complex float* dst, const complex float* src) +static void gather(const struct nufft_fi_s* d, bart_dim_t grid, complex float* dst, const complex float* src) { #ifdef USE_CUDA bartorch_cuda_gather(grid, d->kept_mask, d->kept_prefix, dst, src); @@ -1198,7 +1198,7 @@ static void gather(const struct nufft_fi_s* d, long grid, complex float* dst, co #endif } -static void scatter(const struct nufft_fi_s* d, long grid, complex float* dst, const complex float* src) +static void scatter(const struct nufft_fi_s* d, bart_dim_t grid, complex float* dst, const complex float* src) { #ifdef USE_CUDA bartorch_cuda_scatter(grid, d->kept_mask, d->kept_prefix, dst, src); @@ -1215,7 +1215,7 @@ static void coefficient_in(struct nufft_fi_s* d, struct bartorch_cb_fft* cb, con { struct nufft_data* t = d->toeplitz_data; int N = t->N; - long grid = md_calc_size(3, t->img_dims); + bart_dim_t grid = md_calc_size(3, t->img_dims); #ifdef USE_CUDA if (NULL != cb) { @@ -1242,7 +1242,7 @@ static void coefficient_out(struct nufft_fi_s* d, struct bartorch_cb_fft* cb, co { struct nufft_data* t = d->toeplitz_data; int N = t->N; - long grid = md_calc_size(3, t->img_dims); + bart_dim_t grid = md_calc_size(3, t->img_dims); #ifdef USE_CUDA if (NULL != cb) { @@ -1278,7 +1278,7 @@ static void coefficient_out(struct nufft_fi_s* d, struct bartorch_cb_fft* cb, co * inside the transforms, they run there. */ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex float* src, - const void* psf, const long map_strs[], const complex float* map, bool last) + const void* psf, const bart_stride_t map_strs[], const complex float* map, bool last) { struct nufft_data* t = d->toeplitz_data; int N = t->N; @@ -1293,9 +1293,9 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex /* The spectrum of one coil, gathered: the places the samples reach on * the first axis, the coefficients on the axis the function contracts. */ - long bank_dims[N]; - long ciT_bank_dims[N]; - long one_bank_dims[N]; + bart_dim_t bank_dims[N]; + bart_dim_t ciT_bank_dims[N]; + bart_dim_t one_bank_dims[N]; md_select_dims(N, ~MD_BIT(3), bank_dims, t->cim_dims); md_select_dims(N, ~MD_BIT(3), ciT_bank_dims, t->ciT_dims); @@ -1305,10 +1305,10 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex md_select_dims(N, MD_BIT(0), one_bank_dims, bank_dims); - long locations = bank_dims[0]; - long coils = t->cim_dims[3]; - long coeffs = md_calc_size(N, bank_dims) / locations; - long out_coeffs = md_calc_size(N, ciT_bank_dims) / locations; + bart_dim_t locations = bank_dims[0]; + bart_dim_t coils = t->cim_dims[3]; + bart_dim_t coeffs = md_calc_size(N, bank_dims) / locations; + bart_dim_t out_coeffs = md_calc_size(N, ciT_bank_dims) / locations; /* Where a coil's coefficient sits in what is read and written. Counted * here rather than read off `cim_strs`, which carries a zero for every @@ -1317,22 +1317,22 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex * With the sensitivity folded in, what is read and written is the image * rather than a coil image: it has no coil axis, every coil accumulates * into the same one, and the map is what tells them apart. */ - long vol = md_calc_size(3, t->cim_dims); + bart_dim_t vol = md_calc_size(3, t->cim_dims); - long coil_step = (NULL == map) ? vol : 0; - long rest_step = (NULL == map) ? vol * t->cim_dims[3] : vol; + bart_dim_t coil_step = (NULL == map) ? vol : 0; + bart_dim_t rest_step = (NULL == map) ? vol * t->cim_dims[3] : vol; - long map_coil_step = (NULL == map) ? 0 : map_strs[3] / (long)CFL_SIZE; + bart_dim_t map_coil_step = (NULL == map) ? 0 : map_strs[3] / (bart_stride_t)CFL_SIZE; complex float* volume = md_alloc_sameplace(N, t->img_dims, CFL_SIZE, dst); - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_coil_step; complex float* bank = md_alloc_sameplace(N, bank_dims, CFL_SIZE, dst); - for (long r = 0; r < coeffs; r++) + for (bart_dim_t r = 0; r < coeffs; r++) coefficient_in(d, cb, shift, bank + r * locations, volume, src + c * coil_step + r * rest_step, m); slot_ready(d); @@ -1342,14 +1342,14 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex bartorch_cuda_contract_upper_real_bf16(locations, (int)coeffs, bank, psf); else #endif - if (md_check_equal_dims(N, bank_dims, ciT_bank_dims, ~0UL)) + if (md_check_equal_dims(N, bank_dims, ciT_bank_dims, ~UINT64_C(0))) bank = multiply_transfer(t, psf, bank_dims, ciT_bank_dims, bank); else contract_bank(t, psf, bank_dims, ciT_bank_dims, bank); slot_read(d, last && (c == coils - 1)); - for (long r = 0; r < out_coeffs; r++) + for (bart_dim_t r = 0; r < out_coeffs; r++) coefficient_out(d, cb, shift, dst + c * coil_step + r * rest_step, volume, bank + r * locations, m); md_free(bank); @@ -1368,11 +1368,11 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex * turn. The kernels read a coil's coefficients a volume apart, which a folded * sensitivity or a single coil gives; anything else goes a set at a time. */ static void paired_unit(struct nufft_fi_s* d, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, bool last) + const bart_stride_t map_strs[], const complex float* map, bool last) { struct nufft_data* t = d->toeplitz_data; - long coils = t->cim_dims[3]; + bart_dim_t coils = t->cim_dims[3]; if ((NULL == map) && (1 != coils)) { @@ -1390,17 +1390,17 @@ static void paired_unit(struct nufft_fi_s* d, complex float* dst, const complex } #ifdef BARTORCH_PAIRED - long vol = md_calc_size(3, t->cim_dims); - long coeffs = md_calc_size(t->N, t->cim_dims) / md_calc_size(4, t->cim_dims); - long map_coil_step = (NULL == map) ? 0 : map_strs[3] / (long)CFL_SIZE; + bart_dim_t vol = md_calc_size(3, t->cim_dims); + bart_dim_t coeffs = md_calc_size(t->N, t->cim_dims) / md_calc_size(4, t->cim_dims); + bart_dim_t map_coil_step = (NULL == map) ? 0 : map_strs[3] / (bart_stride_t)CFL_SIZE; - long sdims[1] = { coeffs * vol }; + bart_dim_t sdims[1] = { coeffs * vol }; complex float* scratch = md_alloc_sameplace(1, sdims, CFL_SIZE, dst); const char* psf0 = d->psf_slot[d->slot]; const char* psf1 = psf0 + (size_t)d->psf_coset * d->psf_size; - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_coil_step; @@ -1438,7 +1438,7 @@ static void paired_unit(struct nufft_fi_s* d, complex float* dst, const complex * triangle -- and the arrangements it does not cover go back to BART's own. */ static bool fused_coset(struct nufft_fi_s* d, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, bool last) + const bart_stride_t map_strs[], const complex float* map, bool last) { if (2 == d->unit) { @@ -1524,7 +1524,7 @@ static void coset_begin(struct nufft_fi_s* d, const void* ref) /* The set that is loaded, convolved with `src` and added to `dst`. */ static void coset_normal(struct nufft_fi_s* d, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, bool last) + const bart_stride_t map_strs[], const complex float* map, bool last) { if (!fused_coset(d, dst, src, map_strs, map, last)) error("bartorch: a streamed set with no function in its slot\n"); @@ -1572,7 +1572,7 @@ int bartorch_nufft_coset_folds(const struct linop_s* op) * each of those is half a gigabyte. Folded in here neither is made. */ void bartorch_nufft_coset_normal_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, int last) + const int64_t map_strs[], const complex float* map, int last) { coset_normal(CAST_DOWN(nufft_fi_s, linop_get_data(op)), dst, src, map_strs, map, (0 != last)); } @@ -1726,9 +1726,9 @@ int bartorch_nufft_fallback_allowed(void) /* Which operator each nufft_create call returned, so a test can say that a * tool ran on FINUFFT rather than that FINUFFT was merely available. */ enum { CNT_FI, CNT_BART }; -static long counters[2]; +static bart_dim_t counters[2]; -long bartorch_nufft_counter(int which) +int64_t bartorch_nufft_counter(int which) { return ((0 == which) || (1 == which)) ? counters[which] : -1; } @@ -1787,8 +1787,8 @@ static int fi_measure_width(int device, int dim, double eps, double upsampling) enum { PROBE = 32 }; /* wider than any kernel FINUFFT will use */ int64_t n_modes[3] = { PROBE, PROBE, PROBE }; - long grid_dims[3] = { 1, 1, 1 }; - long one[1] = { 1 }; + bart_dim_t grid_dims[3] = { 1, 1, 1 }; + bart_dim_t one[1] = { 1 }; for (int i = 0; i < dim; i++) grid_dims[i] = PROBE; @@ -1833,8 +1833,8 @@ static int fi_measure_width(int device, int dim, double eps, double upsampling) md_free(grid); /* Along one axis through the middle, where the sample was put. */ - long stride = 1; - long centre = 0; + bart_stride_t stride = 1; + bart_dim_t centre = 0; for (int i = 1; i < dim; i++) { @@ -1909,7 +1909,7 @@ static double fi_tolerance_for(int device, int dim, double width, double upsampl * is also what keeps the doubled grid from ever being allocated, which is the * whole reason the decomposition is there. */ -static int spread_mask(struct nufft_data* data, const complex float* traj, long* max_idx) +static int spread_mask(struct nufft_data* data, const complex float* traj, bart_dim_t* max_idx) { int N = data->N; int ND = N + 1; @@ -1964,7 +1964,7 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* int dim = 0; int axis[3]; int64_t n_modes[3]; - unsigned long whole = 0UL; + bart_flags_t whole = 0; for (int i = 0; i < 3; i++) { @@ -1982,24 +1982,24 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* dim++; } - long factors[N]; + bart_dim_t factors[N]; for (int i = 0; i < N; i++) factors[i] = ((data->img_dims[i] > 1) && MD_IS_SET(data->flags, i)) ? 2 : 1; - long sets = md_calc_size(N, factors); + bart_dim_t sets = md_calc_size(N, factors); - long one_dims[ND]; - long one_strs[ND]; - long trj_strs[ND]; - long wgh_strs[ND]; + bart_dim_t one_dims[ND]; + bart_stride_t one_strs[ND]; + bart_stride_t trj_strs[ND]; + bart_stride_t wgh_strs[ND]; - md_select_dims(ND, ~1UL, one_dims, data->trj_dims); + md_select_dims(ND, ~UINT64_C(1), one_dims, data->trj_dims); md_calc_strides(ND, one_strs, one_dims, CFL_SIZE); md_calc_strides(ND, trj_strs, data->trj_dims, CFL_SIZE); md_calc_strides(ND, wgh_strs, data->wgh_dims, CFL_SIZE); - long samples = md_calc_size(ND, one_dims); + bart_dim_t samples = md_calc_size(ND, one_dims); /* One component of the trajectory per transformed axis, in grid samples, * before the shift of a set turns it into that set's coordinates. */ @@ -2023,7 +2023,7 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* else md_copy2(ND, one_dims, one_strs, samples_in, wgh_strs, pattern, CFL_SIZE); - long spread_dims[ND]; + bart_dim_t spread_dims[ND]; md_select_dims(ND, ~whole, spread_dims, data->com_dims); complex float* reach = alloc_on(device, ND, spread_dims, CFL_SIZE); @@ -2045,7 +2045,7 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* if (0 == dim) md_zfill(ND, spread_dims, mask, 1.); - for (long set = 0; (0 < dim) && (0 == ret) && (set < sets); set++) { + for (bart_dim_t set = 0; (0 < dim) && (0 == ret) && (set < sets); set++) { float shift[3]; bartorch_psf_shift(3, shift, N, factors, (int)set); @@ -2104,12 +2104,12 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* MD_STRIDES(ND, spread_dims, CFL_SIZE), mask, CFL_SIZE); md_free(mask); - long* idx = md_alloc(ND, data->com_dims, sizeof(long)); + bart_dim_t* idx = md_alloc(ND, data->com_dims, sizeof(bart_dim_t)); *max_idx = md_compress_mask_to_index(ND, data->com_dims, idx, mask_cpu); md_free(mask_cpu); multiplace_free(data->compress); - data->compress = multiplace_move_F(ND, data->com_dims, sizeof(long), idx); + data->compress = multiplace_move_F(ND, data->com_dims, sizeof(bart_dim_t), idx); debug_printf(DP_DEBUG1, "Compressing PSF to %.0f%%\n", 100. * *max_idx / md_calc_size(ND, data->com_dims)); @@ -2135,12 +2135,12 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* * and cancels. A basis is small enough to ask about on the host, and the * angle is half the argument of the sum of its squares -- which is `e^{2it}` * times something real when there is one angle to find. */ -static bool basis_is_real(int N, const long bas_dims[N], const complex float* basis) +static bool basis_is_real(int N, const bart_dim_t bas_dims[N], const complex float* basis) { if (NULL == basis) return true; - long size = md_calc_size(N, bas_dims); + bart_dim_t size = md_calc_size(N, bas_dims); complex float* host = md_alloc(N, bas_dims, CFL_SIZE); md_copy(N, bas_dims, host, basis, CFL_SIZE); @@ -2148,7 +2148,7 @@ static bool basis_is_real(int N, const long bas_dims[N], const complex float* ba complex float squares = 0.; double energy = 0.; - for (long i = 0; i < size; i++) { + for (bart_dim_t i = 0; i < size; i++) { squares += host[i] * host[i]; energy += (double)crealf(host[i]) * crealf(host[i]) + (double)cimagf(host[i]) * cimagf(host[i]); @@ -2158,7 +2158,7 @@ static bool basis_is_real(int N, const long bas_dims[N], const complex float* ba double left = 0.; - for (long i = 0; i < size; i++) { + for (bart_dim_t i = 0; i < size; i++) { float part = cimagf(host[i] * turn); left += (double)part * part; @@ -2205,7 +2205,7 @@ static void install_psf(struct nufft_data* data, const complex float* traj, comp /* The places the samples reach. Worked out before the function is * built, because an entry is compressed as it is made. */ - long max_idx = 0; + bart_dim_t max_idx = 0; if (data->conf.compress_psf) { @@ -2232,13 +2232,13 @@ static void install_psf(struct nufft_data* data, const complex float* traj, comp * dimensional radial trajectory leaves the corners of the cube * outside its ball, which is a fraction 1 - pi/6 of it before * the spreading kernel's width is added back. */ - long grid = md_calc_size(ND, data->com_dims); - long entries = md_calc_size(N, data->psf_dims) / md_calc_size(3, data->psf_dims); + bart_dim_t grid = md_calc_size(ND, data->com_dims); + bart_dim_t entries = md_calc_size(N, data->psf_dims) / md_calc_size(3, data->psf_dims); size_t element = store_real ? FL_SIZE : CFL_SIZE; double dropped = 1. - (double)max_idx / (double)grid; - if (dropped * (double)entries * (double)element <= (double)sizeof(long)) { + if (dropped * (double)entries * (double)element <= (double)sizeof(bart_dim_t)) { debug_printf(DP_DEBUG1, "Not compressing: %.0f%% of the grid is reached\n", 100. * max_idx / grid); @@ -2254,9 +2254,9 @@ static void install_psf(struct nufft_data* data, const complex float* traj, comp #pragma omp atomic toeplitz_counters[TP_COMPRESSED]++; - long com_psf_dims[ND]; - long com_psf_dims3[ND]; - const long* idx = NULL; + bart_dim_t com_psf_dims[ND]; + bart_dim_t com_psf_dims3[ND]; + const bart_dim_t* idx = NULL; if (stream && (NULL != data->compress)) { @@ -2313,7 +2313,7 @@ static void install_psf(struct nufft_data* data, const complex float* traj, comp size_t size = store_real ? FL_SIZE : CFL_SIZE; - long com_psf_dims[ND]; + bart_dim_t com_psf_dims[ND]; md_compress_dims(ND, com_psf_dims, data->psf_dims, data->com_dims, max_idx); complex float* com_psf = md_alloc_sameplace(ND, com_psf_dims, size, traj); @@ -2388,21 +2388,21 @@ static void stream_psf(struct nufft_fi_s* d) int ND = t->N + 1; - long grid = md_calc_size(ND, t->com_dims); - const long* map = multiplace_read(t->compress, d->radians[0]); + bart_dim_t grid = md_calc_size(ND, t->com_dims); + const bart_dim_t* map = multiplace_read(t->compress, d->radians[0]); - long wdims[1] = { (grid + 31) / 32 }; + bart_dim_t wdims[1] = { (grid + 31) / 32 }; unsigned int* mask = xmalloc((size_t)wdims[0] * sizeof(unsigned int)); int* prefix = xmalloc((size_t)wdims[0] * sizeof(int)); - long n = 0; + bart_dim_t n = 0; - for (long w = 0; w < wdims[0]; w++) { + for (bart_dim_t w = 0; w < wdims[0]; w++) { unsigned int bits = 0; - for (long b = 0; (b < 32) && (w * 32 + b < grid); b++) + for (bart_dim_t b = 0; (b < 32) && (w * 32 + b < grid); b++) if (0 <= map[w * 32 + b]) bits |= 1u << b; @@ -2443,7 +2443,7 @@ static void stream_psf(struct nufft_fi_s* d) for (int i = 0; i < 8; i++) coset_shift(d, i, shifts[i]); - long coeffs = md_calc_size(t->N, t->cim_dims) / md_calc_size(4, t->cim_dims); + bart_dim_t coeffs = md_calc_size(t->N, t->cim_dims) / md_calc_size(4, t->cim_dims); d->paired = bartorch_paired_create(t->img_dims, (int)coeffs, 8, (const float (*)[3])shifts); } @@ -2462,11 +2462,11 @@ static void stream_psf(struct nufft_fi_s* d) * contraction kernel, so that kernel is a condition. */ if ((2 == d->unit) && bf16_enabled && contraction_kernel) { - long n = (long)cosets * d->psf_coset; + bart_dim_t n = (bart_dim_t)cosets * d->psf_coset; const float* in = (const float*)d->psf_host; uint16_t* out = (uint16_t*)d->psf_host; - for (long i = 0; i < n; i++) + for (bart_dim_t i = 0; i < n; i++) out[i] = to_bf16(in[i]); void* shrunk = realloc(d->psf_host, (size_t)n * sizeof(uint16_t)); @@ -2481,18 +2481,18 @@ static void stream_psf(struct nufft_fi_s* d) /* Page-locked, so a set's crossing runs behind the convolution rather * than holding up the host that issued it. */ d->psf_registered = (0 == bartorch_cuda_host_register(d->psf_host, - (long)cosets * d->psf_coset * (long)d->psf_size)); + (bart_dim_t)cosets * d->psf_coset * (bart_dim_t)d->psf_size)); /* One set is all BART is told it has, so its own loop runs once. */ t->lph_dims[t->N] = 1; - debug_printf(DP_DEBUG1, "Streaming the function: %d sets of %ld, %d to a slot\n", cosets, d->psf_coset, d->unit); + debug_printf(DP_DEBUG1, "Streaming the function: %d sets of %" PRId64 ", %d to a slot\n", cosets, d->psf_coset, d->unit); } -static const struct linop_s* toeplitz_for(int N, const long ksp_dims[N], const long cim_dims[N], - const long traj_dims[N], const complex float* traj, - const long wgh_dims[N], const complex float* weights, - const long bas_dims[N], const complex float* basis, struct nufft_conf_s conf, +static const struct linop_s* toeplitz_for(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], + const bart_dim_t traj_dims[N], const complex float* traj, + const bart_dim_t wgh_dims[N], const complex float* weights, + const bart_dim_t bas_dims[N], const complex float* basis, struct nufft_conf_s conf, complex float** to_host) { if (!conf.toeplitz) { @@ -2574,14 +2574,14 @@ static const struct linop_s* toeplitz_for(int N, const long ksp_dims[N], const l * its place before the function is built from it. */ bool along_samples = (NULL != basis) && ((1 != bas_dims[1]) || (1 != bas_dims[2])); - long stand_dims[N]; + bart_dim_t stand_dims[N]; complex float* stand = NULL; if (along_samples) { md_select_dims(N, ~(MD_BIT(1) | MD_BIT(2)), stand_dims, bas_dims); - long pos[N]; + bart_dim_t pos[N]; md_set_dims(N, pos, 0); complex float* whole = md_alloc(N, bas_dims, CFL_SIZE); @@ -2649,10 +2649,10 @@ static const struct linop_s* toeplitz_for(int N, const long ksp_dims[N], const l return op; } -static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_dims[N], - const long traj_dims[N], const complex float* traj, - const long wgh_dims[N], const complex float* weights, - const long bas_dims[N], const complex float* basis, struct nufft_conf_s conf) +static struct linop_s* try_create(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], + const bart_dim_t traj_dims[N], const complex float* traj, + const bart_dim_t wgh_dims[N], const complex float* weights, + const bart_dim_t bas_dims[N], const complex float* basis, struct nufft_conf_s conf) { /* Which sides the operator may be applied on. BART on a card applies * one to either: `pics` takes its first adjoint from the k-space it @@ -2676,7 +2676,7 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ if (1 != ksp_dims[0]) DECLINE(5); - if ((0UL != conf.flags) && (7UL != conf.flags)) + if ((0 != conf.flags) && (7 != conf.flags)) DECLINE(6); int dim = 0; @@ -2704,8 +2704,8 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ * A caller may pass either one in: `nufft` gives k-space with a single * coefficient and `pics` gives it with all of them, which is why BART * fills the coefficient axis in rather than reading it. */ - long out_dims[N]; - long grd_dims[N]; + bart_dim_t out_dims[N]; + bart_dim_t grd_dims[N]; md_copy_dims(N, out_dims, ksp_dims); md_copy_dims(N, grd_dims, ksp_dims); @@ -2747,8 +2747,8 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ * rest are separate transforms. A frame is a sample axis when the * trajectory varies across frames, which is what lets one plan over the * whole raveled trajectory serve every coefficient and every coil. */ - long samples = md_calc_size(N - 1, traj_dims + 1); - long batch = 1; + bart_dim_t samples = md_calc_size(N - 1, traj_dims + 1); + bart_dim_t batch = 1; for (int i = 1; i < N; i++) { @@ -2832,10 +2832,10 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ DECLINE(17); } - long image_elements = 1; + bart_dim_t image_elements = 1; for (int i = 0; i < dim; i++) - image_elements *= (long)n_modes[i]; + image_elements *= (bart_dim_t)n_modes[i]; PTR_ALLOC(struct nufft_fi_s, d); SET_TYPEID(nufft_fi_s, d); @@ -2888,17 +2888,17 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ d->N = N; d->host_weights = NULL; d->host_basis = NULL; - d->cim_dims = xmalloc((size_t)N * sizeof(long)); - d->out_dims = xmalloc((size_t)N * sizeof(long)); - d->out_strs = xmalloc((size_t)N * sizeof(long)); - d->grd_dims = xmalloc((size_t)N * sizeof(long)); - d->trf_strs = xmalloc((size_t)N * sizeof(long)); - d->wgh_dims = xmalloc((size_t)N * sizeof(long)); - d->wgh_strs = xmalloc((size_t)N * sizeof(long)); - d->bas_dims = xmalloc((size_t)N * sizeof(long)); - d->bas_strs = xmalloc((size_t)N * sizeof(long)); - d->trj_dims = xmalloc((size_t)N * sizeof(long)); - d->ksp_dims = xmalloc((size_t)N * sizeof(long)); + d->cim_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->out_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->out_strs = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->grd_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->trf_strs = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->wgh_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->wgh_strs = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->bas_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->bas_strs = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->trj_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->ksp_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); md_copy_dims(N, d->trj_dims, traj_dims); md_copy_dims(N, d->ksp_dims, ksp_dims); d->conf = conf; @@ -2916,26 +2916,26 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ * stepping over them, so the sample axes take the fastest strides and * the rest follow. BART's own order is already that unless a sample * axis sits above a batch axis. */ - long stride = 1; + bart_stride_t stride = 1; for (int i = 1; i < N; i++) if (1 < traj_dims[i]) { - d->trf_strs[i] = (1 == grd_dims[i]) ? 0 : stride * (long)CFL_SIZE; + d->trf_strs[i] = (1 == grd_dims[i]) ? 0 : stride * (bart_stride_t)CFL_SIZE; stride *= grd_dims[i]; } for (int i = 0; i < N; i++) if ((0 == i) || (1 >= traj_dims[i])) { - d->trf_strs[i] = (1 == grd_dims[i]) ? 0 : stride * (long)CFL_SIZE; + d->trf_strs[i] = (1 == grd_dims[i]) ? 0 : stride * (bart_stride_t)CFL_SIZE; stride *= grd_dims[i]; } - long grd_strs[N]; + bart_stride_t grd_strs[N]; md_calc_strides(N, grd_strs, grd_dims, CFL_SIZE); - d->needs_tmp = (NULL != basis) || (0 != memcmp(d->trf_strs, grd_strs, (size_t)N * sizeof(long))); + d->needs_tmp = (NULL != basis) || (0 != memcmp(d->trf_strs, grd_strs, (size_t)N * sizeof(bart_dim_t))); if (NULL != basis) { @@ -2991,10 +2991,10 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ return op; } -struct linop_s* nufft_create2(int N, const long ksp_dims[N], const long cim_dims[N], - const long traj_dims[N], const complex float* traj, - const long wgh_dims[N], const complex float* weights, - const long bas_dims[N], const complex float* basis, struct nufft_conf_s conf) +struct linop_s* nufft_create2(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], + const bart_dim_t traj_dims[N], const complex float* traj, + const bart_dim_t wgh_dims[N], const complex float* weights, + const bart_dim_t bas_dims[N], const complex float* basis, struct nufft_conf_s conf) { struct linop_s* op = try_create(N, ksp_dims, cim_dims, traj_dims, traj, wgh_dims, weights, bas_dims, basis, conf); @@ -3014,11 +3014,11 @@ struct linop_s* nufft_create2(int N, const long ksp_dims[N], const long cim_dims return bart_nufft_create2(N, ksp_dims, cim_dims, traj_dims, traj, wgh_dims, weights, bas_dims, basis, barts_conf(conf)); } -struct linop_s* nufft_create(int N, const long ksp_dims[N], const long cim_dims[N], - const long traj_dims[N], const complex float* traj, +struct linop_s* nufft_create(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], + const bart_dim_t traj_dims[N], const complex float* traj, const complex float* weights, struct nufft_conf_s conf) { - long wgh_dims[N]; + bart_dim_t wgh_dims[N]; md_select_dims(N, ~MD_BIT(0), wgh_dims, traj_dims); return nufft_create2(N, ksp_dims, cim_dims, traj_dims, traj, wgh_dims, weights, NULL, NULL, conf); @@ -3033,7 +3033,7 @@ static void refuse(const char* what) "Turn the substitution off with bartorch.finufft.use_in_tools(False).\n", what); } -int nufft_get_psf_dims(const struct linop_s* nufft, int N, long psf_dims[N]) +int nufft_get_psf_dims(const struct linop_s* nufft, int N, bart_dim_t psf_dims[N]) { if (is_ours(nufft)) refuse("reading a point spread function"); @@ -3041,7 +3041,7 @@ int nufft_get_psf_dims(const struct linop_s* nufft, int N, long psf_dims[N]) return bart_nufft_get_psf_dims(nufft, N, psf_dims); } -void nufft_get_psf(const struct linop_s* nufft, int N, const long psf_dims[N], complex float* psf) +void nufft_get_psf(const struct linop_s* nufft, int N, const bart_dim_t psf_dims[N], complex float* psf) { if (is_ours(nufft)) refuse("reading a point spread function"); @@ -3049,7 +3049,7 @@ void nufft_get_psf(const struct linop_s* nufft, int N, const long psf_dims[N], c bart_nufft_get_psf(nufft, N, psf_dims, psf); } -void nufft_get_psf2(const struct linop_s* nufft, int N, const long psf_dims[N], const long psf_strs[N], complex float* psf) +void nufft_get_psf2(const struct linop_s* nufft, int N, const bart_dim_t psf_dims[N], const bart_stride_t psf_strs[N], complex float* psf) { if (is_ours(nufft)) refuse("reading a point spread function"); @@ -3057,7 +3057,7 @@ void nufft_get_psf2(const struct linop_s* nufft, int N, const long psf_dims[N], bart_nufft_get_psf2(nufft, N, psf_dims, psf_strs, psf); } -void nufft_update_psf(const struct linop_s* nufft, int ND, const long psf_dims[ND], const complex float* psf) +void nufft_update_psf(const struct linop_s* nufft, int ND, const bart_dim_t psf_dims[ND], const complex float* psf) { if (is_ours(nufft)) refuse("supplying a point spread function"); @@ -3065,7 +3065,7 @@ void nufft_update_psf(const struct linop_s* nufft, int ND, const long psf_dims[N bart_nufft_update_psf(nufft, ND, psf_dims, psf); } -void nufft_update_psf2(const struct linop_s* nufft, int ND, const long psf_dims[ND], const long psf_strs[ND], const complex float* psf) +void nufft_update_psf2(const struct linop_s* nufft, int ND, const bart_dim_t psf_dims[ND], const bart_stride_t psf_strs[ND], const complex float* psf) { if (is_ours(nufft)) refuse("supplying a point spread function"); @@ -3077,7 +3077,7 @@ void nufft_update_psf2(const struct linop_s* nufft, int ND, const long psf_dims[ * alone and hand the trajectory over here, once per frame of a run. The * plans belong to it, so both sides go and are made again on the next * transform, over whatever else arrived with it. */ -void nufft_update_traj(const struct linop_s* nufft, int N, const long trj_dims[N], const complex float* traj, const long wgh_dims[N], const complex float* weights, const long bas_dims[N], const complex float* basis) +void nufft_update_traj(const struct linop_s* nufft, int N, const bart_dim_t trj_dims[N], const complex float* traj, const bart_dim_t wgh_dims[N], const complex float* weights, const bart_dim_t bas_dims[N], const complex float* basis) { if (!is_ours(nufft)) { @@ -3091,13 +3091,13 @@ void nufft_update_traj(const struct linop_s* nufft, int N, const long trj_dims[N error("bartorch: a trajectory of %d axes for an operator of %d\n", N, d->N); if (d->samples != md_calc_size(N - 1, trj_dims + 1)) - error("bartorch: a trajectory of %ld samples for an operator of %ld\n", + error("bartorch: a trajectory of %" PRId64 " samples for an operator of %" PRId64 "\n", md_calc_size(N - 1, trj_dims + 1), d->samples); - if ((NULL != weights) && !md_check_equal_dims(N, wgh_dims, d->wgh_dims, ~0UL)) + if ((NULL != weights) && !md_check_equal_dims(N, wgh_dims, d->wgh_dims, ~UINT64_C(0))) error("bartorch: weights of a shape the operator was not built for\n"); - if ((NULL != basis) && !md_check_equal_dims(N, bas_dims, d->bas_dims, ~0UL)) + if ((NULL != basis) && !md_check_equal_dims(N, bas_dims, d->bas_dims, ~UINT64_C(0))) error("bartorch: a basis of a shape the operator was not built for\n"); pthread_mutex_lock(&d->lock); diff --git a/src/csrc/substitute/psf.c b/src/csrc/substitute/psf.c index 498471f9..c65f006e 100644 --- a/src/csrc/substitute/psf.c +++ b/src/csrc/substitute/psf.c @@ -45,7 +45,7 @@ #include "noncart/nufft.h" /* The weights the normal carries are the transform's, squared. */ -static complex float* square_weights(int N, const long wgh_dims[N], const complex float* weights) +static complex float* square_weights(int N, const bart_dim_t wgh_dims[N], const complex float* weights) { if (NULL == weights) return NULL; @@ -58,8 +58,8 @@ static complex float* square_weights(int N, const long wgh_dims[N], const comple /* A subspace basis enters the normal as its own Gram matrix, laid out along * the coefficient axis the transform contracts. */ -static complex float* square_basis(bool upper_triag, int N, long sqr_bas_dims[N], - const long bas_dims[N], const complex float* basis, const long ksp_dims[N]) +static complex float* square_basis(bool upper_triag, int N, bart_dim_t sqr_bas_dims[N], + const bart_dim_t bas_dims[N], const complex float* basis, const bart_dim_t ksp_dims[N]) { if (NULL == basis) { @@ -69,10 +69,10 @@ static complex float* square_basis(bool upper_triag, int N, long sqr_bas_dims[N] assert(1 == bas_dims[7]); - long bas_dimsT[N]; + bart_dim_t bas_dimsT[N]; md_transpose_dims(N, 6, 7, bas_dimsT, bas_dims); - md_max_dims(N, ~0UL, sqr_bas_dims, bas_dims, bas_dimsT); + md_max_dims(N, ~UINT64_C(0), sqr_bas_dims, bas_dims, bas_dimsT); sqr_bas_dims[5] = ksp_dims[5]; complex float* sqr = md_alloc_sameplace(N, sqr_bas_dims, CFL_SIZE, basis); @@ -97,7 +97,7 @@ static complex float* square_basis(bool upper_triag, int N, long sqr_bas_dims[N] if (upper_triag) { - long tri_dims[N]; + bart_dim_t tri_dims[N]; complex float* tri = hermite_to_uppertriag(6, 6, 6, N, tri_dims, sqr_bas_dims, sqr); md_free(sqr); @@ -133,23 +133,23 @@ static struct nufft_conf_s psf_conf(bool periodic, bool lowmem, bool vptr) } /* The adjoint transform of ones, which is what a point spread function is. */ -static complex float* psf_int(int N, const long img_dims[N], const long trj_dims[N], const complex float* traj, - const long bas_dims[N], const complex float* basis, - const long wgh_dims[N], const complex float* weights, +static complex float* psf_int(int N, const bart_dim_t img_dims[N], const bart_dim_t trj_dims[N], const complex float* traj, + const bart_dim_t bas_dims[N], const complex float* basis, + const bart_dim_t wgh_dims[N], const complex float* weights, bool periodic, bool lowmem, bool upper_triag) { - long ksp_dims[N]; + bart_dim_t ksp_dims[N]; md_select_dims(N, ~MD_BIT(0), ksp_dims, trj_dims); if (NULL != weights) - md_max_dims(N, ~0UL, ksp_dims, ksp_dims, wgh_dims); + md_max_dims(N, ~UINT64_C(0), ksp_dims, ksp_dims, wgh_dims); - long sqr_bas_dims[N]; + bart_dim_t sqr_bas_dims[N]; complex float* sqr_basis = square_basis(upper_triag, N, sqr_bas_dims, bas_dims, basis, ksp_dims); complex float* sqr_weights = square_weights(N, wgh_dims, weights); - long img_dims2[N]; + bart_dim_t img_dims2[N]; md_copy_dims(N, img_dims2, img_dims); if (upper_triag) { @@ -185,9 +185,9 @@ static complex float* psf_int(int N, const long img_dims[N], const long trj_dims return psf; } -complex float* compute_psf(int N, const long img_dims[N], const long trj_dims[N], const complex float* traj, - const long bas_dims[N], const complex float* basis, - const long wgh_dims[N], const complex float* weights, +complex float* compute_psf(int N, const bart_dim_t img_dims[N], const bart_dim_t trj_dims[N], const complex float* traj, + const bart_dim_t bas_dims[N], const complex float* basis, + const bart_dim_t wgh_dims[N], const complex float* weights, bool periodic, bool lowmem) { return psf_int(N, img_dims, trj_dims, traj, bas_dims, basis, wgh_dims, weights, @@ -196,16 +196,16 @@ complex float* compute_psf(int N, const long img_dims[N], const long trj_dims[N] /* On the grid twice over, which is where a convolution the size of the image * has room to be one. */ -complex float* compute_psf2(int N, const long psf_dims[N + 1], unsigned long flags, const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, const long wgh_dims[N + 1], const complex float* weights, +complex float* compute_psf2(int N, const bart_dim_t psf_dims[N + 1], bart_flags_t flags, const bart_dim_t trj_dims[N + 1], const complex float* traj, + const bart_dim_t bas_dims[N + 1], const complex float* basis, const bart_dim_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag) { int ND = N + 1; - long img_dims[ND]; + bart_dim_t img_dims[ND]; md_select_dims(ND, ~MD_BIT(N + 0), img_dims, psf_dims); - long img2_dims[ND]; + bart_dim_t img2_dims[ND]; md_copy_dims(ND, img2_dims, img_dims); for (int i = 0; i < N; i++) @@ -224,7 +224,7 @@ complex float* compute_psf2(int N, const long psf_dims[N + 1], unsigned long fla complex float* psf = md_alloc_sameplace(ND, psf_dims, CFL_SIZE, traj); - long factors[N]; + bart_dim_t factors[N]; for (int i = 0; i < N; i++) factors[i] = ((img_dims[i] > 1) && (MD_IS_SET(flags, i))) ? 2 : 1; @@ -236,7 +236,7 @@ complex float* compute_psf2(int N, const long psf_dims[N + 1], unsigned long fla return psf; } -static void psf_factors(int N, unsigned long flags, long factors[N], const long dims[N]) +static void psf_factors(int N, bart_flags_t flags, bart_dim_t factors[N], const bart_dim_t dims[N]) { flags = flags & md_nontriv_dims(N, dims); @@ -247,7 +247,7 @@ static void psf_factors(int N, unsigned long flags, long factors[N], const long /* The shift of one set of frequencies, as `nufft.c` computes it. Shared * because the mask a compressed function keeps is gridded at the same shifts * the function itself was decomposed at. */ -void bartorch_psf_shift(int NS, float shift[NS], int N, const long factors[N], int idx) +void bartorch_psf_shift(int NS, float shift[NS], int N, const int64_t factors[N], int idx) { assert(NS <= N); @@ -272,34 +272,34 @@ void bartorch_psf_shift(int NS, float shift[NS], int N, const long factors[N], i * samples reach, and the shape it takes once only those are kept. */ struct psf_packing { - const long* com_dims; - const long* idx; - const long* com_psf_dims; /* the whole compressed function */ - const long* com_psf_dims3; /* one set of frequencies of it */ + const bart_dim_t* com_dims; + const bart_dim_t* idx; + const bart_dim_t* com_psf_dims; /* the whole compressed function */ + const bart_dim_t* com_psf_dims3; /* one set of frequencies of it */ }; static complex float* psf_decomposed(bool to_host, bool real_out, const struct psf_packing* pack, - int N, const long psf_dims[N + 1], unsigned long flags, const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, const long wgh_dims[N + 1], const complex float* weights, + int N, const bart_dim_t psf_dims[N + 1], bart_flags_t flags, const bart_dim_t trj_dims[N + 1], const complex float* traj, + const bart_dim_t bas_dims[N + 1], const complex float* basis, const bart_dim_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag) { assert(to_host || !real_out); int ND = N + 1; - long ksp_dims[ND]; + bart_dim_t ksp_dims[ND]; md_select_dims(ND, ~MD_BIT(0), ksp_dims, trj_dims); ksp_dims[N] = psf_dims[N]; if (NULL != weights) - md_max_dims(ND, ~0UL, ksp_dims, ksp_dims, wgh_dims); + md_max_dims(ND, ~UINT64_C(0), ksp_dims, ksp_dims, wgh_dims); - long sqr_bas_dims[ND]; + bart_dim_t sqr_bas_dims[ND]; complex float* sqr_basis = square_basis(upper_triag, ND, sqr_bas_dims, bas_dims, basis, ksp_dims); complex float* sqr_weights = square_weights(ND, wgh_dims, weights); - long psf_dims2[ND]; + bart_dim_t psf_dims2[ND]; md_copy_dims(ND, psf_dims2, psf_dims); if (upper_triag) { @@ -314,11 +314,11 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p struct nufft_conf_s conf = psf_conf(periodic, lowmem, is_vptr(traj)); - long trj_dims2[ND]; + bart_dim_t trj_dims2[ND]; md_copy_dims(ND, trj_dims2, trj_dims); trj_dims2[N] = psf_dims2[N]; - long factors[ND]; + bart_dim_t factors[ND]; psf_factors(ND, flags, factors, psf_dims); complex float tp[trj_dims2[N]][trj_dims2[0]]; @@ -332,9 +332,9 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p tp[k][j] = (1 != psf_dims2[j] ? 0.5 * psf_dims2[j] : 0.) + shift[j]; } - long ksp_dims2[ND]; - long psf_dims3[ND]; - long trj_dims3[ND]; + bart_dim_t ksp_dims2[ND]; + bart_dim_t psf_dims3[ND]; + bart_dim_t trj_dims3[ND]; md_select_dims(ND, ~MD_BIT(N), ksp_dims2, ksp_dims); md_select_dims(ND, ~MD_BIT(N), psf_dims3, psf_dims2); @@ -348,14 +348,14 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p * which is far cheaper than allocating it page-locked. A real function * is made real here, an entry at a time, so what crosses is half of what * it would be. */ - const long* whole_dims = (NULL != pack) ? pack->com_psf_dims : psf_dims; + const bart_dim_t* whole_dims = (NULL != pack) ? pack->com_psf_dims : psf_dims; size_t out_size = real_out ? FL_SIZE : CFL_SIZE; complex float* psf = to_host - ? bartorch_host_alloc(md_calc_size(ND, whole_dims) * (long)out_size, 0) + ? bartorch_host_alloc(md_calc_size(ND, whole_dims) * (bart_dim_t)out_size, 0) : md_alloc_sameplace(ND, whole_dims, CFL_SIZE, traj); - long psf_coset = md_calc_size(ND, (NULL != pack) ? pack->com_psf_dims3 : psf_dims3); + bart_dim_t psf_coset = md_calc_size(ND, (NULL != pack) ? pack->com_psf_dims3 : psf_dims3); /* One coefficient of the function at a time. * @@ -364,20 +364,20 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p * together: at 256^3 with four coefficients that is ten images where * one would do. Each entry is its own gridding of the samples weighted * by its own pair of basis coefficients. */ - long one_dims[ND]; - long one_bas_dims[ND]; + bart_dim_t one_dims[ND]; + bart_dim_t one_bas_dims[ND]; md_copy_dims(ND, one_dims, psf_dims3); md_copy_dims(ND, one_bas_dims, sqr_bas_dims); - long pairs = psf_dims3[COEFF_DIM]; + bart_dim_t pairs = psf_dims3[COEFF_DIM]; one_dims[COEFF_DIM] = 1; one_bas_dims[COEFF_DIM] = 1; - long pair_stride = md_calc_size(ND, one_dims); - long basis_stride = (NULL == sqr_basis) ? 0 : md_calc_size(ND, one_bas_dims); - long out_pair_stride = (NULL != pack) ? md_calc_size(ND, pack->com_psf_dims3) / pairs : pair_stride; + bart_stride_t pair_stride = md_calc_size(ND, one_dims); + bart_stride_t basis_stride = (NULL == sqr_basis) ? 0 : md_calc_size(ND, one_bas_dims); + bart_stride_t out_pair_stride = (NULL != pack) ? md_calc_size(ND, pack->com_psf_dims3) / pairs : pair_stride; /* One transform for the whole function, over the trajectory as it is, * with neither weights nor basis: both are per-sample factors and go @@ -389,8 +389,8 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p struct linop_s* op = nufft_create2(ND, ksp_dims2, one_dims, trj_dims3, traj, NULL, NULL, NULL, NULL, conf); - long kstrs[ND]; - long tstrs[ND]; + bart_stride_t kstrs[ND]; + bart_stride_t tstrs[ND]; md_calc_strides(ND, kstrs, ksp_dims2, CFL_SIZE); md_calc_strides(ND, tstrs, trj_dims3, CFL_SIZE); @@ -406,7 +406,7 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p complex float* ramp_h = md_alloc(ND, one_dims, CFL_SIZE); complex float* ramp = on_device ? md_alloc_sameplace(ND, one_dims, CFL_SIZE, traj) : ramp_h; - long one_com[ND]; + bart_dim_t one_com[ND]; if (NULL != pack) { @@ -414,7 +414,7 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p one_com[COEFF_DIM] = 1; } - const long* made_dims = (NULL != pack) ? one_com : one_dims; + const bart_dim_t* made_dims = (NULL != pack) ? one_com : one_dims; complex float* packed = (NULL != pack) ? md_alloc_sameplace(ND, one_com, CFL_SIZE, traj) : NULL; float* packed_real = real_out ? md_alloc_sameplace(ND, made_dims, FL_SIZE, traj) : NULL; @@ -466,7 +466,7 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p * one set of frequencies: one image, gridded, transformed, and * -- where a compressed function was asked for -- reduced to the * places the samples reach before the next one is made. */ - for (long q = 0; q < pairs; q++) { + for (bart_dim_t q = 0; q < pairs; q++) { if (NULL != sqr_basis) md_zmulc2(ND, ksp_dims2, kstrs, kern_q, kstrs, kern, @@ -522,8 +522,8 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p return psf; } -complex float* compute_psf2_decomposed(int N, const long psf_dims[N + 1], unsigned long flags, const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, const long wgh_dims[N + 1], const complex float* weights, +complex float* compute_psf2_decomposed(int N, const bart_dim_t psf_dims[N + 1], bart_flags_t flags, const bart_dim_t trj_dims[N + 1], const complex float* traj, + const bart_dim_t bas_dims[N + 1], const complex float* basis, const bart_dim_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag) { return psf_decomposed(false, false, NULL, N, psf_dims, flags, trj_dims, traj, bas_dims, basis, @@ -532,11 +532,11 @@ complex float* compute_psf2_decomposed(int N, const long psf_dims[N + 1], unsign /* The same function, left where the card is not: neither it nor any set of * frequencies but the one being made is ever resident. */ -complex float* bartorch_psf_to_host(int N, const long psf_dims[N + 1], unsigned long flags, const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, const long wgh_dims[N + 1], const complex float* weights, +complex float* bartorch_psf_to_host(int N, const int64_t psf_dims[N + 1], uint64_t flags, const int64_t trj_dims[N + 1], const complex float* traj, + const int64_t bas_dims[N + 1], const complex float* basis, const int64_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag, - const long com_dims[N + 1], const long* idx, - const long com_psf_dims[N + 1], const long com_psf_dims3[N + 1], int real) + const int64_t com_dims[N + 1], const int64_t* idx, + const int64_t com_psf_dims[N + 1], const int64_t com_psf_dims3[N + 1], int real) { struct psf_packing pack = { com_dims, idx, com_psf_dims, com_psf_dims3 }; diff --git a/tests/test_abi.py b/tests/test_abi.py index 6a799b15..aa91179c 100644 --- a/tests/test_abi.py +++ b/tests/test_abi.py @@ -9,6 +9,7 @@ import ast import importlib +import re import sys from pathlib import Path @@ -104,3 +105,11 @@ def test_the_generator_refuses_a_type_it_has_not_been_taught(): def test_bartorch_imports_without_naming_ctypes_at_the_top_level(): """The package's own surface is shapes and tensors.""" importlib.import_module("bartorch") + + +def test_the_header_names_no_type_whose_width_is_the_platforms(): + # long is 32 bits on Windows and 64 elsewhere, so a shape or a stride + # carried in one would be truncated there and not here. + code = gen_abi.strip_directives(gen_abi.strip_comments(gen_abi.HEADER.read_text())) + assert not re.search(r"\blong\b", code) + From 1401687eabab57f65744ab8eb8b3af8e6c374919 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 15:19:14 +0000 Subject: [PATCH 02/10] Build and test on Windows with clang The library builds with clang from MSYS2's CLANG64 environment and loads into official CPython: the Blocks runtime is linked in as objects, the toolchain's runtime statically, and OpenMP is bound to the libiomp5md.dll torch loads, through an import library made from src/csrc/compat/libiomp5md.def, since LLVM's runtime refuses to start beside Intel's. BART's C runtime assertions reach its error path through the import slots MinGW's assert() calls. CI tests Python 3.10 and 3.14 on Linux (GCC 14 and clang), macOS and Windows, prints the data model on Windows, and checks that the DLL imports nothing from MSYS2. The wheel workflow builds one py3 wheel per platform, Windows included, and tests it on 3.10 and 3.14. build_info() reports sizeof(long), and a test holds it to 4 on Windows and 8 elsewhere. Windows is CPU only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- .github/workflows/publish.yml | 32 +++++++++-- .github/workflows/tests.yml | 74 ++++++++++++++++++++++---- AGENTS.md | 23 +++++--- CMakeLists.txt | 65 ++++++++++++++++++++-- docs/guides/developer/prerequisites.md | 8 ++- docs/guides/developer/pull-requests.md | 4 +- docs/guides/user/prerequisites.md | 4 +- pyproject.toml | 4 +- src/bartorch/_backend.py | 13 +++-- src/bartorch/_finufft.py | 2 +- src/bartorch/_lib.py | 18 +++++-- src/csrc/abi/api.c | 28 +++++++++- src/csrc/abi/cuda.c | 11 ++++ src/csrc/compat/libiomp5md.def | 68 +++++++++++++++++++++++ src/csrc/include/bartorch.h | 4 ++ tests/test_abi.py | 8 +++ 16 files changed, 326 insertions(+), 40 deletions(-) create mode 100644 src/csrc/compat/libiomp5md.def diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8297ef88..ca327c42 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -54,13 +54,34 @@ jobs: # ninja and a C++ compiler from someone installing a wheel. aarch64 # is served by the sdist, where that toolchain is already the price of # entry. - os: [ubuntu-latest, macos-latest] + # + # Windows is CPU only: clang from MSYS2's CLANG64 environment, which + # links against the Universal C Runtime official CPython uses. + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 with: submodules: recursive # setuptools-scm reads the version from the tag history. fetch-depth: 0 + - name: Install clang, CMake and Ninja (Windows) + if: runner.os == 'Windows' + id: msys2 + uses: msys2/setup-msys2@v2 + with: + msystem: CLANG64 + update: true + install: >- + mingw-w64-clang-x86_64-clang + mingw-w64-clang-x86_64-cmake + mingw-w64-clang-x86_64-ninja + mingw-w64-clang-x86_64-openmp + mingw-w64-clang-x86_64-llvm-tools + - name: Put CLANG64 on the path (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + "${{ steps.msys2.outputs.msys2-location }}\clang64\bin" >> $env:GITHUB_PATH - uses: pypa/cibuildwheel@v2.23 env: # One build per platform: the wheel is tagged py3 and carries no @@ -75,6 +96,10 @@ jobs: # wheel in any case: torch publishes no i686 build, and it is a hard # dependency here. CIBW_ARCHS_LINUX: x86_64 + CIBW_ARCHS_WINDOWS: AMD64 + # The library imports the system's DLLs and the OpenMP runtime torch + # carries, so there is nothing to vendor and no repair step. + CIBW_ENVIRONMENT_WINDOWS: CC=clang CXX=clang++ CMAKE_GENERATOR=Ninja # BART's nested functions need heap trampolines -- GCC 14 -- or # clang's Blocks. cibuildwheel's default Linux image has neither: # manylinux2014 is CentOS 7, whose GCC is 10 and whose `clang` is @@ -104,8 +129,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + os: [ubuntu-latest, macos-latest, windows-latest] + # The oldest and the newest Python the package supports. + python: ["3.10", "3.14"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bbef760a..9c79f073 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,11 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: - # libbartorch is a clang build: BART's nested functions become Blocks. + # BART's nested functions become Blocks under clang and heap trampolines + # under GCC 14. Each platform runs the oldest and the newest Python the + # package supports; the wheel workflow tests the wheel on both as well. + # Windows is clang from MSYS2's CLANG64 environment, linked against the + # Universal C Runtime that official CPython uses. tests: strategy: fail-fast: false @@ -18,20 +22,29 @@ jobs: include: - os: ubuntu-latest python-version: "3.10" - cc: clang + cc: gcc-14 + extras: "[mkl]" - os: ubuntu-latest - python-version: "3.12" + python-version: "3.14" cc: clang # One job measures coverage for Codecov; the rest only test. coverage: true - - os: ubuntu-latest - python-version: "3.12" - cc: gcc-14 - extras: "[mkl]" - os: macos-latest - python-version: "3.12" + python-version: "3.10" + cc: clang + - os: macos-latest + python-version: "3.14" + cc: clang + - os: windows-latest + python-version: "3.10" + cc: clang + - os: windows-latest + python-version: "3.14" cc: clang runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash # Codecov takes the upload on GitHub's OIDC token, so no secret is needed. permissions: contents: read @@ -57,6 +70,38 @@ jobs: if: runner.os == 'macOS' run: brew install cmake + # omp.h comes from the openmp package; the library links against the + # OpenMP runtime torch carries, not against MSYS2's. + - name: Install clang, CMake and Ninja (Windows) + if: runner.os == 'Windows' + id: msys2 + uses: msys2/setup-msys2@v2 + with: + msystem: CLANG64 + update: true + install: >- + mingw-w64-clang-x86_64-clang + mingw-w64-clang-x86_64-cmake + mingw-w64-clang-x86_64-ninja + mingw-w64-clang-x86_64-openmp + mingw-w64-clang-x86_64-llvm-tools + + - name: Put CLANG64 on the path (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + "${{ steps.msys2.outputs.msys2-location }}\clang64\bin" >> $env:GITHUB_PATH + "CMAKE_GENERATOR=Ninja" >> $env:GITHUB_ENV + + # long is 32 bits here, which is the case the fixed-width types exist + # for; this prints it, and tests/test_abi.py holds the library to it. + - name: The data model (Windows) + if: runner.os == 'Windows' + run: | + printf '#include \nint main(void) { printf("long %%d, long long %%d, void* %%d\\n", (int)sizeof(long), (int)sizeof(long long), (int)sizeof(void*)); return 4 != sizeof(long); }\n' > model.c + clang model.c -o model.exe + ./model.exe + - name: Install torch (CPU) and the build backend run: | pip install scikit-build-core setuptools-scm numpy scipy pytest pytest-cov numpydoc @@ -73,7 +118,7 @@ jobs: # Linux only because that index publishes no macOS wheel -- there both # come from PyPI, which is consistent with itself. - name: Install torchvision from the same index as torch - if: runner.os == 'Linux' + if: runner.os != 'macOS' run: pip install torchvision --index-url https://download.pytorch.org/whl/cpu - name: Build and install bartorch @@ -82,6 +127,15 @@ jobs: CXX: ${{ matrix.cc == 'gcc-14' && 'g++-14' || 'clang++' }} run: pip install -e ".${{ matrix.extras }}" --no-build-isolation -v + # The library imports the system's DLLs and torch's OpenMP runtime and + # nothing from MSYS2, so it loads into official CPython without it. + - name: Nothing from MSYS2 in the library (Windows) + if: runner.os == 'Windows' + run: | + lib=$(python -c "from bartorch._lib import library_path; print(library_path())") + llvm-readobj --coff-imports "$lib" | grep 'Name:' | sort -u | tee imports.txt + ! grep -Ei 'libomp|libc\+\+|libunwind|libwinpthread|msys' imports.txt + # Both wheels carry their own copy of LLVM's OpenMP runtime, and the # runtime ends the process rather than run beside a second copy. The # repair is bartorch's own, on the first NUFFT, so this reaches it the @@ -107,6 +161,8 @@ jobs: logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") import bartorch print("build:", bartorch.build_info()) + import ctypes, sys + print("platform:", sys.platform, "sizeof(long):", ctypes.sizeof(ctypes.c_long)) print("backends:", bartorch.backend_sources()) try: from bartorch import _finufft diff --git a/AGENTS.md b/AGENTS.md index 310c08b2..76c609c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -620,10 +620,22 @@ configure time. BART's own `NOEXEC_STACK` workaround does not help here: it parses a trampoline layout GCC emits only for non-PIC executables, not for a shared library. Both compilers are built and tested in CI. -Windows is not a platform here. BART does not build on it, and nothing in -this repository carries a path toward one: no `.dll` among the names the -loader tries, no `__declspec(dllexport)`, no `win32` branch picking a -different library. WSL2 is a Linux install and is the answer. +**Windows** is LLP64: `long` is 32 bits, so BART keeps every extent, stride +and flag set in the fixed-width types of `misc/dimtypes.h` (`bart_dim_t`, +`bart_stride_t`, `bart_flags_t`, all 64 bits), and the ABI spells them +`int64_t` and `uint64_t`; `api.c` asserts the two agree, and +`tests/test_abi.py` holds the header to having no `long` and `build_info()`'s +`long=` to the platform. The library is clang from MSYS2's CLANG64 +environment: Blocks from the vendored runtime, linked in as objects because +clang declares it `dllimport`; everything the toolchain would bring as a DLL +linked statically; the Universal C Runtime official CPython uses. OpenMP is +torch's: torch loads Intel's `libiomp5md.dll`, LLVM's runtime refuses to start +beside it, and the two share an ABI, so the library links against an import +library made from `src/csrc/compat/libiomp5md.def` and the loader binds it to +the copy torch has loaded. A build without OpenMP there is refused unless +`-DBARTORCH_OPENMP=OFF` asks for it. BART's POSIX gaps -- `getsubopt`, +C11 threads, `SIGSTOP`, `SIGPIPE`, `readlink` -- are patches on the fork, and +named pipes are off (`NO_FIFO`). CPU only: there is no CUDA build on Windows. The compiler's own runtime is linked statically on Linux, because otherwise the toolchain's floor becomes the target system's: a GCC 14 build asks @@ -1113,8 +1125,7 @@ of it is written. What comes back is finite stack memory. `_call`'s `_PRECONDITIONS` refuses those combinations before the command runs, which is where any further "BART does not define this" case belongs. -Windows is not on this list because it is not a target: BART does not build -there, and WSL2 is a Linux install like any other. +A CUDA build on Windows. A tool that takes device memory as it stands. BART guards the host reads that would break -- `estimate_im_dims` copies to the host when it is handed one -- diff --git a/CMakeLists.txt b/CMakeLists.txt index e1cd74c8..74ddf3c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,7 +77,16 @@ set(BART_EXTRA_LIBS "") if(CMAKE_C_COMPILER_ID MATCHES "Clang") list(APPEND BART_C_OPTIONS -fblocks) set(BARTORCH_NESTED "clang-blocks") - if(NOT APPLE) + if(WIN32) + # clang declares the runtime dllimport on Windows, and the linker + # resolves an import to a definition in the same image only when that + # definition is already present: objects are, archive members are not + # pulled in for it. + add_library(blocksruntime OBJECT + external/blocksruntime/runtime.c + external/blocksruntime/data.c) + list(APPEND BART_EXTRA_LIBS $) + elseif(NOT APPLE) add_library(blocksruntime STATIC external/blocksruntime/runtime.c external/blocksruntime/data.c) @@ -109,6 +118,11 @@ endif() # the way BART's own fallback does. Tested rather than guessed, because # defining it where it does exist collides with the system header. include(CheckCSourceCompiles) +# The data model, reported in build_info: long is 64 bits on LP64 (Linux, +# macOS) and 32 on LLP64 (Windows), and nothing BART or the ABI stores in an +# array's shape depends on it. +include(CheckTypeSize) +check_type_size(long BARTORCH_SIZEOF_LONG LANGUAGE C) # Through , because that is where BART's translation units get it: # glibc typedefs it in , which and # pull in and does not on every glibc, and clang does not offer it as @@ -125,7 +139,14 @@ endif() # would carry the toolchain's floor into every target system -- a GCC 14 build # asks for GCC_14.0.0 from libgcc_s, which no released distribution has -- and # that is a load-time failure, not a fallback. Static leaves libc and libgomp. -if(NOT APPLE) +# +# On Windows the same goes for everything the MinGW toolchain would otherwise +# bring as a DLL -- winpthreads and the compiler's runtime -- because Python +# does not search PATH for a library's dependencies. What is left is the +# Universal C Runtime and the system's own libraries. +if(WIN32) + list(APPEND BART_LINK_OPTIONS -static) +elseif(NOT APPLE) list(APPEND BART_LINK_OPTIONS -static-libgcc -static-libstdc++) endif() @@ -164,7 +185,28 @@ if(BARTORCH_CUDA) message(STATUS "CUDA ${CUDAToolkit_VERSION}: architectures ${BARTORCH_CUDA_ARCH_SPEC}") endif() -if(BARTORCH_OPENMP) +if(BARTORCH_OPENMP AND WIN32) + # torch loads Intel's OpenMP runtime on Windows, and LLVM's refuses to + # initialise beside it (OMP: Error #15), as on macOS. The two share an + # ABI, so the library is bound to the runtime torch has already loaded: + # clang compiles with -fopenmp, and the link is against an import library + # for libiomp5md.dll made from the entry points clang calls. The loader + # resolves that name to the copy in the process. + if(NOT CMAKE_C_COMPILER_ID MATCHES "Clang") + message(FATAL_ERROR "OpenMP on Windows is built with clang; pass -DBARTORCH_OPENMP=OFF to build without it") + endif() + find_program(BARTORCH_DLLTOOL NAMES llvm-dlltool dlltool x86_64-w64-mingw32-dlltool REQUIRED) + set(_iomp_def "${CMAKE_CURRENT_SOURCE_DIR}/src/csrc/compat/libiomp5md.def") + set(_iomp_lib "${CMAKE_CURRENT_BINARY_DIR}/libiomp5md.dll.a") + add_custom_command(OUTPUT "${_iomp_lib}" + COMMAND "${BARTORCH_DLLTOOL}" -m i386:x86-64 -d "${_iomp_def}" -l "${_iomp_lib}" -D libiomp5md.dll + DEPENDS "${_iomp_def}" + VERBATIM) + add_custom_target(iomp5_import DEPENDS "${_iomp_lib}") + set(OpenMP_C_FOUND TRUE) + list(APPEND BART_C_OPTIONS -fopenmp) + list(APPEND BART_EXTRA_LIBS "${_iomp_lib}") +elseif(BARTORCH_OPENMP) find_package(OpenMP COMPONENTS C) if(OpenMP_C_FOUND) list(APPEND BART_EXTRA_LIBS OpenMP::OpenMP_C) @@ -230,7 +272,12 @@ file(GLOB _module_dirs LIST_DIRECTORIES true CONFIGURE_DEPENDS "${BART_SRC}/*") foreach(_dir ${_module_dirs}) if(IS_DIRECTORY "${_dir}") get_filename_component(_mod "${_dir}" NAME) - if(NOT _mod MATCHES "^(ismrm|lapacke|win)$") + # `win/` is BART's own port of mmap, fmemopen, getsubopt and the rest + # of what the Windows C runtime lacks, and is built there alone. + if(_mod STREQUAL "win" AND NOT WIN32) + continue() + endif() + if(NOT _mod MATCHES "^(ismrm|lapacke)$") file(GLOB _srcs CONFIGURE_DEPENDS "${_dir}/*.c") list(APPEND BART_SOURCES ${_srcs}) endif() @@ -407,7 +454,11 @@ target_compile_definitions(bartorch PRIVATE NO_PNG _GNU_SOURCE ${BART_C_DEFS} - BARTORCH_BUILD_INFO="bart=${BART_VERSION},compiler=${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION},nested=${BARTORCH_NESTED},openmp=${OpenMP_C_FOUND},cuda=${BARTORCH_CUDA}") + # Windows: a 64-bit off_t, so a file past 2 GiB can be mapped; BART's own + # functions linked into this library rather than imported from its DLL + # (misc/dllspec.h); and no named pipes, which are a POSIX file type. + $<$:_FILE_OFFSET_BITS=64;BARTLIB_STATIC;NO_FIFO> + BARTORCH_BUILD_INFO="bart=${BART_VERSION},compiler=${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION},nested=${BARTORCH_NESTED},openmp=${OpenMP_C_FOUND},cuda=${BARTORCH_CUDA},long=${BARTORCH_SIZEOF_LONG}") target_compile_options(bartorch PRIVATE $<$:${BART_C_OPTIONS}> @@ -436,6 +487,10 @@ target_link_libraries(bartorch PRIVATE Threads::Threads ${BART_EXTRA_LIBS}) target_link_libraries(bartorch PRIVATE m) +if(TARGET iomp5_import) + add_dependencies(bartorch iomp5_import) +endif() + set_target_properties(bartorch PROPERTIES OUTPUT_NAME bartorch PREFIX "lib") diff --git a/docs/guides/developer/prerequisites.md b/docs/guides/developer/prerequisites.md index 8980dd2f..fcf3d29f 100644 --- a/docs/guides/developer/prerequisites.md +++ b/docs/guides/developer/prerequisites.md @@ -15,5 +15,9 @@ suite is run from `src/` without installing, install it separately skip, because the substitution declining is an error. `pip install mkl deepinv` enables the tests that need MKL and the DeepInverse adapter. -Windows is not a target: BART does not build on it, and WSL2 is used as a Linux -environment. +On Windows the compiler is clang from MSYS2's CLANG64 environment, with the +packages `mingw-w64-clang-x86_64-{clang,cmake,ninja,openmp,llvm-tools}`, +`clang64/bin` on `PATH` and `CMAKE_GENERATOR=Ninja`. The build links against +the OpenMP runtime PyTorch installs rather than MSYS2's, and loads into the +official CPython with nothing from MSYS2 at run time. There is no CUDA build on +Windows. diff --git a/docs/guides/developer/pull-requests.md b/docs/guides/developer/pull-requests.md index 40edf072..88ed04b4 100644 --- a/docs/guides/developer/pull-requests.md +++ b/docs/guides/developer/pull-requests.md @@ -11,8 +11,8 @@ template: | Validation | The commands run and their results; for a numerical change, the reference and the tolerance; hardware or optional dependencies that were not available | | Public API and documentation | Changes to public behaviour and the documentation updated for them | -CI runs the test matrix (Linux with clang and GCC 14, macOS, and a CUDA build -without a device), the lint and spelling checks, and two documentation builds: +CI runs the test matrix (Linux with GCC 14 and clang, macOS and Windows with +clang, each on Python 3.10 and 3.14, and a CUDA build without a device), the lint and spelling checks, and two documentation builds: the reference without the compiled library, and the executed gallery, which is published from `main` and from release tags. A pull request is merged when CI passes and a maintainer has approved it. diff --git a/docs/guides/user/prerequisites.md b/docs/guides/user/prerequisites.md index c86ef41d..a29cd6c9 100644 --- a/docs/guides/user/prerequisites.md +++ b/docs/guides/user/prerequisites.md @@ -4,7 +4,7 @@ | Requirement | Version | | --- | --- | -| Python | 3.10 or later; wheels are tested on 3.10 to 3.14 | +| Python | 3.10 or later; each wheel is tested on 3.10 and 3.14 | | PyTorch | 2.2 or later, CPU or CUDA build | | NumPy, SciPy | NumPy 1.24 and SciPy 1.10 or later; MRI-NUFFT, a dependency, raises these to NumPy 2.2 and SciPy 1.13 | | FINUFFT | 2.2 or later, installed as a dependency | @@ -18,7 +18,7 @@ | macOS 11 or later on Apple silicon | Wheel | CPU only; see {ref}`macos-openmp` | | Linux aarch64 | Source distribution | BART is compiled on installation; FINUFFT publishes no wheel for this platform and is built from source as well | | macOS on Intel | Source distribution | BART is compiled on installation; FINUFFT releases after 2.4.0 have no wheel for this platform and are built from source as well | -| Windows | Not supported | BART does not build on Windows; WSL2 provides a Linux environment | +| Windows 10 or later on x86-64 | Wheel | CPU only; OpenMP is the runtime PyTorch installs | A source installation needs the toolchain listed under {ref}`source-builds`. Apple MPS devices are not supported; the device paths are CPU and CUDA. diff --git a/pyproject.toml b/pyproject.toml index 62c1eef3..11c6f8ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,8 +79,8 @@ deepinv = [ ] # MKL covers every BLAS and LAPACK routine BART calls and is about twice as # fast as the alternatives on the work that leans on LAPACK. There is no MKL -# wheel for macOS, where SciPy's OpenBLAS and Accelerate serve instead, and -# Windows is not a target: BART does not build there. +# wheel for macOS, where SciPy's OpenBLAS and Accelerate serve instead; on +# Windows SciPy's OpenBLAS serves. mkl = [ "mkl>=2024; sys_platform == 'linux' and platform_machine == 'x86_64'", ] diff --git a/src/bartorch/_backend.py b/src/bartorch/_backend.py index 640a15eb..8396b8fc 100644 --- a/src/bartorch/_backend.py +++ b/src/bartorch/_backend.py @@ -84,9 +84,11 @@ def _mkl_library() -> Path | None: "libmkl_rt.so.2", "libmkl_rt.so.3", "libmkl_rt.so", + "mkl_rt.2.dll", + "mkl_rt.3.dll", ) for root in sorted(roots): - for sub in ("lib", "lib64"): + for sub in ("lib", "lib64", "Library/bin"): for name in names: candidate = root / sub / name if candidate.exists(): @@ -100,7 +102,9 @@ def _torch_library() -> Path | None: except ImportError: return None libdir = Path(torch.__file__).resolve().parent / "lib" - names = ["libtorch_cpu.dylib"] if sys.platform == "darwin" else ["libtorch_cpu.so"] + names = {"darwin": ["libtorch_cpu.dylib"], "win32": ["torch_cpu.dll"]}.get( + sys.platform, ["libtorch_cpu.so"] + ) for name in names: if (libdir / name).exists(): return libdir / name @@ -128,7 +132,10 @@ def _torch_providers() -> list[_Provider | None]: found.append( _open("Accelerate", "/System/Library/Frameworks/Accelerate.framework/Accelerate") ) - found.append(_open("process", None)) + # Windows has no handle on the process's own symbols: each DLL exports + # its own, and torch_cpu.dll is asked above. + if sys.platform != "win32": + found.append(_open("process", None)) return found diff --git a/src/bartorch/_finufft.py b/src/bartorch/_finufft.py index 6ffa491f..2743b322 100644 --- a/src/bartorch/_finufft.py +++ b/src/bartorch/_finufft.py @@ -68,7 +68,7 @@ def _library_path(package: str, stem: str) -> str | None: except ImportError: return None here = Path(module.__file__).resolve().parent - for name in (f"lib{stem}.so", f"lib{stem}.dylib"): + for name in (f"lib{stem}.so", f"lib{stem}.dylib", f"lib{stem}.dll"): candidate = here / name if candidate.exists(): return str(candidate) diff --git a/src/bartorch/_lib.py b/src/bartorch/_lib.py index 0cf661bf..bedd8ad2 100644 --- a/src/bartorch/_lib.py +++ b/src/bartorch/_lib.py @@ -27,13 +27,25 @@ def _library_names() -> list[str]: - # No Windows name: BART does not build there, so neither does this, and - # WSL2 is a Linux install like any other. if sys.platform == "darwin": return ["libbartorch.dylib"] + if sys.platform == "win32": + return ["libbartorch.dll"] return ["libbartorch.so"] +def _load(path: Path) -> ctypes.CDLL: + if sys.platform != "win32": + return ctypes.CDLL(str(path)) + # The Windows library imports libiomp5md.dll, the OpenMP runtime torch + # carries and loads; importing torch first makes it the copy the loader + # binds to, and torch's directory is where it is found otherwise. + import torch + + os.add_dll_directory(str(Path(torch.__file__).resolve().parent / "lib")) + return ctypes.CDLL(str(path)) + + def _package_dirs() -> list[Path]: """Every directory the package's files are in. @@ -74,7 +86,7 @@ def library() -> ctypes.CDLL: global _lib, _path if _lib is None: _path = _find_library() - _lib = bind(ctypes.CDLL(str(_path))) + _lib = bind(_load(_path)) return _lib diff --git a/src/csrc/abi/api.c b/src/csrc/abi/api.c index 05b7916c..4bc3affd 100644 --- a/src/csrc/abi/api.c +++ b/src/csrc/abi/api.c @@ -41,9 +41,33 @@ extern int bart_command(int len, char* buf, int argc, char* argv[]); * Apple's libc calls __assert_rtn(func, file, line, expr) -- a different name, * a different order, and a different type for the line. Answering only the * first left every one of BART's assertions aborting on macOS, which is a - * process death where every other platform gets an exception. + * process death where every other platform gets an exception. MinGW's + * assert() calls the C runtime's _assert, or _wassert under the Universal C + * Runtime, through the import table, so the import-table slots are what is + * answered there. */ -#ifdef __APPLE__ +#if defined(_WIN32) + +#include + +__attribute__((noreturn)) +static void crt_assert(const char* assertion, const char* file, unsigned line) +{ + error("Assertion '%s' failed in %s:%u\n", assertion, file, line); +} + +__attribute__((noreturn)) +static void crt_wassert(const wchar_t* assertion, const wchar_t* file, unsigned line) +{ + error("Assertion '%ls' failed in %ls:%u\n", assertion, file, line); +} + +/* declares both dllimport, so a caller reaches them through these + * slots; defined here, the import library's are never linked. */ +void (*__imp__assert)(const char*, const char*, unsigned) = crt_assert; +void (*__imp__wassert)(const wchar_t*, const wchar_t*, unsigned) = crt_wassert; + +#elif defined(__APPLE__) __attribute__((noreturn)) void __assert_rtn(const char* function, const char* file, int line, const char* assertion); diff --git a/src/csrc/abi/cuda.c b/src/csrc/abi/cuda.c index 49b401a0..c5ff3419 100644 --- a/src/csrc/abi/cuda.c +++ b/src/csrc/abi/cuda.c @@ -16,7 +16,12 @@ #include #include #include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#else #include +#endif #include "include/bartorch.h" @@ -40,7 +45,13 @@ struct prefault { static void* prefault_run(void* arg) { struct prefault* p = arg; +#ifdef _WIN32 + SYSTEM_INFO info; + GetSystemInfo(&info); + bart_dim_t page = info.dwPageSize; +#else bart_dim_t page = sysconf(_SC_PAGESIZE); +#endif for (bart_dim_t i = 0; i < p->size; i += page) p->ptr[i] = p->ptr[i]; diff --git a/src/csrc/compat/libiomp5md.def b/src/csrc/compat/libiomp5md.def new file mode 100644 index 00000000..81eecef5 --- /dev/null +++ b/src/csrc/compat/libiomp5md.def @@ -0,0 +1,68 @@ +; The OpenMP entry points clang's code calls, exported by the Intel OpenMP +; runtime torch loads on Windows. An import library made from this file +; binds the library to that runtime, so the process has one OpenMP runtime. +; A name clang emits and this list lacks is a link error. +LIBRARY libiomp5md.dll +EXPORTS + __kmpc_barrier + __kmpc_critical + __kmpc_critical_with_hint + __kmpc_dispatch_fini_4 + __kmpc_dispatch_fini_4u + __kmpc_dispatch_fini_8 + __kmpc_dispatch_fini_8u + __kmpc_dispatch_init_4 + __kmpc_dispatch_init_4u + __kmpc_dispatch_init_8 + __kmpc_dispatch_init_8u + __kmpc_dispatch_next_4 + __kmpc_dispatch_next_4u + __kmpc_dispatch_next_8 + __kmpc_dispatch_next_8u + __kmpc_end_critical + __kmpc_end_master + __kmpc_end_ordered + __kmpc_end_reduce + __kmpc_end_reduce_nowait + __kmpc_end_serialized_parallel + __kmpc_end_single + __kmpc_flush + __kmpc_for_static_fini + __kmpc_for_static_init_4 + __kmpc_for_static_init_4u + __kmpc_for_static_init_8 + __kmpc_for_static_init_8u + __kmpc_fork_call + __kmpc_global_thread_num + __kmpc_master + __kmpc_omp_taskwait + __kmpc_ordered + __kmpc_push_num_threads + __kmpc_reduce + __kmpc_reduce_nowait + __kmpc_serialized_parallel + __kmpc_single + __kmpc_threadprivate_cached + omp_destroy_lock + omp_destroy_nest_lock + omp_get_ancestor_thread_num + omp_get_dynamic + omp_get_level + omp_get_max_threads + omp_get_nested + omp_get_num_procs + omp_get_num_threads + omp_get_team_size + omp_get_thread_num + omp_get_wtime + omp_in_parallel + omp_init_lock + omp_init_nest_lock + omp_set_dynamic + omp_set_lock + omp_set_nest_lock + omp_set_nested + omp_set_num_threads + omp_test_lock + omp_unset_lock + omp_unset_nest_lock diff --git a/src/csrc/include/bartorch.h b/src/csrc/include/bartorch.h index e5077bf0..162466f6 100644 --- a/src/csrc/include/bartorch.h +++ b/src/csrc/include/bartorch.h @@ -19,7 +19,11 @@ extern "C" { #endif +#ifdef _WIN32 +#define BARTORCH_API __declspec(dllexport) +#else #define BARTORCH_API __attribute__((visibility("default"))) +#endif /* Number of dimensions BART carries for every array. */ #define BARTORCH_DIMS 16 diff --git a/tests/test_abi.py b/tests/test_abi.py index aa91179c..e7cf534a 100644 --- a/tests/test_abi.py +++ b/tests/test_abi.py @@ -8,6 +8,7 @@ """ import ast +import ctypes import importlib import re import sys @@ -113,3 +114,10 @@ def test_the_header_names_no_type_whose_width_is_the_platforms(): code = gen_abi.strip_directives(gen_abi.strip_comments(gen_abi.HEADER.read_text())) assert not re.search(r"\blong\b", code) + +def test_build_info_reports_the_platforms_data_model(): + import bartorch + + info = dict(item.split("=", 1) for item in bartorch.build_info().split(",")) + assert int(info["long"]) == ctypes.sizeof(ctypes.c_long) + assert info["long"] == ("4" if sys.platform == "win32" else "8") From d28b97e045a64f0365bdc6f467b7aaaed52ae184 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 15:20:29 +0000 Subject: [PATCH 03/10] State why MKL's LP64 interface is enough on Windows Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 76c609c0..a36cf3d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,6 +124,12 @@ runtime spinning against BART's, which inside a tool costs several times more than the transform gains. Where no source has DFTI, which is macOS, and for a description DFTI declines, the transform compiled into the library serves instead. On a device it is cuFFT, through BART's own `fft-cuda.c`. +DFTI is taken through MKL's LP64 interface, whose `MKL_LONG` is `long`: 32 +bits on Windows. A length or stride past that is a description DFTI is not +handed -- the loop axis is walked instead, and a transformed axis leaves the +plan to pocketfft -- so LP64 is enough and ILP64 is not needed; the loop is +over BART's own `ptrdiff_t` strides. BLAS and LAPACK take `int` everywhere, +and BART range-checks each extent before a call (`checked_int`). **CUDA is the same ABI.** `-DBARTORCH_CUDA=ON` compiles BART's thirteen `.cu` files with nvcc and links cudart, cuFFT and cuBLAS dynamically, which are the From c7e6d6cb4b8f2fe2f5ceeb2b172299074880342a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 15:28:24 +0000 Subject: [PATCH 04/10] Fix the CUDA callbacks' types and the Windows OpenMP package The LTO-IR callbacks are compiled on their own, without BART's headers, so coset.cuh and grid.cuh spell their indices int64_t from . MSYS2 ships LLVM's OpenMP headers as llvm-openmp. The BART pin moves to the fork's head, which builds from a MINGW64 shell and prints ssize_t with %zd. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- .github/workflows/publish.yml | 2 +- .github/workflows/tests.yml | 2 +- docs/guides/developer/prerequisites.md | 2 +- external/bart | 2 +- src/csrc/ops/coset.cuh | 17 +++++++++-------- src/csrc/ops/fft_callbacks_lto.cu | 10 +++++----- src/csrc/ops/grid.cuh | 6 +++--- 7 files changed, 21 insertions(+), 20 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ca327c42..ce1edfa7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -75,7 +75,7 @@ jobs: mingw-w64-clang-x86_64-clang mingw-w64-clang-x86_64-cmake mingw-w64-clang-x86_64-ninja - mingw-w64-clang-x86_64-openmp + mingw-w64-clang-x86_64-llvm-openmp mingw-w64-clang-x86_64-llvm-tools - name: Put CLANG64 on the path (Windows) if: runner.os == 'Windows' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9c79f073..e93bff54 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -83,7 +83,7 @@ jobs: mingw-w64-clang-x86_64-clang mingw-w64-clang-x86_64-cmake mingw-w64-clang-x86_64-ninja - mingw-w64-clang-x86_64-openmp + mingw-w64-clang-x86_64-llvm-openmp mingw-w64-clang-x86_64-llvm-tools - name: Put CLANG64 on the path (Windows) diff --git a/docs/guides/developer/prerequisites.md b/docs/guides/developer/prerequisites.md index fcf3d29f..e6fe587f 100644 --- a/docs/guides/developer/prerequisites.md +++ b/docs/guides/developer/prerequisites.md @@ -16,7 +16,7 @@ skip, because the substitution declining is an error. `pip install mkl deepinv` enables the tests that need MKL and the DeepInverse adapter. On Windows the compiler is clang from MSYS2's CLANG64 environment, with the -packages `mingw-w64-clang-x86_64-{clang,cmake,ninja,openmp,llvm-tools}`, +packages `mingw-w64-clang-x86_64-{clang,cmake,ninja,llvm-openmp,llvm-tools}`, `clang64/bin` on `PATH` and `CMAKE_GENERATOR=Ninja`. The build links against the OpenMP runtime PyTorch installs rather than MSYS2's, and loads into the official CPython with nothing from MSYS2 at run time. There is no CUDA build on diff --git a/external/bart b/external/bart index 04991fb5..4106c2cc 160000 --- a/external/bart +++ b/external/bart @@ -1 +1 @@ -Subproject commit 04991fb50501a927b7f8ab3743c72ad7cef7e843 +Subproject commit 4106c2ccb7024f1875e4891612835fd92fd96939 diff --git a/src/csrc/ops/coset.cuh b/src/csrc/ops/coset.cuh index 3c5b8f9e..2913f7d5 100644 --- a/src/csrc/ops/coset.cuh +++ b/src/csrc/ops/coset.cuh @@ -19,6 +19,7 @@ #include #include #include +#include #ifndef M_PI #define M_PI 3.14159265358979323846 @@ -26,15 +27,15 @@ struct phase_conf { - bart_dim_t dims[3]; - bart_dim_t tot; - bart_dim_t batch; + int64_t dims[3]; + int64_t tot; + int64_t batch; float shifts[3]; float cn; float scale; }; -static inline struct phase_conf phase_setup(int N, const bart_dim_t dims[], const float shift[3], float scale) +static inline struct phase_conf phase_setup(int N, const int64_t dims[], const float shift[3], float scale) { struct phase_conf c; @@ -55,7 +56,7 @@ static inline struct phase_conf phase_setup(int N, const bart_dim_t dims[], cons c.dims[n] = dims[n]; c.tot *= dims[n]; - bart_dim_t centre = dims[n] / 2; + int64_t centre = dims[n] / 2; double half = (double)centre / (double)dims[n]; c.shifts[n] += 2. * M_PI * half; @@ -70,7 +71,7 @@ static inline struct phase_conf phase_setup(int N, const bart_dim_t dims[], cons return c; } -__device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, bart_dim_t x, bart_dim_t y, bart_dim_t z, bool conj) +__device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, int64_t x, int64_t y, int64_t z, bool conj) { float val = c.cn + x * c.shifts[0] + y * c.shifts[1] + z * c.shifts[2]; @@ -85,7 +86,7 @@ __device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, bar } /* Where grid point `i` sits in the gathered spectrum, or -1 where it is not kept. */ -__device__ static inline bart_dim_t kept_at(const unsigned int* mask, const int* prefix, bart_dim_t i) +__device__ static inline int64_t kept_at(const unsigned int* mask, const int* prefix, int64_t i) { unsigned int word = mask[i >> 5]; unsigned int bit = 1u << (i & 31); @@ -93,7 +94,7 @@ __device__ static inline bart_dim_t kept_at(const unsigned int* mask, const int* if (0 == (word & bit)) return -1; - return (bart_dim_t)prefix[i >> 5] + __popc(word & (bit - 1)); + return (int64_t)prefix[i >> 5] + __popc(word & (bit - 1)); } /* What the callbacks of one transform read: a set's phase, a coil's diff --git a/src/csrc/ops/fft_callbacks_lto.cu b/src/csrc/ops/fft_callbacks_lto.cu index 5e21fbb1..772d3c2f 100644 --- a/src/csrc/ops/fft_callbacks_lto.cu +++ b/src/csrc/ops/fft_callbacks_lto.cu @@ -26,7 +26,7 @@ __device__ static inline cuFloatComplex phase_of(const struct coset_info* c, uns unsigned int ny = (unsigned int)c->phase.dims[1]; unsigned int yz = i / nx; - return phase_at(c->phase, (bart_dim_t)(i - yz * nx), (bart_dim_t)(yz % ny), (bart_dim_t)(yz / ny), conj); + return phase_at(c->phase, (int64_t)(i - yz * nx), (int64_t)(yz % ny), (int64_t)(yz / ny), conj); } __device__ cufftComplex bartorch_load_in(void* in, unsigned long long off, void* info, void* shared) @@ -45,7 +45,7 @@ __device__ void bartorch_store_gather(void* out, unsigned long long off, cufftCo { const struct coset_info* c = (const struct coset_info*)info; - bart_dim_t j = kept_at(c->mask, c->prefix, (bart_stride_t)off); + int64_t j = kept_at(c->mask, c->prefix, (int64_t)off); if (0 <= j) c->bank[j] = val; @@ -55,7 +55,7 @@ __device__ cufftComplex bartorch_load_scatter(void* in, unsigned long long off, { const struct coset_info* c = (const struct coset_info*)info; - bart_dim_t j = kept_at(c->mask, c->prefix, (bart_stride_t)off); + int64_t j = kept_at(c->mask, c->prefix, (int64_t)off); return (0 <= j) ? c->bank[j] : make_cuFloatComplex(0.f, 0.f); } @@ -102,7 +102,7 @@ __device__ void bartorch_grid_store_gather(void* out, unsigned long long off, cu { const struct grid_info* g = (const struct grid_info*)info; - bart_dim_t j = grid_kept(g, (unsigned int)off); + int64_t j = grid_kept(g, (unsigned int)off); if (0 <= j) g->bank[j] = val; @@ -112,7 +112,7 @@ __device__ cufftComplex bartorch_grid_load_scatter(void* in, unsigned long long { const struct grid_info* g = (const struct grid_info*)info; - bart_dim_t j = grid_kept(g, (unsigned int)off); + int64_t j = grid_kept(g, (unsigned int)off); return (0 <= j) ? g->bank[j] : make_cuFloatComplex(0.f, 0.f); } diff --git a/src/csrc/ops/grid.cuh b/src/csrc/ops/grid.cuh index 4eea19ae..887d16fb 100644 --- a/src/csrc/ops/grid.cuh +++ b/src/csrc/ops/grid.cuh @@ -67,14 +67,14 @@ __device__ static inline unsigned int grid_index(const struct grid_info* g, unsi /* Where cuFFT offset `off` sits in the gathered spectrum, or -1 where the * pattern does not keep its place. */ -__device__ static inline bart_dim_t grid_kept(const struct grid_info* g, unsigned int off) +__device__ static inline int64_t grid_kept(const struct grid_info* g, unsigned int off) { unsigned int batch = off / g->plane; unsigned int place = off - batch * g->plane; - bart_dim_t j = kept_at(g->mask, g->prefix, (bart_dim_t)place); + int64_t j = kept_at(g->mask, g->prefix, (int64_t)place); - return (0 > j) ? -1 : (bart_dim_t)batch * g->L + j; + return (0 > j) ? -1 : (int64_t)batch * g->L + j; } #endif From 524554773335d5622bde3ba45a857780695984f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 15:34:00 +0000 Subject: [PATCH 05/10] Use the __sync builtins for the Blocks runtime on Windows The vendored runtime took its Win32 branch whenever _WIN32 is defined, where InterlockedCompareExchange takes a LONG and the reference count is an int: distinct types even at one width, and an error under current clang. config.h already declares the __sync builtins, which clang and GCC provide on every target, so that branch is tried first. The BART pin moves to the fork's head, whose Windows basename takes a char *. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- external/bart | 2 +- external/blocksruntime/runtime.c | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/external/bart b/external/bart index 4106c2cc..9a398d2a 160000 --- a/external/bart +++ b/external/bart @@ -1 +1 @@ -Subproject commit 4106c2ccb7024f1875e4891612835fd92fd96939 +Subproject commit 9a398d2aaf5e835b73cd73357b796d0c55a53695 diff --git a/external/blocksruntime/runtime.c b/external/blocksruntime/runtime.c index a059c223..c059f890 100644 --- a/external/blocksruntime/runtime.c +++ b/external/blocksruntime/runtime.c @@ -44,22 +44,6 @@ #include #endif /* HAVE_LIBKERN_OSATOMIC_H */ -#elif defined(__WIN32__) || defined(_WIN32) -#define _CRT_SECURE_NO_WARNINGS 1 -#include - -static __inline bool OSAtomicCompareAndSwapLong(long oldl, long newl, long volatile *dst) { - /* fixme barrier is overkill -- see objc-os.h */ - long original = InterlockedCompareExchange(dst, newl, oldl); - return (original == oldl); -} - -static __inline bool OSAtomicCompareAndSwapInt(int oldi, int newi, int volatile *dst) { - /* fixme barrier is overkill -- see objc-os.h */ - int original = InterlockedCompareExchange(dst, newi, oldi); - return (original == oldi); -} - /* * Check to see if the GCC atomic built-ins are available. If we're on * a 64-bit system, make sure we have an 8-byte atomic function @@ -77,6 +61,22 @@ static __inline bool OSAtomicCompareAndSwapInt(int oldi, int newi, int volatile return __sync_bool_compare_and_swap(dst, oldi, newi); } +#elif defined(__WIN32__) || defined(_WIN32) +#define _CRT_SECURE_NO_WARNINGS 1 +#include + +static __inline bool OSAtomicCompareAndSwapLong(long oldl, long newl, long volatile *dst) { + /* fixme barrier is overkill -- see objc-os.h */ + long original = InterlockedCompareExchange(dst, newl, oldl); + return (original == oldl); +} + +static __inline bool OSAtomicCompareAndSwapInt(int oldi, int newi, int volatile *dst) { + /* fixme barrier is overkill -- see objc-os.h */ + int original = InterlockedCompareExchange(dst, newi, oldi); + return (original == oldi); +} + #else #error unknown atomic compare-and-swap primitive #endif /* HAVE_OSATOMIC_COMPARE_AND_SWAP_INT && HAVE_OSATOMIC_COMPARE_AND_SWAP_LONG */ From ea1669c076c140b55d47bc8befdf4408e2eebe15 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 15:38:33 +0000 Subject: [PATCH 06/10] Define the C runtime's assertion entry points on Windows The C++ runtime is linked in statically and calls _assert by name, which pulled the Universal C Runtime's import of it in beside the import slot defined here. Defining _assert and _wassert themselves, with the slots pointing at them, leaves the import library nothing to supply. The noreturn attribute is spelled __noreturn__ so does not rewrite it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- src/csrc/abi/api.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/csrc/abi/api.c b/src/csrc/abi/api.c index 4bc3affd..e01b7efb 100644 --- a/src/csrc/abi/api.c +++ b/src/csrc/abi/api.c @@ -50,29 +50,34 @@ extern int bart_command(int len, char* buf, int argc, char* argv[]); #include -__attribute__((noreturn)) -static void crt_assert(const char* assertion, const char* file, unsigned line) +void _assert(const char* assertion, const char* file, unsigned line); +void _wassert(const wchar_t* assertion, const wchar_t* file, unsigned line); + +__attribute__((__noreturn__)) +void _assert(const char* assertion, const char* file, unsigned line) { error("Assertion '%s' failed in %s:%u\n", assertion, file, line); } -__attribute__((noreturn)) -static void crt_wassert(const wchar_t* assertion, const wchar_t* file, unsigned line) +__attribute__((__noreturn__)) +void _wassert(const wchar_t* assertion, const wchar_t* file, unsigned line) { error("Assertion '%ls' failed in %ls:%u\n", assertion, file, line); } -/* declares both dllimport, so a caller reaches them through these - * slots; defined here, the import library's are never linked. */ -void (*__imp__assert)(const char*, const char*, unsigned) = crt_assert; -void (*__imp__wassert)(const wchar_t*, const wchar_t*, unsigned) = crt_wassert; +/* declares both dllimport, so BART reaches them through these + * slots, and the C++ runtime linked in statically calls them by name. + * Defining both keeps the C runtime's import library from supplying + * either. */ +void (*__imp__assert)(const char*, const char*, unsigned) = _assert; +void (*__imp__wassert)(const wchar_t*, const wchar_t*, unsigned) = _wassert; #elif defined(__APPLE__) -__attribute__((noreturn)) +__attribute__((__noreturn__)) void __assert_rtn(const char* function, const char* file, int line, const char* assertion); -__attribute__((noreturn)) +__attribute__((__noreturn__)) void __assert_rtn(const char* function, const char* file, int line, const char* assertion) { error("Assertion '%s' failed in %s:%d (%s)\n", assertion, file, line, function); From 670f0011f927384a7eeabe0ad66fcd163ad723cb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 15:49:01 +0000 Subject: [PATCH 07/10] Move the BART pin to the fork's MinGW format fixes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- external/bart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/bart b/external/bart index 9a398d2a..7e69ca58 160000 --- a/external/bart +++ b/external/bart @@ -1 +1 @@ -Subproject commit 9a398d2aaf5e835b73cd73357b796d0c55a53695 +Subproject commit 7e69ca58ffdd2a9b05e216ab62f1c5e2fe7e9211 From 74b77dd40d43185473ed54de51b1d06410a12dde Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 15:54:14 +0000 Subject: [PATCH 08/10] Spell the macOS rpath with slashes, and stop each item's solve on its own rpath_for builds a dyld path, which takes slashes whichever platform the suite runs on; os.path.relpath gave backslashes on Windows. The test that items share nothing ran BART's batched conjugate gradients at zero tolerance, where an item stops only once its residual is exactly zero and the loop runs until every item's is. Whether one item then waits on another is a question of underflow, which came out differently on Windows. A tolerance stops each item on its own residual, which is the independence the test asserts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- src/bartorch/_macos_openmp.py | 4 ++-- tests/test_nlop_step.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/bartorch/_macos_openmp.py b/src/bartorch/_macos_openmp.py index e4b8119e..113e6045 100644 --- a/src/bartorch/_macos_openmp.py +++ b/src/bartorch/_macos_openmp.py @@ -21,7 +21,7 @@ from __future__ import annotations -import os +import posixpath import shutil import struct import subprocess @@ -251,7 +251,7 @@ def rpath_for(finufft_lib: Path, torch_omp: Path) -> str: Relative to ``@loader_path`` so that the environment can be moved or renamed, which an absolute path into site-packages would not survive. """ - relative = os.path.relpath(torch_omp.parent, finufft_lib.parent) + relative = posixpath.relpath(torch_omp.parent.as_posix(), finufft_lib.parent.as_posix()) return f"@loader_path/{relative}" diff --git a/tests/test_nlop_step.py b/tests/test_nlop_step.py index 3a632472..377e397b 100644 --- a/tests/test_nlop_step.py +++ b/tests/test_nlop_step.py @@ -488,7 +488,11 @@ def test_a_model_of_items_steps_each_as_it_would_step_alone(off_grid): kspace = torch.stack([single.forward(*pair) for pair in zip(*_phantoms(items, n, coils))]) def run(F, y): - block = nlop.IRGNMBlock(cg_maxiter=30) + # A tolerance stops each item's conjugate gradients on its own residual. + # At zero, BART's batched solve stops an item only once its residual is + # exactly zero and runs until every item's is, so whether an item waits + # on the others is a question of underflow. + block = nlop.IRGNMBlock(cg_maxiter=30, cg_tol=1e-4) state = block.start(y, F) for _ in range(4): state = block(state, F) From 4b019384e7ad187d735b627413843705e16ddbdf Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 17:04:25 +0000 Subject: [PATCH 09/10] Pin BART at the fork head that builds and tests natively on Windows Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- external/bart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/bart b/external/bart index 7e69ca58..12cbf14f 160000 --- a/external/bart +++ b/external/bart @@ -1 +1 @@ -Subproject commit 7e69ca58ffdd2a9b05e216ab62f1c5e2fe7e9211 +Subproject commit 12cbf14f4592e897b7d8348994e3cc566d2810e5 From a5de38ad65f47cbea6a33edaf5bac026a727375a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 17:22:49 +0000 Subject: [PATCH 10/10] Pin BART at the fork head under review in pulserver/bart#1 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HV1XJrQxFRsusBwNuvwh3j --- external/bart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/bart b/external/bart index 12cbf14f..694ab5fe 160000 --- a/external/bart +++ b/external/bart @@ -1 +1 @@ -Subproject commit 12cbf14f4592e897b7d8348994e3cc566d2810e5 +Subproject commit 694ab5fe8241c43d67df5382c38dcf074aaa95a3