diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8297ef88..ce1edfa7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -54,13 +54,34 @@ jobs: # ninja and a C++ compiler from someone installing a wheel. aarch64 # is served by the sdist, where that toolchain is already the price of # entry. - os: [ubuntu-latest, macos-latest] + # + # Windows is CPU only: clang from MSYS2's CLANG64 environment, which + # links against the Universal C Runtime official CPython uses. + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 with: submodules: recursive # setuptools-scm reads the version from the tag history. fetch-depth: 0 + - name: Install clang, CMake and Ninja (Windows) + if: runner.os == 'Windows' + id: msys2 + uses: msys2/setup-msys2@v2 + with: + msystem: CLANG64 + update: true + install: >- + mingw-w64-clang-x86_64-clang + mingw-w64-clang-x86_64-cmake + mingw-w64-clang-x86_64-ninja + mingw-w64-clang-x86_64-llvm-openmp + mingw-w64-clang-x86_64-llvm-tools + - name: Put CLANG64 on the path (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + "${{ steps.msys2.outputs.msys2-location }}\clang64\bin" >> $env:GITHUB_PATH - uses: pypa/cibuildwheel@v2.23 env: # One build per platform: the wheel is tagged py3 and carries no @@ -75,6 +96,10 @@ jobs: # wheel in any case: torch publishes no i686 build, and it is a hard # dependency here. CIBW_ARCHS_LINUX: x86_64 + CIBW_ARCHS_WINDOWS: AMD64 + # The library imports the system's DLLs and the OpenMP runtime torch + # carries, so there is nothing to vendor and no repair step. + CIBW_ENVIRONMENT_WINDOWS: CC=clang CXX=clang++ CMAKE_GENERATOR=Ninja # BART's nested functions need heap trampolines -- GCC 14 -- or # clang's Blocks. cibuildwheel's default Linux image has neither: # manylinux2014 is CentOS 7, whose GCC is 10 and whose `clang` is @@ -104,8 +129,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + os: [ubuntu-latest, macos-latest, windows-latest] + # The oldest and the newest Python the package supports. + python: ["3.10", "3.14"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bbef760a..e93bff54 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,11 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: - # libbartorch is a clang build: BART's nested functions become Blocks. + # BART's nested functions become Blocks under clang and heap trampolines + # under GCC 14. Each platform runs the oldest and the newest Python the + # package supports; the wheel workflow tests the wheel on both as well. + # Windows is clang from MSYS2's CLANG64 environment, linked against the + # Universal C Runtime that official CPython uses. tests: strategy: fail-fast: false @@ -18,20 +22,29 @@ jobs: include: - os: ubuntu-latest python-version: "3.10" - cc: clang + cc: gcc-14 + extras: "[mkl]" - os: ubuntu-latest - python-version: "3.12" + python-version: "3.14" cc: clang # One job measures coverage for Codecov; the rest only test. coverage: true - - os: ubuntu-latest - python-version: "3.12" - cc: gcc-14 - extras: "[mkl]" - os: macos-latest - python-version: "3.12" + python-version: "3.10" + cc: clang + - os: macos-latest + python-version: "3.14" + cc: clang + - os: windows-latest + python-version: "3.10" + cc: clang + - os: windows-latest + python-version: "3.14" cc: clang runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash # Codecov takes the upload on GitHub's OIDC token, so no secret is needed. permissions: contents: read @@ -57,6 +70,38 @@ jobs: if: runner.os == 'macOS' run: brew install cmake + # omp.h comes from the openmp package; the library links against the + # OpenMP runtime torch carries, not against MSYS2's. + - name: Install clang, CMake and Ninja (Windows) + if: runner.os == 'Windows' + id: msys2 + uses: msys2/setup-msys2@v2 + with: + msystem: CLANG64 + update: true + install: >- + mingw-w64-clang-x86_64-clang + mingw-w64-clang-x86_64-cmake + mingw-w64-clang-x86_64-ninja + mingw-w64-clang-x86_64-llvm-openmp + mingw-w64-clang-x86_64-llvm-tools + + - name: Put CLANG64 on the path (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + "${{ steps.msys2.outputs.msys2-location }}\clang64\bin" >> $env:GITHUB_PATH + "CMAKE_GENERATOR=Ninja" >> $env:GITHUB_ENV + + # long is 32 bits here, which is the case the fixed-width types exist + # for; this prints it, and tests/test_abi.py holds the library to it. + - name: The data model (Windows) + if: runner.os == 'Windows' + run: | + printf '#include \nint main(void) { printf("long %%d, long long %%d, void* %%d\\n", (int)sizeof(long), (int)sizeof(long long), (int)sizeof(void*)); return 4 != sizeof(long); }\n' > model.c + clang model.c -o model.exe + ./model.exe + - name: Install torch (CPU) and the build backend run: | pip install scikit-build-core setuptools-scm numpy scipy pytest pytest-cov numpydoc @@ -73,7 +118,7 @@ jobs: # Linux only because that index publishes no macOS wheel -- there both # come from PyPI, which is consistent with itself. - name: Install torchvision from the same index as torch - if: runner.os == 'Linux' + if: runner.os != 'macOS' run: pip install torchvision --index-url https://download.pytorch.org/whl/cpu - name: Build and install bartorch @@ -82,6 +127,15 @@ jobs: CXX: ${{ matrix.cc == 'gcc-14' && 'g++-14' || 'clang++' }} run: pip install -e ".${{ matrix.extras }}" --no-build-isolation -v + # The library imports the system's DLLs and torch's OpenMP runtime and + # nothing from MSYS2, so it loads into official CPython without it. + - name: Nothing from MSYS2 in the library (Windows) + if: runner.os == 'Windows' + run: | + lib=$(python -c "from bartorch._lib import library_path; print(library_path())") + llvm-readobj --coff-imports "$lib" | grep 'Name:' | sort -u | tee imports.txt + ! grep -Ei 'libomp|libc\+\+|libunwind|libwinpthread|msys' imports.txt + # Both wheels carry their own copy of LLVM's OpenMP runtime, and the # runtime ends the process rather than run beside a second copy. The # repair is bartorch's own, on the first NUFFT, so this reaches it the @@ -107,6 +161,8 @@ jobs: logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") import bartorch print("build:", bartorch.build_info()) + import ctypes, sys + print("platform:", sys.platform, "sizeof(long):", ctypes.sizeof(ctypes.c_long)) print("backends:", bartorch.backend_sources()) try: from bartorch import _finufft diff --git a/AGENTS.md b/AGENTS.md index 310c08b2..a36cf3d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,6 +124,12 @@ runtime spinning against BART's, which inside a tool costs several times more than the transform gains. Where no source has DFTI, which is macOS, and for a description DFTI declines, the transform compiled into the library serves instead. On a device it is cuFFT, through BART's own `fft-cuda.c`. +DFTI is taken through MKL's LP64 interface, whose `MKL_LONG` is `long`: 32 +bits on Windows. A length or stride past that is a description DFTI is not +handed -- the loop axis is walked instead, and a transformed axis leaves the +plan to pocketfft -- so LP64 is enough and ILP64 is not needed; the loop is +over BART's own `ptrdiff_t` strides. BLAS and LAPACK take `int` everywhere, +and BART range-checks each extent before a call (`checked_int`). **CUDA is the same ABI.** `-DBARTORCH_CUDA=ON` compiles BART's thirteen `.cu` files with nvcc and links cudart, cuFFT and cuBLAS dynamically, which are the @@ -620,10 +626,22 @@ configure time. BART's own `NOEXEC_STACK` workaround does not help here: it parses a trampoline layout GCC emits only for non-PIC executables, not for a shared library. Both compilers are built and tested in CI. -Windows is not a platform here. BART does not build on it, and nothing in -this repository carries a path toward one: no `.dll` among the names the -loader tries, no `__declspec(dllexport)`, no `win32` branch picking a -different library. WSL2 is a Linux install and is the answer. +**Windows** is LLP64: `long` is 32 bits, so BART keeps every extent, stride +and flag set in the fixed-width types of `misc/dimtypes.h` (`bart_dim_t`, +`bart_stride_t`, `bart_flags_t`, all 64 bits), and the ABI spells them +`int64_t` and `uint64_t`; `api.c` asserts the two agree, and +`tests/test_abi.py` holds the header to having no `long` and `build_info()`'s +`long=` to the platform. The library is clang from MSYS2's CLANG64 +environment: Blocks from the vendored runtime, linked in as objects because +clang declares it `dllimport`; everything the toolchain would bring as a DLL +linked statically; the Universal C Runtime official CPython uses. OpenMP is +torch's: torch loads Intel's `libiomp5md.dll`, LLVM's runtime refuses to start +beside it, and the two share an ABI, so the library links against an import +library made from `src/csrc/compat/libiomp5md.def` and the loader binds it to +the copy torch has loaded. A build without OpenMP there is refused unless +`-DBARTORCH_OPENMP=OFF` asks for it. BART's POSIX gaps -- `getsubopt`, +C11 threads, `SIGSTOP`, `SIGPIPE`, `readlink` -- are patches on the fork, and +named pipes are off (`NO_FIFO`). CPU only: there is no CUDA build on Windows. The compiler's own runtime is linked statically on Linux, because otherwise the toolchain's floor becomes the target system's: a GCC 14 build asks @@ -1113,8 +1131,7 @@ of it is written. What comes back is finite stack memory. `_call`'s `_PRECONDITIONS` refuses those combinations before the command runs, which is where any further "BART does not define this" case belongs. -Windows is not on this list because it is not a target: BART does not build -there, and WSL2 is a Linux install like any other. +A CUDA build on Windows. A tool that takes device memory as it stands. BART guards the host reads that would break -- `estimate_im_dims` copies to the host when it is handed one -- diff --git a/CMakeLists.txt b/CMakeLists.txt index e1cd74c8..74ddf3c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,7 +77,16 @@ set(BART_EXTRA_LIBS "") if(CMAKE_C_COMPILER_ID MATCHES "Clang") list(APPEND BART_C_OPTIONS -fblocks) set(BARTORCH_NESTED "clang-blocks") - if(NOT APPLE) + if(WIN32) + # clang declares the runtime dllimport on Windows, and the linker + # resolves an import to a definition in the same image only when that + # definition is already present: objects are, archive members are not + # pulled in for it. + add_library(blocksruntime OBJECT + external/blocksruntime/runtime.c + external/blocksruntime/data.c) + list(APPEND BART_EXTRA_LIBS $) + elseif(NOT APPLE) add_library(blocksruntime STATIC external/blocksruntime/runtime.c external/blocksruntime/data.c) @@ -109,6 +118,11 @@ endif() # the way BART's own fallback does. Tested rather than guessed, because # defining it where it does exist collides with the system header. include(CheckCSourceCompiles) +# The data model, reported in build_info: long is 64 bits on LP64 (Linux, +# macOS) and 32 on LLP64 (Windows), and nothing BART or the ABI stores in an +# array's shape depends on it. +include(CheckTypeSize) +check_type_size(long BARTORCH_SIZEOF_LONG LANGUAGE C) # Through , because that is where BART's translation units get it: # glibc typedefs it in , which and # pull in and does not on every glibc, and clang does not offer it as @@ -125,7 +139,14 @@ endif() # would carry the toolchain's floor into every target system -- a GCC 14 build # asks for GCC_14.0.0 from libgcc_s, which no released distribution has -- and # that is a load-time failure, not a fallback. Static leaves libc and libgomp. -if(NOT APPLE) +# +# On Windows the same goes for everything the MinGW toolchain would otherwise +# bring as a DLL -- winpthreads and the compiler's runtime -- because Python +# does not search PATH for a library's dependencies. What is left is the +# Universal C Runtime and the system's own libraries. +if(WIN32) + list(APPEND BART_LINK_OPTIONS -static) +elseif(NOT APPLE) list(APPEND BART_LINK_OPTIONS -static-libgcc -static-libstdc++) endif() @@ -164,7 +185,28 @@ if(BARTORCH_CUDA) message(STATUS "CUDA ${CUDAToolkit_VERSION}: architectures ${BARTORCH_CUDA_ARCH_SPEC}") endif() -if(BARTORCH_OPENMP) +if(BARTORCH_OPENMP AND WIN32) + # torch loads Intel's OpenMP runtime on Windows, and LLVM's refuses to + # initialise beside it (OMP: Error #15), as on macOS. The two share an + # ABI, so the library is bound to the runtime torch has already loaded: + # clang compiles with -fopenmp, and the link is against an import library + # for libiomp5md.dll made from the entry points clang calls. The loader + # resolves that name to the copy in the process. + if(NOT CMAKE_C_COMPILER_ID MATCHES "Clang") + message(FATAL_ERROR "OpenMP on Windows is built with clang; pass -DBARTORCH_OPENMP=OFF to build without it") + endif() + find_program(BARTORCH_DLLTOOL NAMES llvm-dlltool dlltool x86_64-w64-mingw32-dlltool REQUIRED) + set(_iomp_def "${CMAKE_CURRENT_SOURCE_DIR}/src/csrc/compat/libiomp5md.def") + set(_iomp_lib "${CMAKE_CURRENT_BINARY_DIR}/libiomp5md.dll.a") + add_custom_command(OUTPUT "${_iomp_lib}" + COMMAND "${BARTORCH_DLLTOOL}" -m i386:x86-64 -d "${_iomp_def}" -l "${_iomp_lib}" -D libiomp5md.dll + DEPENDS "${_iomp_def}" + VERBATIM) + add_custom_target(iomp5_import DEPENDS "${_iomp_lib}") + set(OpenMP_C_FOUND TRUE) + list(APPEND BART_C_OPTIONS -fopenmp) + list(APPEND BART_EXTRA_LIBS "${_iomp_lib}") +elseif(BARTORCH_OPENMP) find_package(OpenMP COMPONENTS C) if(OpenMP_C_FOUND) list(APPEND BART_EXTRA_LIBS OpenMP::OpenMP_C) @@ -230,7 +272,12 @@ file(GLOB _module_dirs LIST_DIRECTORIES true CONFIGURE_DEPENDS "${BART_SRC}/*") foreach(_dir ${_module_dirs}) if(IS_DIRECTORY "${_dir}") get_filename_component(_mod "${_dir}" NAME) - if(NOT _mod MATCHES "^(ismrm|lapacke|win)$") + # `win/` is BART's own port of mmap, fmemopen, getsubopt and the rest + # of what the Windows C runtime lacks, and is built there alone. + if(_mod STREQUAL "win" AND NOT WIN32) + continue() + endif() + if(NOT _mod MATCHES "^(ismrm|lapacke)$") file(GLOB _srcs CONFIGURE_DEPENDS "${_dir}/*.c") list(APPEND BART_SOURCES ${_srcs}) endif() @@ -407,7 +454,11 @@ target_compile_definitions(bartorch PRIVATE NO_PNG _GNU_SOURCE ${BART_C_DEFS} - BARTORCH_BUILD_INFO="bart=${BART_VERSION},compiler=${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION},nested=${BARTORCH_NESTED},openmp=${OpenMP_C_FOUND},cuda=${BARTORCH_CUDA}") + # Windows: a 64-bit off_t, so a file past 2 GiB can be mapped; BART's own + # functions linked into this library rather than imported from its DLL + # (misc/dllspec.h); and no named pipes, which are a POSIX file type. + $<$:_FILE_OFFSET_BITS=64;BARTLIB_STATIC;NO_FIFO> + BARTORCH_BUILD_INFO="bart=${BART_VERSION},compiler=${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION},nested=${BARTORCH_NESTED},openmp=${OpenMP_C_FOUND},cuda=${BARTORCH_CUDA},long=${BARTORCH_SIZEOF_LONG}") target_compile_options(bartorch PRIVATE $<$:${BART_C_OPTIONS}> @@ -436,6 +487,10 @@ target_link_libraries(bartorch PRIVATE Threads::Threads ${BART_EXTRA_LIBS}) target_link_libraries(bartorch PRIVATE m) +if(TARGET iomp5_import) + add_dependencies(bartorch iomp5_import) +endif() + set_target_properties(bartorch PROPERTIES OUTPUT_NAME bartorch PREFIX "lib") diff --git a/docs/guides/developer/prerequisites.md b/docs/guides/developer/prerequisites.md index 8980dd2f..e6fe587f 100644 --- a/docs/guides/developer/prerequisites.md +++ b/docs/guides/developer/prerequisites.md @@ -15,5 +15,9 @@ suite is run from `src/` without installing, install it separately skip, because the substitution declining is an error. `pip install mkl deepinv` enables the tests that need MKL and the DeepInverse adapter. -Windows is not a target: BART does not build on it, and WSL2 is used as a Linux -environment. +On Windows the compiler is clang from MSYS2's CLANG64 environment, with the +packages `mingw-w64-clang-x86_64-{clang,cmake,ninja,llvm-openmp,llvm-tools}`, +`clang64/bin` on `PATH` and `CMAKE_GENERATOR=Ninja`. The build links against +the OpenMP runtime PyTorch installs rather than MSYS2's, and loads into the +official CPython with nothing from MSYS2 at run time. There is no CUDA build on +Windows. diff --git a/docs/guides/developer/pull-requests.md b/docs/guides/developer/pull-requests.md index 40edf072..88ed04b4 100644 --- a/docs/guides/developer/pull-requests.md +++ b/docs/guides/developer/pull-requests.md @@ -11,8 +11,8 @@ template: | Validation | The commands run and their results; for a numerical change, the reference and the tolerance; hardware or optional dependencies that were not available | | Public API and documentation | Changes to public behaviour and the documentation updated for them | -CI runs the test matrix (Linux with clang and GCC 14, macOS, and a CUDA build -without a device), the lint and spelling checks, and two documentation builds: +CI runs the test matrix (Linux with GCC 14 and clang, macOS and Windows with +clang, each on Python 3.10 and 3.14, and a CUDA build without a device), the lint and spelling checks, and two documentation builds: the reference without the compiled library, and the executed gallery, which is published from `main` and from release tags. A pull request is merged when CI passes and a maintainer has approved it. diff --git a/docs/guides/user/prerequisites.md b/docs/guides/user/prerequisites.md index c86ef41d..a29cd6c9 100644 --- a/docs/guides/user/prerequisites.md +++ b/docs/guides/user/prerequisites.md @@ -4,7 +4,7 @@ | Requirement | Version | | --- | --- | -| Python | 3.10 or later; wheels are tested on 3.10 to 3.14 | +| Python | 3.10 or later; each wheel is tested on 3.10 and 3.14 | | PyTorch | 2.2 or later, CPU or CUDA build | | NumPy, SciPy | NumPy 1.24 and SciPy 1.10 or later; MRI-NUFFT, a dependency, raises these to NumPy 2.2 and SciPy 1.13 | | FINUFFT | 2.2 or later, installed as a dependency | @@ -18,7 +18,7 @@ | macOS 11 or later on Apple silicon | Wheel | CPU only; see {ref}`macos-openmp` | | Linux aarch64 | Source distribution | BART is compiled on installation; FINUFFT publishes no wheel for this platform and is built from source as well | | macOS on Intel | Source distribution | BART is compiled on installation; FINUFFT releases after 2.4.0 have no wheel for this platform and are built from source as well | -| Windows | Not supported | BART does not build on Windows; WSL2 provides a Linux environment | +| Windows 10 or later on x86-64 | Wheel | CPU only; OpenMP is the runtime PyTorch installs | A source installation needs the toolchain listed under {ref}`source-builds`. Apple MPS devices are not supported; the device paths are CPU and CUDA. diff --git a/external/bart b/external/bart index 731bfd3e..694ab5fe 160000 --- a/external/bart +++ b/external/bart @@ -1 +1 @@ -Subproject commit 731bfd3ec4f09ffa59b497ded1274a2c0357c8af +Subproject commit 694ab5fe8241c43d67df5382c38dcf074aaa95a3 diff --git a/external/blocksruntime/runtime.c b/external/blocksruntime/runtime.c index a059c223..c059f890 100644 --- a/external/blocksruntime/runtime.c +++ b/external/blocksruntime/runtime.c @@ -44,22 +44,6 @@ #include #endif /* HAVE_LIBKERN_OSATOMIC_H */ -#elif defined(__WIN32__) || defined(_WIN32) -#define _CRT_SECURE_NO_WARNINGS 1 -#include - -static __inline bool OSAtomicCompareAndSwapLong(long oldl, long newl, long volatile *dst) { - /* fixme barrier is overkill -- see objc-os.h */ - long original = InterlockedCompareExchange(dst, newl, oldl); - return (original == oldl); -} - -static __inline bool OSAtomicCompareAndSwapInt(int oldi, int newi, int volatile *dst) { - /* fixme barrier is overkill -- see objc-os.h */ - int original = InterlockedCompareExchange(dst, newi, oldi); - return (original == oldi); -} - /* * Check to see if the GCC atomic built-ins are available. If we're on * a 64-bit system, make sure we have an 8-byte atomic function @@ -77,6 +61,22 @@ static __inline bool OSAtomicCompareAndSwapInt(int oldi, int newi, int volatile return __sync_bool_compare_and_swap(dst, oldi, newi); } +#elif defined(__WIN32__) || defined(_WIN32) +#define _CRT_SECURE_NO_WARNINGS 1 +#include + +static __inline bool OSAtomicCompareAndSwapLong(long oldl, long newl, long volatile *dst) { + /* fixme barrier is overkill -- see objc-os.h */ + long original = InterlockedCompareExchange(dst, newl, oldl); + return (original == oldl); +} + +static __inline bool OSAtomicCompareAndSwapInt(int oldi, int newi, int volatile *dst) { + /* fixme barrier is overkill -- see objc-os.h */ + int original = InterlockedCompareExchange(dst, newi, oldi); + return (original == oldi); +} + #else #error unknown atomic compare-and-swap primitive #endif /* HAVE_OSATOMIC_COMPARE_AND_SWAP_INT && HAVE_OSATOMIC_COMPARE_AND_SWAP_LONG */ diff --git a/pyproject.toml b/pyproject.toml index 62c1eef3..11c6f8ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,8 +79,8 @@ deepinv = [ ] # MKL covers every BLAS and LAPACK routine BART calls and is about twice as # fast as the alternatives on the work that leans on LAPACK. There is no MKL -# wheel for macOS, where SciPy's OpenBLAS and Accelerate serve instead, and -# Windows is not a target: BART does not build there. +# wheel for macOS, where SciPy's OpenBLAS and Accelerate serve instead; on +# Windows SciPy's OpenBLAS serves. mkl = [ "mkl>=2024; sys_platform == 'linux' and platform_machine == 'x86_64'", ] diff --git a/scripts/gen_abi.py b/scripts/gen_abi.py index cd956180..bbf0ce9b 100644 --- a/scripts/gen_abi.py +++ b/scripts/gen_abi.py @@ -28,16 +28,17 @@ # Every one of these appears in bartorch.h; a type that does not is a type the # generator has not been taught, and it stops rather than guessing. Pointers # to the opaque operator structs are void pointers here: the host never reads -# through them. +# through them. There is no ``long``: it is 32 bits on Windows and 64 +# elsewhere, and every integer in the ABI has one width on every platform. SCALARS = { "void": "None", "int": "ctypes.c_int", - "long": "ctypes.c_long", + "int64_t": "ctypes.c_int64", + "uint64_t": "ctypes.c_uint64", "float": "ctypes.c_float", "double": "ctypes.c_double", "size_t": "ctypes.c_size_t", - "unsigned long": "ctypes.c_ulong", "unsigned int": "ctypes.c_uint", } @@ -46,7 +47,8 @@ "char**": "ctypes.POINTER(ctypes.c_char_p)", "void*": "ctypes.c_void_p", "void**": "ctypes.POINTER(ctypes.c_void_p)", - "long*": "ctypes.POINTER(ctypes.c_long)", + "int64_t*": "ctypes.POINTER(ctypes.c_int64)", + "uint64_t*": "ctypes.POINTER(ctypes.c_uint64)", "int*": "ctypes.POINTER(ctypes.c_int)", "float*": "ctypes.POINTER(ctypes.c_float)", "double*": "ctypes.POINTER(ctypes.c_double)", @@ -84,7 +86,7 @@ def normalise(decl: str) -> str: #: Words that begin a type rather than name a parameter. -TYPE_WORDS = {"void", "int", "long", "float", "double", "size_t", "unsigned", "char"} +TYPE_WORDS = {"void", "int", "int64_t", "uint64_t", "float", "double", "size_t", "unsigned", "char"} def ctype(decl: str, callbacks: dict[str, str]) -> str: diff --git a/src/bartorch/_abi.py b/src/bartorch/_abi.py index 862151ec..d905b7fb 100644 --- a/src/bartorch/_abi.py +++ b/src/bartorch/_abi.py @@ -59,17 +59,17 @@ class Encoding(ctypes.Structure): ("traj_dims", ctypes.c_void_p), ("traj", ctypes.c_void_p), ("stacked", ctypes.c_int), - ("stack_count", ctypes.c_long), + ("stack_count", ctypes.c_int64), ("stack_positions", ctypes.c_void_p), - ("frames", ctypes.c_long), - ("shots", ctypes.c_long), + ("frames", ctypes.c_int64), + ("shots", ctypes.c_int64), ("components", ctypes.c_int), ("positions", ctypes.c_void_p), ("kspace_readout", ctypes.c_int), - ("readout", ctypes.c_long), + ("readout", ctypes.c_int64), ("psf", ctypes.c_void_p), ("centred", ctypes.c_int), - ("segments", ctypes.c_long), + ("segments", ctypes.c_int64), ("segment_sample_dims", ctypes.c_void_p), ("segment_sample", ctypes.c_void_p), ("segment_image_dims", ctypes.c_void_p), @@ -89,7 +89,7 @@ class Encoding(ctypes.Structure): ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ) FREE_FN = ctypes.CFUNCTYPE( None, @@ -361,7 +361,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_register.argtypes = [ ctypes.c_char_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_exists.restype = ctypes.c_int @@ -370,7 +370,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_lookup.argtypes = [ ctypes.c_char_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.POINTER(ctypes.c_void_p), ] lib.bartorch_unlink.restype = ctypes.c_int @@ -427,7 +427,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nufft_bf16.restype = ctypes.c_int lib.bartorch_nufft_bf16.argtypes = [] lib.bartorch_host_alloc.restype = ctypes.c_void_p - lib.bartorch_host_alloc.argtypes = [ctypes.c_long, ctypes.c_int] + lib.bartorch_host_alloc.argtypes = [ctypes.c_int64, ctypes.c_int] lib.bartorch_host_free.restype = None lib.bartorch_host_free.argtypes = [ctypes.c_void_p] lib.bartorch_cuda_stage_open.restype = ctypes.c_int @@ -440,20 +440,20 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, - ctypes.c_long, + ctypes.c_int64, ] lib.bartorch_cuda_stage_wait.restype = ctypes.c_int lib.bartorch_cuda_stage_wait.argtypes = [ctypes.c_void_p, ctypes.c_int] lib.bartorch_cuda_stage_release.restype = ctypes.c_int lib.bartorch_cuda_stage_release.argtypes = [ctypes.c_void_p, ctypes.c_int] lib.bartorch_cuda_copy_pageable.restype = ctypes.c_int - lib.bartorch_cuda_copy_pageable.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long] + lib.bartorch_cuda_copy_pageable.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64] lib.bartorch_host_prefault_begin.restype = ctypes.c_void_p - lib.bartorch_host_prefault_begin.argtypes = [ctypes.c_void_p, ctypes.c_long] + lib.bartorch_host_prefault_begin.argtypes = [ctypes.c_void_p, ctypes.c_int64] lib.bartorch_host_prefault_end.restype = None lib.bartorch_host_prefault_end.argtypes = [ctypes.c_void_p] lib.bartorch_cuda_host_register.restype = ctypes.c_int - lib.bartorch_cuda_host_register.argtypes = [ctypes.c_void_p, ctypes.c_long] + lib.bartorch_cuda_host_register.argtypes = [ctypes.c_void_p, ctypes.c_int64] lib.bartorch_cuda_host_unregister.restype = None lib.bartorch_cuda_host_unregister.argtypes = [ctypes.c_void_p] lib.bartorch_sense_set_coil_batch.restype = None @@ -464,7 +464,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_sense_set_fold_maps.argtypes = [ctypes.c_int] lib.bartorch_sense_fold_maps.restype = ctypes.c_int lib.bartorch_sense_fold_maps.argtypes = [] - lib.bartorch_sense_counter.restype = ctypes.c_long + lib.bartorch_sense_counter.restype = ctypes.c_int64 lib.bartorch_sense_counter.argtypes = [ctypes.c_int] lib.bartorch_sense_reset_counters.restype = None lib.bartorch_sense_reset_counters.argtypes = [] @@ -472,16 +472,16 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_fft_set.argtypes = [ctypes.c_char_p, ctypes.c_void_p] lib.bartorch_fft_usable.restype = ctypes.c_int lib.bartorch_fft_usable.argtypes = [] - lib.bartorch_fft_counter.restype = ctypes.c_long + lib.bartorch_fft_counter.restype = ctypes.c_int64 lib.bartorch_fft_counter.argtypes = [ctypes.c_int] lib.bartorch_fft_reset_counters.restype = None lib.bartorch_fft_reset_counters.argtypes = [] lib.bartorch_linop_callback.restype = ctypes.c_void_p lib.bartorch_linop_callback.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), APPLY_FN, APPLY_FN, APPLY_FN, @@ -491,42 +491,42 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_fft.restype = ctypes.c_void_p lib.bartorch_linop_fft.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ctypes.c_int, ctypes.c_int, ] lib.bartorch_linop_cdiag.restype = ctypes.c_void_p lib.bartorch_linop_cdiag.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ctypes.c_void_p, ] lib.bartorch_linop_fmac.restype = ctypes.c_void_p lib.bartorch_linop_fmac.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_sampling.restype = ctypes.c_void_p lib.bartorch_linop_sampling.argtypes = [ - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_nufft.restype = ctypes.c_void_p lib.bartorch_linop_nufft.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ctypes.c_int, ctypes.c_float, @@ -536,31 +536,31 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_set_diagonal.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_blocks.restype = ctypes.c_void_p lib.bartorch_linop_blocks.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.c_long, + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.c_int64, ] lib.bartorch_linop_reshaped.restype = ctypes.c_void_p lib.bartorch_linop_reshaped.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_encoding.restype = ctypes.c_void_p lib.bartorch_linop_encoding.argtypes = [ctypes.POINTER(Encoding)] - lib.bartorch_encoding_counter.restype = ctypes.c_long + lib.bartorch_encoding_counter.restype = ctypes.c_int64 lib.bartorch_encoding_counter.argtypes = [ctypes.c_int] lib.bartorch_encoding_reset_counters.restype = None lib.bartorch_encoding_reset_counters.argtypes = [] - lib.bartorch_grid_fused.restype = ctypes.c_long + lib.bartorch_grid_fused.restype = ctypes.c_int64 lib.bartorch_grid_fused.argtypes = [] lib.bartorch_linop_with_normal.restype = ctypes.c_void_p lib.bartorch_linop_with_normal.argtypes = [ctypes.c_void_p, ctypes.c_void_p] @@ -588,80 +588,92 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_scale.restype = ctypes.c_void_p lib.bartorch_linop_scale.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ctypes.c_float, ] lib.bartorch_linop_zconj.restype = ctypes.c_void_p - lib.bartorch_linop_zconj.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_linop_zconj.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_linop_identity.restype = ctypes.c_void_p - lib.bartorch_linop_identity.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_linop_identity.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_linop_null.restype = ctypes.c_void_p lib.bartorch_linop_null.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_maxeigen.restype = ctypes.c_double lib.bartorch_linop_maxeigen.argtypes = [ctypes.c_void_p] lib.bartorch_linop_zreal.restype = ctypes.c_void_p - lib.bartorch_linop_zreal.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_linop_zreal.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_linop_rdiag.restype = ctypes.c_void_p lib.bartorch_linop_rdiag.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ctypes.c_void_p, ] lib.bartorch_linop_matrix.restype = ctypes.c_void_p lib.bartorch_linop_matrix.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_conv.restype = ctypes.c_void_p lib.bartorch_linop_conv.argtypes = [ ctypes.c_int, - ctypes.c_ulong, + ctypes.c_uint64, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_linop_grad.restype = ctypes.c_void_p lib.bartorch_linop_grad.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.c_ulong, + ctypes.c_uint64, ] lib.bartorch_linop_sum.restype = ctypes.c_void_p - lib.bartorch_linop_sum.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_ulong] + lib.bartorch_linop_sum.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, + ] lib.bartorch_linop_scaled_sum.restype = ctypes.c_void_p lib.bartorch_linop_scaled_sum.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ] lib.bartorch_linop_avg.restype = ctypes.c_void_p - lib.bartorch_linop_avg.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_ulong] + lib.bartorch_linop_avg.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, + ] lib.bartorch_linop_repmat.restype = ctypes.c_void_p lib.bartorch_linop_repmat.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ] lib.bartorch_linop_flip.restype = ctypes.c_void_p - lib.bartorch_linop_flip.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_ulong] + lib.bartorch_linop_flip.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, + ] lib.bartorch_linop_hankel.restype = ctypes.c_void_p lib.bartorch_linop_hankel.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, ctypes.c_int, ctypes.c_int, @@ -669,51 +681,51 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_reshape.restype = ctypes.c_void_p lib.bartorch_linop_reshape.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_resize.restype = ctypes.c_void_p lib.bartorch_linop_resize.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_extract.restype = ctypes.c_void_p lib.bartorch_linop_extract.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_transpose.restype = ctypes.c_void_p lib.bartorch_linop_transpose.argtypes = [ ctypes.c_int, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_permute.restype = ctypes.c_void_p lib.bartorch_linop_permute.argtypes = [ ctypes.c_int, ctypes.POINTER(ctypes.c_int), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_shift.restype = ctypes.c_void_p lib.bartorch_linop_shift.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.c_long, + ctypes.c_int64, ctypes.c_int, ] lib.bartorch_linop_padding.restype = ctypes.c_void_p lib.bartorch_linop_padding.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_has_pseudo_inv.restype = ctypes.c_int lib.bartorch_linop_has_pseudo_inv.argtypes = [ctypes.c_void_p] @@ -728,13 +740,13 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_linop_domain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_codomain.restype = ctypes.c_int lib.bartorch_linop_codomain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_linop_forward.restype = ctypes.c_int lib.bartorch_linop_forward.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] @@ -759,8 +771,8 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_char_p), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_uint64), + ctypes.POINTER(ctypes.c_uint64), ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_void_p), @@ -791,13 +803,13 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.POINTER(ctypes.c_float), ctypes.c_void_p, ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_solve_error.restype = ctypes.c_char_p lib.bartorch_solve_error.argtypes = [ctypes.c_int] lib.bartorch_scaling_norm.restype = ctypes.c_float lib.bartorch_scaling_norm.argtypes = [ - ctypes.c_long, + ctypes.c_int64, ctypes.c_void_p, ctypes.c_float, ctypes.c_int, @@ -806,22 +818,22 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_prox_create.restype = ctypes.c_int lib.bartorch_prox_create.argtypes = [ ctypes.c_char_p, - ctypes.c_long, - ctypes.c_long, + ctypes.c_uint64, + ctypes.c_uint64, ctypes.c_float, ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.POINTER(ctypes.c_void_p), ] lib.bartorch_prox_set_create.restype = ctypes.c_int lib.bartorch_prox_set_create.argtypes = [ ctypes.c_int, ctypes.POINTER(ctypes.c_char_p), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_uint64), + ctypes.POINTER(ctypes.c_uint64), ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_int), ctypes.c_int, @@ -829,17 +841,17 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_float), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_int), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_prox_domain.restype = ctypes.c_int lib.bartorch_prox_domain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_prox_apply.restype = ctypes.c_int lib.bartorch_prox_apply.argtypes = [ @@ -866,9 +878,9 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nlop_callback.restype = ctypes.c_void_p lib.bartorch_nlop_callback.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), APPLY_FN, APPLY_FN, APPLY_FN, @@ -879,10 +891,10 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nlop_callback_generic.argtypes = [ ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), GENERIC_APPLY_FN, PAIR_APPLY_FN, PAIR_APPLY_FN, @@ -897,13 +909,13 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nlop_domain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_codomain.restype = ctypes.c_int lib.bartorch_nlop_codomain.argtypes = [ ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_apply.restype = ctypes.c_int lib.bartorch_nlop_apply.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] @@ -920,14 +932,14 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_output_codomain.restype = ctypes.c_int lib.bartorch_nlop_output_codomain.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_apply_generic.restype = ctypes.c_int lib.bartorch_nlop_apply_generic.argtypes = [ @@ -942,14 +954,14 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_reshape_out.restype = ctypes.c_void_p lib.bartorch_nlop_reshape_out.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_chain2.restype = ctypes.c_void_p lib.bartorch_nlop_chain2.argtypes = [ @@ -992,62 +1004,62 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nlop_tenmul.restype = ctypes.c_void_p lib.bartorch_nlop_tenmul.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_nlop_zdiv.restype = ctypes.c_void_p - lib.bartorch_nlop_zdiv.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_float] + lib.bartorch_nlop_zdiv.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_float] lib.bartorch_nlop_zaxpbz.restype = ctypes.c_void_p lib.bartorch_nlop_zaxpbz.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ctypes.c_float, ] lib.bartorch_nlop_zexp.restype = ctypes.c_void_p - lib.bartorch_nlop_zexp.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_nlop_zexp.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_nlop_zlog.restype = ctypes.c_void_p - lib.bartorch_nlop_zlog.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_nlop_zlog.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_nlop_zinv.restype = ctypes.c_void_p - lib.bartorch_nlop_zinv.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_float] + lib.bartorch_nlop_zinv.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_float] lib.bartorch_nlop_zsqrt.restype = ctypes.c_void_p - lib.bartorch_nlop_zsqrt.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_nlop_zsqrt.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_nlop_zspow.restype = ctypes.c_void_p lib.bartorch_nlop_zspow.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ctypes.c_float, ] lib.bartorch_nlop_zsadd.restype = ctypes.c_void_p lib.bartorch_nlop_zsadd.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ctypes.c_float, ] lib.bartorch_nlop_zabs.restype = ctypes.c_void_p - lib.bartorch_nlop_zabs.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long)] + lib.bartorch_nlop_zabs.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64)] lib.bartorch_nlop_smo_abs.restype = ctypes.c_void_p lib.bartorch_nlop_smo_abs.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_float, ] lib.bartorch_nlop_zrss.restype = ctypes.c_void_p lib.bartorch_nlop_zrss.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.c_ulong, + ctypes.POINTER(ctypes.c_int64), + ctypes.c_uint64, ctypes.c_float, ] lib.bartorch_nlop_zss.restype = ctypes.c_void_p - lib.bartorch_nlop_zss.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_long), ctypes.c_ulong] + lib.bartorch_nlop_zss.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_uint64] lib.bartorch_nlop_const.restype = ctypes.c_void_p lib.bartorch_nlop_const.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_nlop_set_input_const.restype = ctypes.c_void_p @@ -1055,7 +1067,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ] lib.bartorch_nlop_free.restype = None @@ -1066,31 +1078,31 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_int, ctypes.c_float, ctypes.c_float, - ctypes.c_long, + ctypes.c_int64, ] lib.bartorch_noir_create.restype = ctypes.c_void_p lib.bartorch_noir_create.argtypes = [ ctypes.c_int, - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int, - ctypes.c_ulong, - ctypes.c_ulong, + ctypes.c_uint64, + ctypes.c_uint64, ctypes.c_int, ctypes.c_int, ctypes.c_float, @@ -1114,7 +1126,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_long), + ctypes.POINTER(ctypes.c_int64), ] lib.bartorch_noir_free.restype = None lib.bartorch_noir_free.argtypes = [ctypes.c_void_p] @@ -1165,7 +1177,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_cuda_wait_for_stream.argtypes = [ctypes.c_void_p] lib.bartorch_cuda_signal_stream.restype = ctypes.c_int lib.bartorch_cuda_signal_stream.argtypes = [ctypes.c_void_p] - lib.bartorch_cuda_free_memory.restype = ctypes.c_long + lib.bartorch_cuda_free_memory.restype = ctypes.c_int64 lib.bartorch_cuda_free_memory.argtypes = [] lib.bartorch_finufft_set.restype = ctypes.c_int lib.bartorch_finufft_set.argtypes = [ctypes.c_char_p, ctypes.c_void_p] @@ -1195,7 +1207,7 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_finufft_usable_on.argtypes = [ctypes.c_int] lib.bartorch_finufft_usable.restype = ctypes.c_int lib.bartorch_finufft_usable.argtypes = [] - lib.bartorch_finufft_live_plans.restype = ctypes.c_long + lib.bartorch_finufft_live_plans.restype = ctypes.c_int64 lib.bartorch_finufft_live_plans.argtypes = [] lib.bartorch_last_error.restype = ctypes.c_char_p lib.bartorch_last_error.argtypes = [] @@ -1209,11 +1221,11 @@ def bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.bartorch_nufft_allow_fallback.argtypes = [ctypes.c_int] lib.bartorch_nufft_fallback_allowed.restype = ctypes.c_int lib.bartorch_nufft_fallback_allowed.argtypes = [] - lib.bartorch_nufft_counter.restype = ctypes.c_long + lib.bartorch_nufft_counter.restype = ctypes.c_int64 lib.bartorch_nufft_counter.argtypes = [ctypes.c_int] lib.bartorch_nufft_reset_counters.restype = None lib.bartorch_nufft_reset_counters.argtypes = [] - lib.bartorch_toeplitz_counter.restype = ctypes.c_long + lib.bartorch_toeplitz_counter.restype = ctypes.c_int64 lib.bartorch_toeplitz_counter.argtypes = [ctypes.c_int] lib.bartorch_toeplitz_reset_counters.restype = None lib.bartorch_toeplitz_reset_counters.argtypes = [] diff --git a/src/bartorch/_backend.py b/src/bartorch/_backend.py index 640a15eb..8396b8fc 100644 --- a/src/bartorch/_backend.py +++ b/src/bartorch/_backend.py @@ -84,9 +84,11 @@ def _mkl_library() -> Path | None: "libmkl_rt.so.2", "libmkl_rt.so.3", "libmkl_rt.so", + "mkl_rt.2.dll", + "mkl_rt.3.dll", ) for root in sorted(roots): - for sub in ("lib", "lib64"): + for sub in ("lib", "lib64", "Library/bin"): for name in names: candidate = root / sub / name if candidate.exists(): @@ -100,7 +102,9 @@ def _torch_library() -> Path | None: except ImportError: return None libdir = Path(torch.__file__).resolve().parent / "lib" - names = ["libtorch_cpu.dylib"] if sys.platform == "darwin" else ["libtorch_cpu.so"] + names = {"darwin": ["libtorch_cpu.dylib"], "win32": ["torch_cpu.dll"]}.get( + sys.platform, ["libtorch_cpu.so"] + ) for name in names: if (libdir / name).exists(): return libdir / name @@ -128,7 +132,10 @@ def _torch_providers() -> list[_Provider | None]: found.append( _open("Accelerate", "/System/Library/Frameworks/Accelerate.framework/Accelerate") ) - found.append(_open("process", None)) + # Windows has no handle on the process's own symbols: each DLL exports + # its own, and torch_cpu.dll is asked above. + if sys.platform != "win32": + found.append(_open("process", None)) return found diff --git a/src/bartorch/_finufft.py b/src/bartorch/_finufft.py index 6ffa491f..2743b322 100644 --- a/src/bartorch/_finufft.py +++ b/src/bartorch/_finufft.py @@ -68,7 +68,7 @@ def _library_path(package: str, stem: str) -> str | None: except ImportError: return None here = Path(module.__file__).resolve().parent - for name in (f"lib{stem}.so", f"lib{stem}.dylib"): + for name in (f"lib{stem}.so", f"lib{stem}.dylib", f"lib{stem}.dll"): candidate = here / name if candidate.exists(): return str(candidate) diff --git a/src/bartorch/_lib.py b/src/bartorch/_lib.py index 0cf661bf..bedd8ad2 100644 --- a/src/bartorch/_lib.py +++ b/src/bartorch/_lib.py @@ -27,13 +27,25 @@ def _library_names() -> list[str]: - # No Windows name: BART does not build there, so neither does this, and - # WSL2 is a Linux install like any other. if sys.platform == "darwin": return ["libbartorch.dylib"] + if sys.platform == "win32": + return ["libbartorch.dll"] return ["libbartorch.so"] +def _load(path: Path) -> ctypes.CDLL: + if sys.platform != "win32": + return ctypes.CDLL(str(path)) + # The Windows library imports libiomp5md.dll, the OpenMP runtime torch + # carries and loads; importing torch first makes it the copy the loader + # binds to, and torch's directory is where it is found otherwise. + import torch + + os.add_dll_directory(str(Path(torch.__file__).resolve().parent / "lib")) + return ctypes.CDLL(str(path)) + + def _package_dirs() -> list[Path]: """Every directory the package's files are in. @@ -74,7 +86,7 @@ def library() -> ctypes.CDLL: global _lib, _path if _lib is None: _path = _find_library() - _lib = bind(ctypes.CDLL(str(_path))) + _lib = bind(_load(_path)) return _lib diff --git a/src/bartorch/_macos_openmp.py b/src/bartorch/_macos_openmp.py index e4b8119e..113e6045 100644 --- a/src/bartorch/_macos_openmp.py +++ b/src/bartorch/_macos_openmp.py @@ -21,7 +21,7 @@ from __future__ import annotations -import os +import posixpath import shutil import struct import subprocess @@ -251,7 +251,7 @@ def rpath_for(finufft_lib: Path, torch_omp: Path) -> str: Relative to ``@loader_path`` so that the environment can be moved or renamed, which an absolute path into site-packages would not survive. """ - relative = os.path.relpath(torch_omp.parent, finufft_lib.parent) + relative = posixpath.relpath(torch_omp.parent.as_posix(), finufft_lib.parent.as_posix()) return f"@loader_path/{relative}" diff --git a/src/bartorch/_marshal.py b/src/bartorch/_marshal.py index 5b3cb980..a6c17c03 100644 --- a/src/bartorch/_marshal.py +++ b/src/bartorch/_marshal.py @@ -21,7 +21,7 @@ "float_buffer", "floats", "ints", - "longs", + "int64s", "null_apply", "null_release", "out_pointer", @@ -29,6 +29,7 @@ "padded_dims", "shape_from_dims", "text_buffer", + "uint64s", ] @@ -37,7 +38,7 @@ def padded_dims(shape: tuple[int, ...]) -> ctypes.Array: rev = list(shape)[::-1] if len(rev) > DIMS: raise ValueError(f"BART supports at most {DIMS} dimensions, got {len(rev)}") - return (ctypes.c_long * DIMS)(*(rev + [1] * (DIMS - len(rev)))) + return (ctypes.c_int64 * DIMS)(*(rev + [1] * (DIMS - len(rev)))) def padded_offsets(values, ndim: int, what: str) -> ctypes.Array: @@ -53,7 +54,7 @@ def padded_offsets(values, ndim: int, what: str) -> ctypes.Array: if ndim > DIMS: raise ValueError(f"BART supports at most {DIMS} dimensions, got {ndim}") rev = [int(v) for v in values][::-1] - return (ctypes.c_long * DIMS)(*(rev + [0] * (DIMS - ndim))) + return (ctypes.c_int64 * DIMS)(*(rev + [0] * (DIMS - ndim))) def padded_order(order, ndim: int) -> ctypes.Array: @@ -84,7 +85,7 @@ def dims(shape: tuple[int, ...]) -> tuple[int, ctypes.Array]: rev = list(shape)[::-1] or [1] if len(rev) > DIMS: raise ValueError(f"BART supports at most {DIMS} dimensions, got {len(rev)}") - return len(rev), (ctypes.c_long * len(rev))(*rev) + return len(rev), (ctypes.c_int64 * len(rev))(*rev) def shape_from_dims(vector: ctypes.Array, min_ndim: int) -> list[int]: @@ -100,8 +101,13 @@ def argv(args: list[str]) -> ctypes.Array: return (ctypes.c_char_p * len(args))(*[a.encode() for a in args]) -def longs(values) -> ctypes.Array: - return (ctypes.c_long * len(values))(*[int(v) for v in values]) +def int64s(values) -> ctypes.Array: + return (ctypes.c_int64 * len(values))(*[int(v) for v in values]) + + +def uint64s(values) -> ctypes.Array: + """Bit masks, as the ABI's ``uint64_t``.""" + return (ctypes.c_uint64 * len(values))(*[int(v) for v in values]) def ints(values) -> ctypes.Array: @@ -141,7 +147,7 @@ def null_release() -> ctypes.Array: def dim_vector() -> ctypes.Array: """A dimension vector of :data:`DIMS` entries for the library to fill in.""" - return (ctypes.c_long * DIMS)() + return (ctypes.c_int64 * DIMS)() def wide_dim_vector() -> ctypes.Array: @@ -151,12 +157,12 @@ def wide_dim_vector() -> ctypes.Array: thresholds a gradient's components, an axis beyond the image's, and total generalized variation's symmetric gradient adds a second (``tgv.c``). """ - return (ctypes.c_long * (DIMS + 2))() + return (ctypes.c_int64 * (DIMS + 2))() -def long_out() -> ctypes.c_long: +def int64_out() -> ctypes.c_int64: """A number the library writes into; read it back off ``.value``.""" - return ctypes.c_long() + return ctypes.c_int64() def int_out() -> ctypes.c_int: diff --git a/src/bartorch/nlop/base.py b/src/bartorch/nlop/base.py index 708b0601..87cc7f1f 100644 --- a/src/bartorch/nlop/base.py +++ b/src/bartorch/nlop/base.py @@ -741,7 +741,7 @@ def _create(self) -> Built: self.x._h.ptr, self.at, len(self.shape), - _marshal.longs(list(self.shape)[::-1]), + _marshal.int64s(list(self.shape)[::-1]), device=self.x.device, ) ishapes, oshapes = list(self.x.ishapes), list(self.x.oshapes) diff --git a/src/bartorch/nlop/callback.py b/src/bartorch/nlop/callback.py index f33b2494..0d6da806 100644 --- a/src/bartorch/nlop/callback.py +++ b/src/bartorch/nlop/callback.py @@ -110,10 +110,10 @@ def _create(self) -> Built: library().bartorch_nlop_callback_generic, len(self.oshapes), DIMS, - _marshal.longs(_flat(self.oshapes)), + _marshal.int64s(_flat(self.oshapes)), len(self.ishapes), DIMS, - _marshal.longs(_flat(self.ishapes)), + _marshal.int64s(_flat(self.ishapes)), fwd, der, adj, diff --git a/src/bartorch/optim/linear.py b/src/bartorch/optim/linear.py index e3c24071..0002742b 100644 --- a/src/bartorch/optim/linear.py +++ b/src/bartorch/optim/linear.py @@ -293,14 +293,14 @@ def _solve( block, family, shift_mode = _shared_options(terms) alpha, gamma = _shared_pairs(terms) - counter = _marshal.long_out() + counter = _marshal.int64_out() with _lock, _on_device(op.device or y.device): code = lib.bartorch_solve( op._h.ptr, algorithm.encode(), _marshal.argv([term.kind for term in terms]) if terms else None, - _marshal.longs([f for f, _ in flags]) if terms else None, - _marshal.longs([j for _, j in flags]) if terms else None, + _marshal.uint64s([f for f, _ in flags]) if terms else None, + _marshal.uint64s([j for _, j in flags]) if terms else None, _marshal.floats([term.weight for term in terms]) if terms else None, _marshal.ints([term.count for term in terms]) if terms else None, _marshal.pointers(handles) if handles else None, @@ -410,13 +410,13 @@ def _penalties(terms, image_shape: tuple[int, ...]): _ensure_ready() handles = _marshal.pointer_buffer(_MAX_PENALTIES) - count, svars = _marshal.int_out(), _marshal.long_out() + count, svars = _marshal.int_out(), _marshal.int64_out() with _lock: code = library().bartorch_prox_set_create( len(terms), _marshal.argv([term.kind for term in terms]), - _marshal.longs([f for f, _ in flags]), - _marshal.longs([j for _, j in flags]), + _marshal.uint64s([f for f, _ in flags]), + _marshal.uint64s([j for _, j in flags]), _marshal.floats([term.weight for term in terms]), _marshal.ints([term.count for term in terms]), block, diff --git a/src/csrc/abi/api.c b/src/csrc/abi/api.c index 39dbf317..e01b7efb 100644 --- a/src/csrc/abi/api.c +++ b/src/csrc/abi/api.c @@ -17,6 +17,12 @@ #include "include/bartorch.h" #include "substitute/backend.h" +/* The ABI spells BART's dimension, stride and flag types by their widths, so + * a dimension vector crosses as it stands, without a copy. */ +_Static_assert(_Generic((bart_dim_t*)0, int64_t*: 1, default: 0), "BART dimensions are the ABI's int64_t"); +_Static_assert(_Generic((bart_stride_t*)0, int64_t*: 1, default: 0), "BART strides are the ABI's int64_t"); +_Static_assert(_Generic((bart_flags_t*)0, uint64_t*: 1, default: 0), "BART flags are the ABI's uint64_t"); + extern int bart_command(int len, char* buf, int argc, char* argv[]); /* A failed assertion reaches the caller instead of the process. @@ -35,14 +41,43 @@ extern int bart_command(int len, char* buf, int argc, char* argv[]); * Apple's libc calls __assert_rtn(func, file, line, expr) -- a different name, * a different order, and a different type for the line. Answering only the * first left every one of BART's assertions aborting on macOS, which is a - * process death where every other platform gets an exception. + * process death where every other platform gets an exception. MinGW's + * assert() calls the C runtime's _assert, or _wassert under the Universal C + * Runtime, through the import table, so the import-table slots are what is + * answered there. */ -#ifdef __APPLE__ +#if defined(_WIN32) + +#include + +void _assert(const char* assertion, const char* file, unsigned line); +void _wassert(const wchar_t* assertion, const wchar_t* file, unsigned line); + +__attribute__((__noreturn__)) +void _assert(const char* assertion, const char* file, unsigned line) +{ + error("Assertion '%s' failed in %s:%u\n", assertion, file, line); +} + +__attribute__((__noreturn__)) +void _wassert(const wchar_t* assertion, const wchar_t* file, unsigned line) +{ + error("Assertion '%ls' failed in %ls:%u\n", assertion, file, line); +} + +/* declares both dllimport, so BART reaches them through these + * slots, and the C++ runtime linked in statically calls them by name. + * Defining both keeps the C runtime's import library from supplying + * either. */ +void (*__imp__assert)(const char*, const char*, unsigned) = _assert; +void (*__imp__wassert)(const wchar_t*, const wchar_t*, unsigned) = _wassert; + +#elif defined(__APPLE__) -__attribute__((noreturn)) +__attribute__((__noreturn__)) void __assert_rtn(const char* function, const char* file, int line, const char* assertion); -__attribute__((noreturn)) +__attribute__((__noreturn__)) void __assert_rtn(const char* function, const char* file, int line, const char* assertion) { error("Assertion '%s' failed in %s:%d (%s)\n", assertion, file, line, function); diff --git a/src/csrc/abi/cuda.c b/src/csrc/abi/cuda.c index 4d0caf60..c5ff3419 100644 --- a/src/csrc/abi/cuda.c +++ b/src/csrc/abi/cuda.c @@ -16,7 +16,12 @@ #include #include #include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#else #include +#endif #include "include/bartorch.h" @@ -34,15 +39,21 @@ struct prefault { pthread_t thread; volatile char* ptr; - long size; + bart_dim_t size; }; static void* prefault_run(void* arg) { struct prefault* p = arg; - long page = sysconf(_SC_PAGESIZE); +#ifdef _WIN32 + SYSTEM_INFO info; + GetSystemInfo(&info); + bart_dim_t page = info.dwPageSize; +#else + bart_dim_t page = sysconf(_SC_PAGESIZE); +#endif - for (long i = 0; i < p->size; i += page) + for (bart_dim_t i = 0; i < p->size; i += page) p->ptr[i] = p->ptr[i]; p->ptr[p->size - 1] = p->ptr[p->size - 1]; @@ -50,7 +61,7 @@ static void* prefault_run(void* arg) return NULL; } -void* bartorch_host_prefault_begin(void* ptr, long size) +void* bartorch_host_prefault_begin(void* ptr, int64_t size) { if ((NULL == ptr) || (0 >= size)) return NULL; @@ -266,7 +277,7 @@ int bartorch_cuda_signal_stream(void* stream) * is what the function a normal is streamed from is kept in. Where there is * no card, or where the card will not lock that much, it is ordinary * memory. */ -void* bartorch_host_alloc(long size, int pinned) +void* bartorch_host_alloc(int64_t size, int pinned) { if (0 >= size) return NULL; @@ -336,7 +347,7 @@ static bool bounce_open(void) static void host_copy(void* dst, const void* src, size_t n) { - long parts = (long)(n >> 22); + bart_dim_t parts = (bart_dim_t)(n >> 22); if (parts < 2) { @@ -345,7 +356,7 @@ static void host_copy(void* dst, const void* src, size_t n) } #pragma omp parallel for - for (long p = 0; p < parts; p++) { + for (bart_dim_t p = 0; p < parts; p++) { size_t a = n * (size_t)p / (size_t)parts; size_t b = n * (size_t)(p + 1) / (size_t)parts; @@ -354,7 +365,7 @@ static void host_copy(void* dst, const void* src, size_t n) } } -int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) +int bartorch_cuda_copy_pageable(void* dst, const void* src, int64_t size) { if ((-1 == current_device) || (0 >= size) || !bounce_open()) return -1; @@ -365,12 +376,12 @@ int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) for (int i = 0; i < 2; i++) cudaEventSynchronize(bounce_done[i]); - long chunks = (size + BOUNCE_BYTES - 1) / BOUNCE_BYTES; + bart_dim_t chunks = (size + BOUNCE_BYTES - 1) / BOUNCE_BYTES; - for (long k = 0; k <= chunks; k++) { + for (bart_dim_t k = 0; k <= chunks; k++) { - long off = k * (long)BOUNCE_BYTES; - long len = MIN((long)BOUNCE_BYTES, size - off); + bart_stride_t off = k * (bart_dim_t)BOUNCE_BYTES; + bart_dim_t len = MIN((bart_dim_t)BOUNCE_BYTES, size - off); int b = (int)(k & 1); if (to_host) { @@ -386,8 +397,8 @@ int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) if (0 < k) { - long poff = (k - 1) * (long)BOUNCE_BYTES; - long plen = MIN((long)BOUNCE_BYTES, size - poff); + bart_dim_t poff = (k - 1) * (bart_dim_t)BOUNCE_BYTES; + bart_dim_t plen = MIN((bart_dim_t)BOUNCE_BYTES, size - poff); cudaEventSynchronize(bounce_done[b ^ 1]); host_copy((char*)dst + poff, bounce[b ^ 1], (size_t)plen); @@ -413,7 +424,7 @@ int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) * its own and a copy out of it runs behind whatever the card is doing. For * gigabytes it is a fraction of a second here, where allocating as much * page-locked takes seconds. */ -int bartorch_cuda_host_register(void* ptr, long size) +int bartorch_cuda_host_register(void* ptr, int64_t size) { if ((NULL == ptr) || (0 >= size) || (-1 == current_device)) return -1; @@ -500,7 +511,7 @@ void bartorch_cuda_stage_close(void* stage) } /* Start a slot's crossing, once the card has finished reading what is in it. */ -int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, long size) +int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, int64_t size) { if ((NULL == stage) || (0 > slot) || (1 < slot)) return -1; @@ -542,7 +553,7 @@ int bartorch_on_device(const void* ptr) return cuda_ondevice(ptr) ? 1 : 0; } -long bartorch_cuda_free_memory(void) +int64_t bartorch_cuda_free_memory(void) { size_t free_bytes = 0; size_t total_bytes = 0; @@ -553,7 +564,7 @@ long bartorch_cuda_free_memory(void) return -1; } - return (long)free_bytes; + return (bart_dim_t)free_bytes; } #else /* !USE_CUDA */ @@ -569,17 +580,17 @@ int bartorch_cuda_use_memcache(int enable) { (void)enable; return -1; } void bartorch_cuda_memcache_clear_all(void) { } int bartorch_cuda_wait_for_stream(void* stream) { (void)stream; return -1; } int bartorch_cuda_signal_stream(void* stream) { (void)stream; return -1; } -void* bartorch_host_alloc(long size, int pinned) { (void)pinned; return (0 < size) ? xmalloc((size_t)size) : NULL; } +void* bartorch_host_alloc(int64_t size, int pinned) { (void)pinned; return (0 < size) ? xmalloc((size_t)size) : NULL; } void bartorch_host_free(void* ptr) { if (NULL != ptr) xfree(ptr); } int bartorch_cuda_stage_open(void** stage) { (void)stage; return -1; } -int bartorch_cuda_copy_pageable(void* dst, const void* src, long size) { (void)dst; (void)src; (void)size; return -1; } -int bartorch_cuda_host_register(void* ptr, long size) { (void)ptr; (void)size; return -1; } +int bartorch_cuda_copy_pageable(void* dst, const void* src, int64_t size) { (void)dst; (void)src; (void)size; return -1; } +int bartorch_cuda_host_register(void* ptr, int64_t size) { (void)ptr; (void)size; return -1; } void bartorch_cuda_host_unregister(void* ptr) { (void)ptr; } void bartorch_cuda_stage_close(void* stage) { (void)stage; } -int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, long size) +int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, int64_t size) { (void)stage; (void)slot; (void)dst; (void)src; (void)size; return -1; } int bartorch_cuda_stage_wait(void* stage, int slot) { (void)stage; (void)slot; return -1; } int bartorch_cuda_stage_release(void* stage, int slot) { (void)stage; (void)slot; return -1; } -long bartorch_cuda_free_memory(void) { return -1; } +int64_t bartorch_cuda_free_memory(void) { return -1; } #endif diff --git a/src/csrc/abi/host_reads.c b/src/csrc/abi/host_reads.c index 0adc48a0..75ce39f6 100644 --- a/src/csrc/abi/host_reads.c +++ b/src/csrc/abi/host_reads.c @@ -23,12 +23,12 @@ #include "include/bartorch.h" -extern void bart_estimate_im_dims(int N, unsigned long flags, long dims[N], const long tdims[N], const complex float* traj); -extern void bart_estimate_fast_sq_im_dims(int N, long dims[3], const long tdims[N], const complex float* traj); +extern void bart_estimate_im_dims(int N, bart_flags_t flags, bart_dim_t dims[N], const bart_dim_t tdims[N], const complex float* traj); +extern void bart_estimate_fast_sq_im_dims(int N, bart_dim_t dims[3], const bart_dim_t tdims[N], const complex float* traj); extern float bart_estimate_scaling_norm(float rescale, int imsize, complex float* tmpnorm, bool compat, float p); /* A copy of the trajectory on the host, or NULL when it is already there. */ -static complex float* host_copy(int N, const long tdims[N], const complex float* traj) +static complex float* host_copy(int N, const bart_dim_t tdims[N], const complex float* traj) { if (!bartorch_on_device(traj)) return NULL; @@ -40,7 +40,7 @@ static complex float* host_copy(int N, const long tdims[N], const complex float* return host; } -void estimate_im_dims(int N, unsigned long flags, long dims[N], const long tdims[N], const complex float* traj) +void estimate_im_dims(int N, bart_flags_t flags, bart_dim_t dims[N], const bart_dim_t tdims[N], const complex float* traj) { complex float* host = host_copy(N, tdims, traj); @@ -49,7 +49,7 @@ void estimate_im_dims(int N, unsigned long flags, long dims[N], const long tdims md_free(host); } -void estimate_fast_sq_im_dims(int N, long dims[3], const long tdims[N], const complex float* traj) +void estimate_fast_sq_im_dims(int N, bart_dim_t dims[3], const bart_dim_t tdims[N], const complex float* traj) { complex float* host = host_copy(N, tdims, traj); @@ -66,7 +66,7 @@ float estimate_scaling_norm(float rescale, int imsize, complex float* tmpnorm, b if (!bartorch_on_device(tmpnorm)) return bart_estimate_scaling_norm(rescale, imsize, tmpnorm, compat, p); - long dims[1] = { imsize }; + bart_dim_t dims[1] = { imsize }; complex float* host = md_alloc(1, dims, CFL_SIZE); md_copy(1, dims, host, tmpnorm, CFL_SIZE); diff --git a/src/csrc/abi/memcfl.c b/src/csrc/abi/memcfl.c index d0f68d69..72ff87a1 100644 --- a/src/csrc/abi/memcfl.c +++ b/src/csrc/abi/memcfl.c @@ -27,7 +27,7 @@ struct entry { char* name; int D; - long* dims; + bart_dim_t* dims; complex float* data; int refcount; enum owner owner; @@ -78,13 +78,13 @@ static struct entry* find_data_locked(const complex float* data) return NULL; } -static void insert_locked(const char* name, int D, const long* dims, complex float* data, enum owner owner) +static void insert_locked(const char* name, int D, const bart_dim_t* dims, complex float* data, enum owner owner) { struct entry* e = malloc(sizeof(*e)); e->name = strdup(name); e->D = D; - e->dims = malloc(sizeof(long) * (size_t)D); - memcpy(e->dims, dims, sizeof(long) * (size_t)D); + e->dims = malloc(sizeof(bart_dim_t) * (size_t)D); + memcpy(e->dims, dims, sizeof(bart_dim_t) * (size_t)D); e->data = data; e->refcount = 1; e->owner = owner; @@ -123,14 +123,14 @@ static void unlink_locked(struct entry** ep) /* BART-facing API (misc/memcfl.h). */ -void memcfl_register(const char* name, int D, const long dims[D], complex float* data, bool managed) +void memcfl_register(const char* name, int D, const bart_dim_t dims[D], complex float* data, bool managed) { pthread_mutex_lock(&lock); insert_locked(name, D, dims, data, managed ? OWNER_XMALLOC : OWNER_HOST); pthread_mutex_unlock(&lock); } -complex float* memcfl_create(const char* name, int D, const long dims[D]) +complex float* memcfl_create(const char* name, int D, const bart_dim_t dims[D]) { pthread_mutex_lock(&lock); @@ -200,7 +200,7 @@ const char** memcfl_list_all(void) return out; } -complex float* memcfl_load(const char* name, int D, long dims[D]) +complex float* memcfl_load(const char* name, int D, bart_dim_t dims[D]) { pthread_mutex_lock(&lock); @@ -265,7 +265,7 @@ void memcfl_unlink(const char* name) /* Host-facing API (include/bartorch.h). */ -int bartorch_register(const char* name, int D, const long* dims, void* data) +int bartorch_register(const char* name, int D, const int64_t* dims, void* data) { if ((D < 1) || (D > BARTORCH_DIMS) || (NULL == data)) return -1; @@ -291,7 +291,7 @@ int bartorch_exists(const char* name) return memcfl_exists(name) ? 1 : 0; } -int bartorch_lookup(const char* name, int D, long* dims, void** data) +int bartorch_lookup(const char* name, int D, int64_t* dims, void** data) { pthread_mutex_lock(&lock); diff --git a/src/csrc/compat/libiomp5md.def b/src/csrc/compat/libiomp5md.def new file mode 100644 index 00000000..81eecef5 --- /dev/null +++ b/src/csrc/compat/libiomp5md.def @@ -0,0 +1,68 @@ +; The OpenMP entry points clang's code calls, exported by the Intel OpenMP +; runtime torch loads on Windows. An import library made from this file +; binds the library to that runtime, so the process has one OpenMP runtime. +; A name clang emits and this list lacks is a link error. +LIBRARY libiomp5md.dll +EXPORTS + __kmpc_barrier + __kmpc_critical + __kmpc_critical_with_hint + __kmpc_dispatch_fini_4 + __kmpc_dispatch_fini_4u + __kmpc_dispatch_fini_8 + __kmpc_dispatch_fini_8u + __kmpc_dispatch_init_4 + __kmpc_dispatch_init_4u + __kmpc_dispatch_init_8 + __kmpc_dispatch_init_8u + __kmpc_dispatch_next_4 + __kmpc_dispatch_next_4u + __kmpc_dispatch_next_8 + __kmpc_dispatch_next_8u + __kmpc_end_critical + __kmpc_end_master + __kmpc_end_ordered + __kmpc_end_reduce + __kmpc_end_reduce_nowait + __kmpc_end_serialized_parallel + __kmpc_end_single + __kmpc_flush + __kmpc_for_static_fini + __kmpc_for_static_init_4 + __kmpc_for_static_init_4u + __kmpc_for_static_init_8 + __kmpc_for_static_init_8u + __kmpc_fork_call + __kmpc_global_thread_num + __kmpc_master + __kmpc_omp_taskwait + __kmpc_ordered + __kmpc_push_num_threads + __kmpc_reduce + __kmpc_reduce_nowait + __kmpc_serialized_parallel + __kmpc_single + __kmpc_threadprivate_cached + omp_destroy_lock + omp_destroy_nest_lock + omp_get_ancestor_thread_num + omp_get_dynamic + omp_get_level + omp_get_max_threads + omp_get_nested + omp_get_num_procs + omp_get_num_threads + omp_get_team_size + omp_get_thread_num + omp_get_wtime + omp_in_parallel + omp_init_lock + omp_init_nest_lock + omp_set_dynamic + omp_set_lock + omp_set_nest_lock + omp_set_nested + omp_set_num_threads + omp_test_lock + omp_unset_lock + omp_unset_nest_lock diff --git a/src/csrc/include/bartorch.h b/src/csrc/include/bartorch.h index e61b80bf..162466f6 100644 --- a/src/csrc/include/bartorch.h +++ b/src/csrc/include/bartorch.h @@ -13,12 +13,17 @@ #define BARTORCH_H #include +#include #ifdef __cplusplus extern "C" { #endif +#ifdef _WIN32 +#define BARTORCH_API __declspec(dllexport) +#else #define BARTORCH_API __attribute__((visibility("default"))) +#endif /* Number of dimensions BART carries for every array. */ #define BARTORCH_DIMS 16 @@ -41,7 +46,7 @@ enum bartorch_log_level { * free callback receives the same pointer. Returning NULL aborts the * running command with an error. */ -typedef void* (*bartorch_alloc_fn)(void* ctx, int D, const long* dims); +typedef void* (*bartorch_alloc_fn)(void* ctx, int D, const int64_t* dims); typedef void (*bartorch_free_fn)(void* ctx, void* data); /* Receives every BART log line at or below the current debug level. */ @@ -63,9 +68,9 @@ BARTORCH_API void bartorch_set_num_threads(int n); * freed by BART; arrays BART creates itself come from the allocator * callback and are released through the free callback on unlink. */ -BARTORCH_API int bartorch_register(const char* name, int D, const long* dims, void* data); +BARTORCH_API int bartorch_register(const char* name, int D, const int64_t* dims, void* data); BARTORCH_API int bartorch_exists(const char* name); -BARTORCH_API int bartorch_lookup(const char* name, int D, long* dims, void** data); +BARTORCH_API int bartorch_lookup(const char* name, int D, int64_t* dims, void** data); BARTORCH_API int bartorch_unlink(const char* name); BARTORCH_API int bartorch_unlink_all(void); @@ -146,7 +151,7 @@ BARTORCH_API int bartorch_nufft_bf16(void); * it is one. Ordinary memory when it is not asked for, or where there is no * card: page-locking is not free, and only a crossing that overlaps something * repays it. */ -BARTORCH_API void* bartorch_host_alloc(long size, int pinned); +BARTORCH_API void* bartorch_host_alloc(int64_t size, int pinned); BARTORCH_API void bartorch_host_free(void* ptr); /* A stream of its own for bringing a function over, so the set that will be @@ -155,18 +160,18 @@ BARTORCH_API void bartorch_host_free(void* ptr); * until it has arrived, `release` says BART is done reading it. */ BARTORCH_API int bartorch_cuda_stage_open(void** stage); BARTORCH_API void bartorch_cuda_stage_close(void* stage); -BARTORCH_API int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, long size); +BARTORCH_API int bartorch_cuda_stage_copy(void* stage, int slot, void* dst, const void* src, int64_t size); BARTORCH_API int bartorch_cuda_stage_wait(void* stage, int slot); BARTORCH_API int bartorch_cuda_stage_release(void* stage, int slot); /* A copy between the card and pageable host memory through two page-locked * buffers, so it runs near the bus's rate even into pages never touched. */ -BARTORCH_API int bartorch_cuda_copy_pageable(void* dst, const void* src, long size); -BARTORCH_API void* bartorch_host_prefault_begin(void* ptr, long size); +BARTORCH_API int bartorch_cuda_copy_pageable(void* dst, const void* src, int64_t size); +BARTORCH_API void* bartorch_host_prefault_begin(void* ptr, int64_t size); BARTORCH_API void bartorch_host_prefault_end(void* handle); /* Page-lock, and release, host memory that already exists. */ -BARTORCH_API int bartorch_cuda_host_register(void* ptr, long size); +BARTORCH_API int bartorch_cuda_host_register(void* ptr, int64_t size); BARTORCH_API void bartorch_cuda_host_unregister(void* ptr); BARTORCH_API void bartorch_sense_set_coil_batch(int coils); @@ -179,13 +184,13 @@ BARTORCH_API void bartorch_sense_set_fold_maps(int enable); BARTORCH_API int bartorch_sense_fold_maps(void); /* Operators built since the last reset: 0 with the coil loop, 1 as BART's * own chain because the arrangement could not be sliced. */ -BARTORCH_API long bartorch_sense_counter(int which); +BARTORCH_API int64_t bartorch_sense_counter(int which); BARTORCH_API void bartorch_sense_reset_counters(void); BARTORCH_API int bartorch_fft_set(const char* symbol, void* fn); BARTORCH_API int bartorch_fft_usable(void); /* Plans built since the last reset: 0 by MKL, 1 by the built-in transform. */ -BARTORCH_API long bartorch_fft_counter(int which); +BARTORCH_API int64_t bartorch_fft_counter(int which); BARTORCH_API void bartorch_fft_reset_counters(void); /* @@ -209,16 +214,16 @@ typedef int (*bartorch_generic_apply_fn)(void* ctx, int N, void** args); typedef int (*bartorch_pair_apply_fn)(void* ctx, int o, int i, void* dst, const void* src); typedef void (*bartorch_release_fn)(void* ctx); -BARTORCH_API bartorch_linop* bartorch_linop_callback(int ON, const long* odims, int IN, const long* idims, +BARTORCH_API bartorch_linop* bartorch_linop_callback(int ON, const int64_t* odims, int IN, const int64_t* idims, bartorch_apply_fn forward, bartorch_apply_fn adjoint, bartorch_apply_fn normal, void* ctx, bartorch_release_fn release); -BARTORCH_API bartorch_linop* bartorch_linop_fft(int N, const long* dims, unsigned long flags, int inverse, int centered); -BARTORCH_API bartorch_linop* bartorch_linop_cdiag(int N, const long* dims, unsigned long flags, const void* diag); -BARTORCH_API bartorch_linop* bartorch_linop_fmac(int N, const long* odims, const long* idims, const long* tdims, const void* tensor); -BARTORCH_API bartorch_linop* bartorch_linop_sampling(const long* dims, const long* pat_dims, const void* pattern); -BARTORCH_API bartorch_linop* bartorch_linop_nufft(int N, const long* ksp_dims, const long* cim_dims, const long* traj_dims, - const void* traj, const long* wgh_dims, const void* weights, - const long* bas_dims, const void* basis, int toeplitz, float os, float width); +BARTORCH_API bartorch_linop* bartorch_linop_fft(int N, const int64_t* dims, uint64_t flags, int inverse, int centered); +BARTORCH_API bartorch_linop* bartorch_linop_cdiag(int N, const int64_t* dims, uint64_t flags, const void* diag); +BARTORCH_API bartorch_linop* bartorch_linop_fmac(int N, const int64_t* odims, const int64_t* idims, const int64_t* tdims, const void* tensor); +BARTORCH_API bartorch_linop* bartorch_linop_sampling(const int64_t* dims, const int64_t* pat_dims, const void* pattern); +BARTORCH_API bartorch_linop* bartorch_linop_nufft(int N, const int64_t* ksp_dims, const int64_t* cim_dims, const int64_t* traj_dims, + const void* traj, const int64_t* wgh_dims, const void* weights, + const int64_t* bas_dims, const void* basis, int toeplitz, float os, float width); /* Write a new diagonal into a `cdiag` or a sampling operator, which is the * one already built rather than a second one. The values are copied and the * cached normal is dropped, so an operator this is applied to -- and every @@ -226,7 +231,7 @@ BARTORCH_API bartorch_linop* bartorch_linop_nufft(int N, const long* ksp_dims, c * the next application. `ddims` is what the operator was built with: the * shape selected by the broadcast flags, not the operator's own. Returns * non-zero where the operator is not a diagonal or the shape disagrees. */ -BARTORCH_API int bartorch_linop_set_diagonal(const bartorch_linop* op, int N, const long* ddims, +BARTORCH_API int bartorch_linop_set_diagonal(const bartorch_linop* op, int N, const int64_t* ddims, const void* diag); /* SENSE, over sensitivities held as maps or as k-space kernels, walking the * coils a slab at a time. A NULL trajectory makes the Cartesian operator, @@ -239,11 +244,11 @@ BARTORCH_API int bartorch_linop_set_diagonal(const bartorch_linop* op, int N, co * are slowest in memory, is built on BART's dimensions without a copy. The * block operator is referenced, so the caller still frees it. */ BARTORCH_API bartorch_linop* bartorch_linop_blocks(const bartorch_linop* block, int N, - const long* odims, const long* idims, long n); + const int64_t* odims, const int64_t* idims, int64_t n); /* `op` over the same memory described by `idims` and `odims`, with nothing * copied. The operator is referenced, so the caller still frees it. */ BARTORCH_API bartorch_linop* bartorch_linop_reshaped(const bartorch_linop* op, int N, - const long* odims, const long* idims); + const int64_t* odims, const int64_t* idims); /* * The MRI encoding form. Every encoding this library builds reduces to * @@ -275,26 +280,26 @@ struct bartorch_encoding { /* The whole operator's dimensions -- spatial axes, coils, sets of * maps, coefficients -- and one coil's samples with the coefficients * a basis contracts still on them. */ - const long* max_dims; - const long* ksp_dims; + const int64_t* max_dims; + const int64_t* ksp_dims; /* The image-side factor: coil sensitivities as maps, or as the * k-space kernels they band-limit to, inflated a slab at a time. */ - const long* sens_dims; + const int64_t* sens_dims; const void* sens; int kernels; /* The k-space factors. Each pointer may be NULL, and its dimensions * are then not read. */ - const long* pat_dims; + const int64_t* pat_dims; const void* pattern; - const long* bas_dims; + const int64_t* bas_dims; const void* basis; - const long* wgh_dims; + const int64_t* wgh_dims; const void* weights; /* A NUFFT's trajectory, in grid units. */ - const long* traj_dims; + const int64_t* traj_dims; const void* traj; /* A stack whose kz lies on the image's own z grid, decoupled: the * trajectory is one position's in-plane shots, kz zero, the transform @@ -304,25 +309,25 @@ struct bartorch_encoding { /* The positions along z a stack's blocks of shots lie at, one per block, * where they are not every position in order; NULL is every position, * block j at position j. */ - long stack_count; - const long* stack_positions; + int64_t stack_count; + const int64_t* stack_positions; /* A table of the phase encodes that were sampled, instead of a dense - * pattern: `frames` x `shots` places of `components` long indices + * pattern: `frames` x `shots` places of `components` indices * each -- (y) for a 2D image, (z, y) for a 3D one, -1 for padding -- * with the whole readout along each. `kspace_readout` says the * samples are in k-space along the readout rather than transformed * back along it. NULL `positions` is dense samples. */ - long frames; - long shots; + int64_t frames; + int64_t shots; int components; - const long* positions; + const int64_t* positions; int kspace_readout; /* A wave: the oversampled readout the coil images are zero-filled to, * the point spread function over it, and whether its two transforms * are centred and unitary rather than BART's own. */ - long readout; + int64_t readout; const void* psf; int centred; @@ -330,10 +335,10 @@ struct bartorch_encoding { * diag(image_l): off-resonance by time segmentation. Each weight is * laid out on its own dimensions, the terms contiguous one after * another. Zero segments is no contraction. */ - long segments; - const long* segment_sample_dims; + int64_t segments; + const int64_t* segment_sample_dims; const void* segment_sample; - const long* segment_image_dims; + const int64_t* segment_image_dims; const void* segment_image; /* The BART dimension a batch the sensitivities vary along lies on, or @@ -347,7 +352,7 @@ struct bartorch_encoding { * trajectory: the extent of each such axis, and one elsewhere, or NULL * for none. Every item is its own transform and normal kernel under one * coil loop. */ - const long* item_dims; + const int64_t* item_dims; /* A k-space factor that differs between sets of maps, applied after * the transform, with the sets summed over after it: the slice phase @@ -355,7 +360,7 @@ struct bartorch_encoding { * sensitivities rather than being contracted by them, so the transform * runs once per set -- which is what the sum being on the far side of * it costs. NULL leaves the sets where the sensitivities sum them. */ - const long* slice_dims; + const int64_t* slice_dims; const void* slice; /* The closed-form normal rather than the two applications. */ @@ -396,11 +401,11 @@ enum bartorch_encoding_count { /* Forms built with a transform per item. */ BARTORCH_ENCODING_ITEMS = 8, }; -BARTORCH_API long bartorch_encoding_counter(int which); +BARTORCH_API int64_t bartorch_encoding_counter(int which); BARTORCH_API void bartorch_encoding_reset_counters(void); /* Normals of a Cartesian encoding applied through cuFFT's callbacks since the * library was loaded; the rest were applied as BART's chain of operators. */ -BARTORCH_API long bartorch_grid_fused(void); +BARTORCH_API int64_t bartorch_grid_fused(void); /* `a`, answering `normal` when it is asked for A^H A, rather than the adjoint * chained onto the forward. */ BARTORCH_API bartorch_linop* bartorch_linop_with_normal(const bartorch_linop* a, const bartorch_linop* normal); @@ -410,33 +415,33 @@ BARTORCH_API bartorch_linop* bartorch_linop_adjoint_op(const bartorch_linop* a); BARTORCH_API bartorch_linop* bartorch_linop_stack_cod(int n, const bartorch_linop** ops, int stack_dim); BARTORCH_API bartorch_linop* bartorch_linop_stack(int cod_dim, int dom_dim, const bartorch_linop* a, const bartorch_linop* b); BARTORCH_API bartorch_linop* bartorch_linop_normal_op(const bartorch_linop* a); -BARTORCH_API bartorch_linop* bartorch_linop_scale(int N, const long* dims, float re, float im); -BARTORCH_API bartorch_linop* bartorch_linop_zconj(int N, const long* dims); -BARTORCH_API bartorch_linop* bartorch_linop_identity(int N, const long* dims); -BARTORCH_API bartorch_linop* bartorch_linop_null(int NO, const long* odims, int NI, const long* idims); +BARTORCH_API bartorch_linop* bartorch_linop_scale(int N, const int64_t* dims, float re, float im); +BARTORCH_API bartorch_linop* bartorch_linop_zconj(int N, const int64_t* dims); +BARTORCH_API bartorch_linop* bartorch_linop_identity(int N, const int64_t* dims); +BARTORCH_API bartorch_linop* bartorch_linop_null(int NO, const int64_t* odims, int NI, const int64_t* idims); BARTORCH_API double bartorch_linop_maxeigen(const bartorch_linop* a); -BARTORCH_API bartorch_linop* bartorch_linop_zreal(int N, const long* dims); -BARTORCH_API bartorch_linop* bartorch_linop_rdiag(int N, const long* dims, unsigned long flags, const void* diag); -BARTORCH_API bartorch_linop* bartorch_linop_matrix(int N, const long* odims, const long* idims, const long* mdims, const void* matrix); -BARTORCH_API bartorch_linop* bartorch_linop_conv(int N, unsigned long flags, int ctype, int cmode, const long* odims, const long* idims, const long* kdims, const void* kernel); -BARTORCH_API bartorch_linop* bartorch_linop_grad(int N, const long* dims, int d, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_sum(int N, const long* dims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_scaled_sum(int N, const long* dims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_avg(int N, const long* dims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_repmat(int N, const long* odims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_flip(int N, const long* dims, unsigned long flags); -BARTORCH_API bartorch_linop* bartorch_linop_hankel(int N, const long* dims, int dim, int window_dim, int window); -BARTORCH_API bartorch_linop* bartorch_linop_reshape(int NO, const long* odims, int NI, const long* idims); -BARTORCH_API bartorch_linop* bartorch_linop_resize(int N, const long* odims, const long* idims); -BARTORCH_API bartorch_linop* bartorch_linop_extract(int N, const long* pos, const long* odims, const long* idims); -BARTORCH_API bartorch_linop* bartorch_linop_transpose(int N, int a, int b, const long* dims); -BARTORCH_API bartorch_linop* bartorch_linop_permute(int N, const int* order, const long* idims); -BARTORCH_API bartorch_linop* bartorch_linop_shift(int N, const long* dims, int dim, long shift, int pad); -BARTORCH_API bartorch_linop* bartorch_linop_padding(int N, const long* dims, int pad, const long* before, const long* after); +BARTORCH_API bartorch_linop* bartorch_linop_zreal(int N, const int64_t* dims); +BARTORCH_API bartorch_linop* bartorch_linop_rdiag(int N, const int64_t* dims, uint64_t flags, const void* diag); +BARTORCH_API bartorch_linop* bartorch_linop_matrix(int N, const int64_t* odims, const int64_t* idims, const int64_t* mdims, const void* matrix); +BARTORCH_API bartorch_linop* bartorch_linop_conv(int N, uint64_t flags, int ctype, int cmode, const int64_t* odims, const int64_t* idims, const int64_t* kdims, const void* kernel); +BARTORCH_API bartorch_linop* bartorch_linop_grad(int N, const int64_t* dims, int d, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_sum(int N, const int64_t* dims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_scaled_sum(int N, const int64_t* dims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_avg(int N, const int64_t* dims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_repmat(int N, const int64_t* odims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_flip(int N, const int64_t* dims, uint64_t flags); +BARTORCH_API bartorch_linop* bartorch_linop_hankel(int N, const int64_t* dims, int dim, int window_dim, int window); +BARTORCH_API bartorch_linop* bartorch_linop_reshape(int NO, const int64_t* odims, int NI, const int64_t* idims); +BARTORCH_API bartorch_linop* bartorch_linop_resize(int N, const int64_t* odims, const int64_t* idims); +BARTORCH_API bartorch_linop* bartorch_linop_extract(int N, const int64_t* pos, const int64_t* odims, const int64_t* idims); +BARTORCH_API bartorch_linop* bartorch_linop_transpose(int N, int a, int b, const int64_t* dims); +BARTORCH_API bartorch_linop* bartorch_linop_permute(int N, const int* order, const int64_t* idims); +BARTORCH_API bartorch_linop* bartorch_linop_shift(int N, const int64_t* dims, int dim, int64_t shift, int pad); +BARTORCH_API bartorch_linop* bartorch_linop_padding(int N, const int64_t* dims, int pad, const int64_t* before, const int64_t* after); BARTORCH_API int bartorch_linop_has_pseudo_inv(const bartorch_linop* h); BARTORCH_API int bartorch_linop_pseudo_inv(const bartorch_linop* h, float lambda, void* dst, const void* src); -BARTORCH_API int bartorch_linop_domain(const bartorch_linop* h, int N, long* dims); -BARTORCH_API int bartorch_linop_codomain(const bartorch_linop* h, int N, long* dims); +BARTORCH_API int bartorch_linop_domain(const bartorch_linop* h, int N, int64_t* dims); +BARTORCH_API int bartorch_linop_codomain(const bartorch_linop* h, int N, int64_t* dims); BARTORCH_API int bartorch_linop_forward(const bartorch_linop* h, void* dst, const void* src); BARTORCH_API int bartorch_linop_adjoint(const bartorch_linop* h, void* dst, const void* src); BARTORCH_API int bartorch_linop_normal(const bartorch_linop* h, void* dst, const void* src); @@ -517,7 +522,7 @@ BARTORCH_API int bartorch_maxeigen(const bartorch_linop* A, const bartorch_linop BARTORCH_API int bartorch_solve(const bartorch_linop* A, const char* algorithm, - const char* const* reg_kinds, const long* reg_xflags, const long* reg_jflags, + const char* const* reg_kinds, const uint64_t* reg_xflags, const uint64_t* reg_jflags, const float* reg_lambda, const int* reg_k, const bartorch_prox* const* reg_ops, int n_reg, float cclambda, int maxiter, float step, int eigen, int hogwild, @@ -538,7 +543,7 @@ BARTORCH_API int bartorch_solve(const bartorch_linop* A, * BART's own. Ignored otherwise. */ int llr_blk, const char* wavelet, int shift_mode, const float* alpha, const float* gamma, - void* x, const void* y, long* iterations); + void* x, const void* y, int64_t* iterations); BARTORCH_API const char* bartorch_solve_error(int code); /* @@ -551,7 +556,7 @@ BARTORCH_API const char* bartorch_solve_error(int code); * sorts what it is given. `p` is a percentile in (0, 1], or negative for the * rule `pics` uses. */ -BARTORCH_API float bartorch_scaling_norm(long size, const void* image, float rescale, +BARTORCH_API float bartorch_scaling_norm(int64_t size, const void* image, float rescale, int compat, float p); /* @@ -572,9 +577,9 @@ BARTORCH_API float bartorch_scaling_norm(long size, const void* image, float res * generator of its own, which `bartorch_solve` rewinds before each solve, so a * term that is reused is a term freshly built as far as the answer goes. */ -BARTORCH_API int bartorch_prox_create(const char* kind, long xflags, long jflags, +BARTORCH_API int bartorch_prox_create(const char* kind, uint64_t xflags, uint64_t jflags, float lambda, int k, int llr_blk, const char* wavelet, int shift_mode, - const long* img_dims, bartorch_prox** out); + const int64_t* img_dims, bartorch_prox** out); /* A set of terms configured together, as `bartorch_solve` configures a set * holding a term that extends the optimisation variable. * @@ -586,13 +591,13 @@ BARTORCH_API int bartorch_prox_create(const char* kind, long xflags, long jflags * `bartorch_prox_free`. `alpha` and `gamma` are the set's pairs, or NULL. */ BARTORCH_API int bartorch_prox_set_create(int n, const char* const* kinds, - const long* xflags, const long* jflags, const float* lambda, const int* k, + const uint64_t* xflags, const uint64_t* jflags, const float* lambda, const int* k, int llr_blk, const char* wavelet, int shift_mode, - const float* alpha, const float* gamma, const long* img_dims, - int max_out, bartorch_prox** out, int* count, long* svars); + const float* alpha, const float* gamma, const int64_t* img_dims, + int max_out, bartorch_prox** out, int* count, int64_t* svars); /* The shape a term's proximal operator works on -- the image's, or the * codomain of the transform the term applies first. Returns the rank. */ -BARTORCH_API int bartorch_prox_domain(const bartorch_prox* h, int N, long* dims); +BARTORCH_API int bartorch_prox_domain(const bartorch_prox* h, int N, int64_t* dims); /* prox_{gamma f}(src) into dst, over that shape. */ BARTORCH_API int bartorch_prox_apply(const bartorch_prox* h, float gamma, void* dst, const void* src); /* The transform applied in place, for the one whose rank an operator here @@ -615,7 +620,7 @@ BARTORCH_API int bartorch_prox_transform_is_identity(const bartorch_prox* h); BARTORCH_API int bartorch_prox_rewind(const bartorch_prox* h); BARTORCH_API void bartorch_prox_free(bartorch_prox* h); -BARTORCH_API bartorch_nlop* bartorch_nlop_callback(int ON, const long* odims, int IN, const long* idims, +BARTORCH_API bartorch_nlop* bartorch_nlop_callback(int ON, const int64_t* odims, int IN, const int64_t* idims, bartorch_apply_fn forward, bartorch_apply_fn derivative, bartorch_apply_fn adjoint, void* ctx, bartorch_release_fn release); @@ -630,16 +635,16 @@ BARTORCH_API bartorch_nlop* bartorch_nlop_callback(int ON, const long* odims, in * What this is for is a Python function with more than one argument standing * inside a BART graph: a denoiser whose weights are an *input* rather than * something it closed over, so that a gradient reaches them. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const long* odims, - int II, int IN, const long* idims, +BARTORCH_API bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const int64_t* odims, + int II, int IN, const int64_t* idims, bartorch_generic_apply_fn forward, bartorch_pair_apply_fn derivative, bartorch_pair_apply_fn adjoint, void* ctx, bartorch_release_fn release); BARTORCH_API bartorch_nlop* bartorch_nlop_from_linop(const bartorch_linop* lin); BARTORCH_API bartorch_nlop* bartorch_nlop_chain(const bartorch_nlop* a, const bartorch_nlop* b); -BARTORCH_API int bartorch_nlop_domain(const bartorch_nlop* h, int N, long* dims); -BARTORCH_API int bartorch_nlop_codomain(const bartorch_nlop* h, int N, long* dims); +BARTORCH_API int bartorch_nlop_domain(const bartorch_nlop* h, int N, int64_t* dims); +BARTORCH_API int bartorch_nlop_codomain(const bartorch_nlop* h, int N, int64_t* dims); BARTORCH_API int bartorch_nlop_apply(const bartorch_nlop* h, void* dst, const void* src); BARTORCH_API int bartorch_nlop_derivative(const bartorch_nlop* h, void* dst, const void* src); BARTORCH_API int bartorch_nlop_adjoint(const bartorch_nlop* h, void* dst, const void* src); @@ -653,8 +658,8 @@ BARTORCH_API int bartorch_nlop_adjoint(const bartorch_nlop* h, void* dst, const */ BARTORCH_API int bartorch_nlop_inputs(const bartorch_nlop* h); BARTORCH_API int bartorch_nlop_outputs(const bartorch_nlop* h); -BARTORCH_API int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, long* dims); -BARTORCH_API int bartorch_nlop_output_codomain(const bartorch_nlop* h, int o, int N, long* dims); +BARTORCH_API int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, int64_t* dims); +BARTORCH_API int bartorch_nlop_output_codomain(const bartorch_nlop* h, int o, int N, int64_t* dims); /* Apply an operator of any arity; `args` is outputs then inputs. Fixes the * point every derivative is taken at, as the one-argument apply does. */ BARTORCH_API int bartorch_nlop_apply_generic(const bartorch_nlop* h, int nargs, void** args); @@ -676,8 +681,8 @@ BARTORCH_API bartorch_linop* bartorch_nlop_derivative_linop(const bartorch_nlop* * the callbacks above is built at DIMS. So two arguments of the same shape * can still refuse to meet. Padding a shape with ones is not a change to it, * and this is how that is said. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_reshape_in(const bartorch_nlop* a, int i, int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_reshape_out(const bartorch_nlop* a, int o, int N, const long* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_reshape_in(const bartorch_nlop* a, int i, int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_reshape_out(const bartorch_nlop* a, int o, int N, const int64_t* dims); BARTORCH_API bartorch_nlop* bartorch_nlop_chain2(const bartorch_nlop* a, int o, const bartorch_nlop* b, int i); BARTORCH_API bartorch_nlop* bartorch_nlop_combine(const bartorch_nlop* a, const bartorch_nlop* b); @@ -706,22 +711,22 @@ BARTORCH_API bartorch_nlop* bartorch_nlop_flatten(const bartorch_nlop* x, int in * constructor cannot be called at all. The same operator is built out of * the two pieces on the Python side, where the indices are right. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_tenmul(int N, const long* odims, const long* idims1, const long* idims2); -BARTORCH_API bartorch_nlop* bartorch_nlop_zdiv(int N, const long* dims, float eps); -BARTORCH_API bartorch_nlop* bartorch_nlop_zaxpbz(int N, const long* dims, float a, float b); -BARTORCH_API bartorch_nlop* bartorch_nlop_zexp(int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_zlog(int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_zinv(int N, const long* dims, float eps); -BARTORCH_API bartorch_nlop* bartorch_nlop_zsqrt(int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_zspow(int N, const long* dims, float re, float im); -BARTORCH_API bartorch_nlop* bartorch_nlop_zsadd(int N, const long* dims, float re, float im); -BARTORCH_API bartorch_nlop* bartorch_nlop_zabs(int N, const long* dims); -BARTORCH_API bartorch_nlop* bartorch_nlop_smo_abs(int N, const long* dims, float eps); -BARTORCH_API bartorch_nlop* bartorch_nlop_zrss(int N, const long* dims, unsigned long flags, float eps); -BARTORCH_API bartorch_nlop* bartorch_nlop_zss(int N, const long* dims, unsigned long flags); +BARTORCH_API bartorch_nlop* bartorch_nlop_tenmul(int N, const int64_t* odims, const int64_t* idims1, const int64_t* idims2); +BARTORCH_API bartorch_nlop* bartorch_nlop_zdiv(int N, const int64_t* dims, float eps); +BARTORCH_API bartorch_nlop* bartorch_nlop_zaxpbz(int N, const int64_t* dims, float a, float b); +BARTORCH_API bartorch_nlop* bartorch_nlop_zexp(int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_zlog(int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_zinv(int N, const int64_t* dims, float eps); +BARTORCH_API bartorch_nlop* bartorch_nlop_zsqrt(int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_zspow(int N, const int64_t* dims, float re, float im); +BARTORCH_API bartorch_nlop* bartorch_nlop_zsadd(int N, const int64_t* dims, float re, float im); +BARTORCH_API bartorch_nlop* bartorch_nlop_zabs(int N, const int64_t* dims); +BARTORCH_API bartorch_nlop* bartorch_nlop_smo_abs(int N, const int64_t* dims, float eps); +BARTORCH_API bartorch_nlop* bartorch_nlop_zrss(int N, const int64_t* dims, uint64_t flags, float eps); +BARTORCH_API bartorch_nlop* bartorch_nlop_zss(int N, const int64_t* dims, uint64_t flags); /* An operator of no inputs, and pinning one input of an operator to a value. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_const(int N, const long* dims, const void* val); -BARTORCH_API bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int N, const long* dims, const void* val); +BARTORCH_API bartorch_nlop* bartorch_nlop_const(int N, const int64_t* dims, const void* val); +BARTORCH_API bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int N, const int64_t* dims, const void* val); BARTORCH_API void bartorch_nlop_free(bartorch_nlop* h); @@ -738,7 +743,7 @@ BARTORCH_API void bartorch_nlop_free(bartorch_nlop* h); * another; with more than one, the conjugate gradients keep their step lengths * and stopping test per item. */ -BARTORCH_API bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* normal, int maxiter, float tol, float l2lambda, long batch); +BARTORCH_API bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* normal, int maxiter, float tol, float l2lambda, int64_t batch); /* The nonlinear SENSE model `nlinv` inverts, from `noir/model2.c`. * @@ -762,15 +767,15 @@ BARTORCH_API bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* n * input, which the arity queries above report. */ BARTORCH_API bartorch_noir* bartorch_noir_create(int N, - const long* ksp_dims, const long* cim_dims, const long* img_dims, - const long* kco_dims, const long* col_dims, - const long* pat_dims, const void* pattern, - const long* trj_dims, const void* traj, - const long* wgh_dims, const void* weights, - const long* bas_dims, const void* basis, - const long* msk_dims, const void* mask, + const int64_t* ksp_dims, const int64_t* cim_dims, const int64_t* img_dims, + const int64_t* kco_dims, const int64_t* col_dims, + const int64_t* pat_dims, const void* pattern, + const int64_t* trj_dims, const void* traj, + const int64_t* wgh_dims, const void* weights, + const int64_t* bas_dims, const void* basis, + const int64_t* msk_dims, const void* mask, int noncart, int optimized, int toeplitz, - unsigned long fft_flags, unsigned long wght_flags, + uint64_t fft_flags, uint64_t wght_flags, int rvc, int sos, float a, float b, float c, float oversampling_coils, int ret_os_coils); BARTORCH_API bartorch_nlop* bartorch_noir_model(const bartorch_noir* h); @@ -778,7 +783,7 @@ BARTORCH_API bartorch_linop* bartorch_noir_coils(const bartorch_noir* h); BARTORCH_API bartorch_linop* bartorch_noir_image(const bartorch_noir* h); BARTORCH_API bartorch_linop* bartorch_noir_data(const bartorch_noir* h); BARTORCH_API bartorch_linop* bartorch_noir_transform(const bartorch_noir* h); -BARTORCH_API int bartorch_noir_dims(const bartorch_noir* h, int which, int N, long* dims); +BARTORCH_API int bartorch_noir_dims(const bartorch_noir* h, int which, int N, int64_t* dims); BARTORCH_API void bartorch_noir_free(bartorch_noir* h); /* The same model, built for a network rather than for a solve: `noir2_net_s` @@ -854,7 +859,7 @@ BARTORCH_API int bartorch_cuda_use_memcache(int enable); BARTORCH_API void bartorch_cuda_memcache_clear_all(void); BARTORCH_API int bartorch_cuda_wait_for_stream(void* stream); BARTORCH_API int bartorch_cuda_signal_stream(void* stream); -BARTORCH_API long bartorch_cuda_free_memory(void); +BARTORCH_API int64_t bartorch_cuda_free_memory(void); /* * FINUFFT under BART's own NUFFT operator, which is what makes nufft, pics, @@ -888,7 +893,7 @@ BARTORCH_API int bartorch_finufft_usable_on(int device); BARTORCH_API int bartorch_finufft_usable(void); /* Plans made and not yet destroyed: zero once every operator, point spread * function and mask that asked for one has been freed. */ -BARTORCH_API long bartorch_finufft_live_plans(void); +BARTORCH_API int64_t bartorch_finufft_live_plans(void); BARTORCH_API const char* bartorch_last_error(void); BARTORCH_API void bartorch_clear_error(void); BARTORCH_API int bartorch_nufft_decline_reason(void); @@ -896,7 +901,7 @@ BARTORCH_API const char* bartorch_nufft_decline_text(void); BARTORCH_API void bartorch_nufft_allow_fallback(int enable); BARTORCH_API int bartorch_nufft_fallback_allowed(void); /* Operators built since the last reset: 0 by FINUFFT, 1 by BART. */ -BARTORCH_API long bartorch_nufft_counter(int which); +BARTORCH_API int64_t bartorch_nufft_counter(int which); BARTORCH_API void bartorch_nufft_reset_counters(void); /* @@ -909,7 +914,7 @@ BARTORCH_API void bartorch_nufft_reset_counters(void); * Normal operators since the last reset: 0 answered by a point spread * function, 1 by the transform pair. */ -BARTORCH_API long bartorch_toeplitz_counter(int which); +BARTORCH_API int64_t bartorch_toeplitz_counter(int which); BARTORCH_API void bartorch_toeplitz_reset_counters(void); /* Whether a pointer is device memory; always false without CUDA. */ diff --git a/src/csrc/ops/coset.cuh b/src/csrc/ops/coset.cuh index 3495b0a1..2913f7d5 100644 --- a/src/csrc/ops/coset.cuh +++ b/src/csrc/ops/coset.cuh @@ -19,6 +19,7 @@ #include #include #include +#include #ifndef M_PI #define M_PI 3.14159265358979323846 @@ -26,15 +27,15 @@ struct phase_conf { - long dims[3]; - long tot; - long batch; + int64_t dims[3]; + int64_t tot; + int64_t batch; float shifts[3]; float cn; float scale; }; -static inline struct phase_conf phase_setup(int N, const long dims[], const float shift[3], float scale) +static inline struct phase_conf phase_setup(int N, const int64_t dims[], const float shift[3], float scale) { struct phase_conf c; @@ -55,7 +56,7 @@ static inline struct phase_conf phase_setup(int N, const long dims[], const floa c.dims[n] = dims[n]; c.tot *= dims[n]; - long centre = dims[n] / 2; + int64_t centre = dims[n] / 2; double half = (double)centre / (double)dims[n]; c.shifts[n] += 2. * M_PI * half; @@ -70,7 +71,7 @@ static inline struct phase_conf phase_setup(int N, const long dims[], const floa return c; } -__device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, long x, long y, long z, bool conj) +__device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, int64_t x, int64_t y, int64_t z, bool conj) { float val = c.cn + x * c.shifts[0] + y * c.shifts[1] + z * c.shifts[2]; @@ -85,7 +86,7 @@ __device__ static inline cuFloatComplex phase_at(const struct phase_conf& c, lon } /* Where grid point `i` sits in the gathered spectrum, or -1 where it is not kept. */ -__device__ static inline long kept_at(const unsigned int* mask, const int* prefix, long i) +__device__ static inline int64_t kept_at(const unsigned int* mask, const int* prefix, int64_t i) { unsigned int word = mask[i >> 5]; unsigned int bit = 1u << (i & 31); @@ -93,7 +94,7 @@ __device__ static inline long kept_at(const unsigned int* mask, const int* prefi if (0 == (word & bit)) return -1; - return (long)prefix[i >> 5] + __popc(word & (bit - 1)); + return (int64_t)prefix[i >> 5] + __popc(word & (bit - 1)); } /* What the callbacks of one transform read: a set's phase, a coil's diff --git a/src/csrc/ops/fft_callbacks.cu b/src/csrc/ops/fft_callbacks.cu index 6c07b170..172b6eda 100644 --- a/src/csrc/ops/fft_callbacks.cu +++ b/src/csrc/ops/fft_callbacks.cu @@ -83,7 +83,7 @@ extern "C" void bartorch_cb_fft_free(struct bartorch_cb_fft* p) /* The pair for a volume of `dims`, or NULL where cuFFT cannot link the * callbacks in. */ -extern "C" struct bartorch_cb_fft* bartorch_cb_fft_create(const long dims[3]) +extern "C" struct bartorch_cb_fft* bartorch_cb_fft_create(const bart_dim_t dims[3]) { set_jit_t set_jit = set_jit_callback(); @@ -105,7 +105,7 @@ extern "C" struct bartorch_cb_fft* bartorch_cb_fft_create(const long dims[3]) return NULL; /* The callbacks index the volume in 32 bits. */ - if (dims[0] * dims[1] * dims[2] > (long)UINT32_MAX) + if (dims[0] * dims[1] * dims[2] > (bart_dim_t)UINT32_MAX) return NULL; struct bartorch_cb_fft* p = (struct bartorch_cb_fft*)xmalloc(sizeof *p); @@ -190,7 +190,7 @@ static void run(struct bartorch_cb_fft* p, int which, const struct coset_info* i CUDA_KERNEL_ERROR; - void* work = (0 < p->work) ? cuda_malloc((long)p->work) : NULL; + void* work = (0 < p->work) ? cuda_malloc((bart_dim_t)p->work) : NULL; cufftHandle plan = p->plan[which]; @@ -205,7 +205,7 @@ static void run(struct bartorch_cb_fft* p, int which, const struct coset_info* i cuda_free(work); } -static struct coset_info info_for(int N, const long dims[], const float shift[3], float scale, +static struct coset_info info_for(int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, const _Complex float* map) { struct coset_info c; @@ -223,7 +223,7 @@ static struct coset_info info_for(int N, const long dims[], const float shift[3] /* The coefficient `src`, multiplied by the set's phase and the coil's * sensitivity, transformed, and gathered into `bank`. `volume` is where the * transform works; `src` is left as it is. */ -extern "C" void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const long dims[], +extern "C" void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, _Complex float* bank, _Complex float* volume, const _Complex float* src, const _Complex float* map) { @@ -236,7 +236,7 @@ extern "C" void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const /* The gathered spectrum `bank`, scattered, transformed back, multiplied by the * conjugates of the phase and the sensitivity, and added to `dst`. */ -extern "C" void bartorch_cb_fft_inverse(struct bartorch_cb_fft* p, int N, const long dims[], +extern "C" void bartorch_cb_fft_inverse(struct bartorch_cb_fft* p, int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, _Complex float* dst, _Complex float* volume, const _Complex float* bank, const _Complex float* map) { @@ -291,7 +291,7 @@ extern "C" void bartorch_cb_grid_free(struct bartorch_cb_grid* p) * and stay the caller's. `unitary` scales each direction by one over the * square root of the plane; without it the forward is cuFFT's unnormalized * transform and the inverse its adjoint, as BART's uncentred `fft` is. */ -extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], unsigned long flags, long kept, +extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const bart_dim_t dims[3], bart_flags_t flags, bart_dim_t kept, const unsigned int* mask, const int* prefix, const _Complex float* mod[3], int unitary) { set_jit_t set_jit = set_jit_callback(); @@ -299,7 +299,7 @@ extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], if (NULL == set_jit) return NULL; - if (dims[0] * dims[1] * dims[2] > (long)UINT32_MAX) + if (dims[0] * dims[1] * dims[2] > (bart_dim_t)UINT32_MAX) return NULL; struct grid_info g; @@ -309,7 +309,7 @@ extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], int rank = 0; for (int i = 2; i >= 0; i--) - if ((0 != ((flags >> i) & 1UL)) && (1 < dims[i])) + if ((0 != ((flags >> i) & 1)) && (1 < dims[i])) n[rank++] = (int)dims[i]; if (0 == rank) @@ -321,7 +321,7 @@ extern "C" struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], for (int a = 0; a < 3; a++) { - bool t = (0 != ((flags >> a) & 1UL)) && (1 < dims[a]); + bool t = (0 != ((flags >> a) & 1)) && (1 < dims[a]); g.n[a] = (unsigned int)dims[a]; g.mstr[a] = ms; @@ -421,7 +421,7 @@ static void run_grid(struct bartorch_cb_grid* p, int which, const struct grid_in CUDA_KERNEL_ERROR; - void* work = (0 < p->work) ? cuda_malloc((long)p->work) : NULL; + void* work = (0 < p->work) ? cuda_malloc((bart_dim_t)p->work) : NULL; cufftHandle plan = p->plan[which]; diff --git a/src/csrc/ops/fft_callbacks_lto.cu b/src/csrc/ops/fft_callbacks_lto.cu index d33a54f5..772d3c2f 100644 --- a/src/csrc/ops/fft_callbacks_lto.cu +++ b/src/csrc/ops/fft_callbacks_lto.cu @@ -26,7 +26,7 @@ __device__ static inline cuFloatComplex phase_of(const struct coset_info* c, uns unsigned int ny = (unsigned int)c->phase.dims[1]; unsigned int yz = i / nx; - return phase_at(c->phase, (long)(i - yz * nx), (long)(yz % ny), (long)(yz / ny), conj); + return phase_at(c->phase, (int64_t)(i - yz * nx), (int64_t)(yz % ny), (int64_t)(yz / ny), conj); } __device__ cufftComplex bartorch_load_in(void* in, unsigned long long off, void* info, void* shared) @@ -45,7 +45,7 @@ __device__ void bartorch_store_gather(void* out, unsigned long long off, cufftCo { const struct coset_info* c = (const struct coset_info*)info; - long j = kept_at(c->mask, c->prefix, (long)off); + int64_t j = kept_at(c->mask, c->prefix, (int64_t)off); if (0 <= j) c->bank[j] = val; @@ -55,7 +55,7 @@ __device__ cufftComplex bartorch_load_scatter(void* in, unsigned long long off, { const struct coset_info* c = (const struct coset_info*)info; - long j = kept_at(c->mask, c->prefix, (long)off); + int64_t j = kept_at(c->mask, c->prefix, (int64_t)off); return (0 <= j) ? c->bank[j] : make_cuFloatComplex(0.f, 0.f); } @@ -102,7 +102,7 @@ __device__ void bartorch_grid_store_gather(void* out, unsigned long long off, cu { const struct grid_info* g = (const struct grid_info*)info; - long j = grid_kept(g, (unsigned int)off); + int64_t j = grid_kept(g, (unsigned int)off); if (0 <= j) g->bank[j] = val; @@ -112,7 +112,7 @@ __device__ cufftComplex bartorch_grid_load_scatter(void* in, unsigned long long { const struct grid_info* g = (const struct grid_info*)info; - long j = grid_kept(g, (unsigned int)off); + int64_t j = grid_kept(g, (unsigned int)off); return (0 <= j) ? g->bank[j] : make_cuFloatComplex(0.f, 0.f); } diff --git a/src/csrc/ops/grid.c b/src/csrc/ops/grid.c index 6299d43a..1dfded18 100644 --- a/src/csrc/ops/grid.c +++ b/src/csrc/ops/grid.c @@ -59,18 +59,18 @@ #ifdef USE_CUDA struct bartorch_cb_grid; -extern struct bartorch_cb_grid* bartorch_cb_grid_create(const long dims[3], unsigned long flags, long kept, +extern struct bartorch_cb_grid* bartorch_cb_grid_create(const bart_dim_t dims[3], bart_flags_t flags, bart_dim_t kept, const unsigned int* mask, const int* prefix, const complex float* mod[3], int unitary); -extern void bartorch_cuda_pad_map(long sx, long wx, long rest, long off, +extern void bartorch_cuda_pad_map(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, complex float* dst, const complex float* src, const complex float* map); -extern void bartorch_cuda_crop_mapc_add(long sx, long wx, long rest, long off, +extern void bartorch_cuda_crop_mapc_add(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, complex float* dst, const complex float* src, const complex float* map); extern void bartorch_cb_grid_free(struct bartorch_cb_grid* p); extern void bartorch_cb_grid_forward(struct bartorch_cb_grid* p, complex float* bank, complex float* volume, const complex float* src, const complex float* map); extern void bartorch_cb_grid_inverse(struct bartorch_cb_grid* p, complex float* dst, complex float* volume, const complex float* bank, const complex float* map); -extern int bartorch_cuda_contract_grid(long L, long B, int R, complex float* bank, const complex float* K); +extern int bartorch_cuda_contract_grid(bart_dim_t L, bart_dim_t B, int R, complex float* bank, const complex float* K); /* What the kernels between the gathered spectrum and a table of samples read * of the transform's layout (kernels.cu). */ @@ -81,17 +81,17 @@ struct bartorch_grid_axes { const complex float* mod[3]; }; -extern void bartorch_cuda_bank_to_table(long E, long X, long S, int R, long L, long per, +extern void bartorch_cuda_bank_to_table(bart_dim_t E, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* entry_u, const int* u_coord, const unsigned int* mask, const int* prefix, const struct bartorch_grid_axes* ax, const complex float* B, const complex float* bank, complex float* table); -extern void bartorch_cuda_table_to_bank(long U, long X, long S, int R, long L, long per, +extern void bartorch_cuda_table_to_bank(bart_dim_t U, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* csr_start, const int* csr, const int* u_coord, const unsigned int* mask, const int* prefix, const struct bartorch_grid_axes* ax, const complex float* B, complex float* bank, const complex float* table); #endif -static long grid_fused_count = 0; +static bart_dim_t grid_fused_count = 0; -long bartorch_grid_fused(void) +int64_t bartorch_grid_fused(void) { return grid_fused_count; } @@ -100,8 +100,8 @@ struct grid_s { linop_data_t super; - long cim_dims[DIMS]; - long out_dims[DIMS]; + bart_dim_t cim_dims[DIMS]; + bart_dim_t out_dims[DIMS]; const struct linop_s* fwd; /* transform, basis, pattern */ const struct linop_s* nrm; /* the normal as BART's chain */ @@ -109,13 +109,13 @@ struct grid_s { /* The normal through cuFFT's callbacks: the axes it transforms, the * places of a plane the pattern keeps, and the kernel at each of them * (NULL where it is one), on the host. */ - unsigned long flags; - long R; - long vol; - long plane; - long batch; - long kept; - long words; + bart_flags_t flags; + bart_dim_t R; + bart_dim_t vol; + bart_dim_t plane; + bart_dim_t batch; + bart_dim_t kept; + bart_dim_t words; unsigned int* mask; int* prefix; complex float* kernel; @@ -143,9 +143,9 @@ struct grid_s { * by `psf` (over the spatial axes of `cim_dims`). Without a wave * `dom_dims` is `cim_dims` and `img_vol` is `vol`. */ bool wave; - long dom_dims[DIMS]; - long img_vol; - long wave_off; + bart_dim_t dom_dims[DIMS]; + bart_dim_t img_vol; + bart_stride_t wave_off; complex float* psf; complex float* dpsf; @@ -157,11 +157,11 @@ struct grid_s { * `kspace_readout` says the samples are in k-space along the readout. */ bool sampled; bool kspace_readout; - long T; - long S; - long X; - long E; - long U; + bart_dim_t T; + bart_dim_t S; + bart_dim_t X; + bart_dim_t E; + bart_dim_t U; int* entry_u; int* u_coord; int* csr_start; @@ -177,9 +177,9 @@ struct grid_s { static DEF_TYPEID(grid_s); /* The spatial axes along which the pattern has more than one value. */ -static unsigned long varying(const long pat_dims[DIMS]) +static bart_flags_t varying(const bart_dim_t pat_dims[DIMS]) { - unsigned long flags = 0UL; + bart_flags_t flags = 0; for (int a = 0; a < 3; a++) if (1 < pat_dims[a]) @@ -192,12 +192,12 @@ static unsigned long varying(const long pat_dims[DIMS]) * arrangement a single multiply-accumulate contracts -- and the pattern's * spatial axes, and one everywhere else. Computed on the host: it is * coefficients squared over a plane of phase encodes. */ -static complex float* grid_kernel(long kdims[DIMS], const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis) +static complex float* grid_kernel(bart_dim_t kdims[DIMS], const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis) { - long R = (NULL == basis) ? 1 : bas_dims[COEFF_DIM]; - long T = (NULL == basis) ? 1 : bas_dims[TE_DIM]; - long F = (NULL == pattern) ? 1 : pat_dims[TE_DIM]; + bart_dim_t R = (NULL == basis) ? 1 : bas_dims[COEFF_DIM]; + bart_dim_t T = (NULL == basis) ? 1 : bas_dims[TE_DIM]; + bart_dim_t F = (NULL == pattern) ? 1 : pat_dims[TE_DIM]; md_singleton_dims(DIMS, kdims); @@ -208,7 +208,7 @@ static complex float* grid_kernel(long kdims[DIMS], const long pat_dims[DIMS], c kdims[TE_DIM] = R; kdims[COEFF_DIM] = R; - long places = md_calc_size(3, kdims); + bart_dim_t places = md_calc_size(3, kdims); complex float* P = NULL; @@ -229,22 +229,22 @@ static complex float* grid_kernel(long kdims[DIMS], const long pat_dims[DIMS], c /* conj(B[k', t]) B[k, t], once per frame. */ complex float* G = xmalloc((size_t)(T * R * R) * sizeof(complex float)); - for (long t = 0; t < T; t++) - for (long kp = 0; kp < R; kp++) - for (long k = 0; k < R; k++) + for (bart_dim_t t = 0; t < T; t++) + for (bart_dim_t kp = 0; kp < R; kp++) + for (bart_dim_t k = 0; k < R; k++) G[t * R * R + kp * R + k] = (NULL == B) ? 1.f : conjf(B[t + T * kp]) * B[t + T * k]; complex float* K = md_alloc(DIMS, kdims, CFL_SIZE); #pragma omp parallel for - for (long p = 0; p < places; p++) { + for (bart_dim_t p = 0; p < places; p++) { - for (long kp = 0; kp < R; kp++) - for (long k = 0; k < R; k++) { + for (bart_dim_t kp = 0; kp < R; kp++) + for (bart_dim_t k = 0; k < R; k++) { complex double acc = 0.; - for (long t = 0; t < T; t++) { + for (bart_dim_t t = 0; t < T; t++) { double w = 1.; @@ -274,12 +274,12 @@ static complex float* grid_kernel(long kdims[DIMS], const long pat_dims[DIMS], c * samples, as a bit per place and a count before each word, and the kernel at * those places alone, `kernel[(l R + r) R + c]` -- or no kernel, where every * kept place's is one. */ -static void grid_compress(struct grid_s* d, const long kdims[DIMS], const complex float* K, - const long pat_dims[DIMS], const complex float* pattern) +static void grid_compress(struct grid_s* d, const bart_dim_t kdims[DIMS], const complex float* K, + const bart_dim_t pat_dims[DIMS], const complex float* pattern) { - long places = md_calc_size(3, kdims); - long R = d->R; - long F = pat_dims[TE_DIM]; + bart_dim_t places = md_calc_size(3, kdims); + bart_dim_t R = d->R; + bart_dim_t F = pat_dims[TE_DIM]; complex float* P = md_alloc(DIMS, pat_dims, CFL_SIZE); md_copy(DIMS, pat_dims, P, pattern, CFL_SIZE); @@ -288,17 +288,17 @@ static void grid_compress(struct grid_s* d, const long kdims[DIMS], const comple d->mask = xmalloc((size_t)d->words * sizeof(unsigned int)); d->prefix = xmalloc((size_t)d->words * sizeof(int)); - long n = 0; + bart_dim_t n = 0; - for (long w = 0; w < d->words; w++) { + for (bart_dim_t w = 0; w < d->words; w++) { unsigned int bits = 0; - for (long b = 0; (b < 32) && (w * 32 + b < places); b++) { + for (bart_dim_t b = 0; (b < 32) && (w * 32 + b < places); b++) { bool taken = false; - for (long t = 0; (t < F) && !taken; t++) + for (bart_dim_t t = 0; (t < F) && !taken; t++) taken = (0. != cabsf(P[w * 32 + b + places * t])); if (taken) @@ -318,15 +318,15 @@ static void grid_compress(struct grid_s* d, const long kdims[DIMS], const comple complex float* kernel = xmalloc((size_t)MAX(1, n * R * R) * sizeof(complex float)); bool ones = (1 == R); - long l = 0; + bart_dim_t l = 0; - for (long p = 0; p < places; p++) { + for (bart_dim_t p = 0; p < places; p++) { if (0 == (d->mask[p >> 5] & (1u << (p & 31)))) continue; - for (long r = 0; r < R; r++) - for (long c = 0; c < R; c++) { + for (bart_dim_t r = 0; r < R; r++) + for (bart_dim_t c = 0; c < R; c++) { complex float v = K[p + places * (r + R * c)]; @@ -354,13 +354,13 @@ static void grid_compress(struct grid_s* d, const long kdims[DIMS], const comple if (!MD_IS_SET(d->flags, a)) continue; - long ad[1] = { d->cim_dims[a] }; + bart_dim_t ad[1] = { d->cim_dims[a] }; d->mod[a] = md_alloc(1, ad, CFL_SIZE); md_zfill(1, ad, d->mod[a], 1.); if (d->centred) - fftmod(1, ad, 1UL, d->mod[a], d->mod[a]); + fftmod(1, ad, 1, d->mod[a], d->mod[a]); } } @@ -368,7 +368,7 @@ static void grid_compress(struct grid_s* d, const long kdims[DIMS], const comple * on, making what it needs there the first time it is asked. */ static bool grid_ready(struct grid_s* d, const void* ref) { - if (0UL == d->flags) + if (0 == d->flags) return false; #ifdef USE_CUDA @@ -379,7 +379,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) d->tried = true; - long wd[1] = { d->words }; + bart_dim_t wd[1] = { d->words }; d->dmask = md_alloc_gpu(1, wd, sizeof(unsigned int)); d->dprefix = md_alloc_gpu(1, wd, sizeof(int)); @@ -388,7 +388,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (NULL != d->kernel) { - long kd[1] = { d->kept * d->R * d->R }; + bart_dim_t kd[1] = { d->kept * d->R * d->R }; d->dkernel = md_alloc_gpu(1, kd, CFL_SIZE); md_copy(1, kd, d->dkernel, d->kernel, CFL_SIZE); @@ -399,7 +399,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (NULL == d->mod[a]) continue; - long ad[1] = { d->cim_dims[a] }; + bart_dim_t ad[1] = { d->cim_dims[a] }; d->dmod[a] = md_alloc_gpu(1, ad, CFL_SIZE); md_copy(1, ad, d->dmod[a], d->mod[a], CFL_SIZE); @@ -407,7 +407,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (NULL != d->psf) { - long pd[1] = { d->vol }; + bart_dim_t pd[1] = { d->vol }; d->dpsf = md_alloc_gpu(1, pd, CFL_SIZE); md_copy(1, pd, d->dpsf, d->psf, CFL_SIZE); @@ -415,10 +415,10 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (d->sampled) { - long ed[1] = { MAX(1, d->E) }; - long ud[1] = { MAX(1, 2 * d->U) }; - long sd[1] = { d->U + 1 }; - long cd[1] = { MAX(1, d->csr_start[d->U]) }; + bart_dim_t ed[1] = { MAX(1, d->E) }; + bart_dim_t ud[1] = { MAX(1, 2 * d->U) }; + bart_dim_t sd[1] = { d->U + 1 }; + bart_dim_t cd[1] = { MAX(1, d->csr_start[d->U]) }; d->d_entry_u = md_alloc_gpu(1, ed, sizeof(int)); d->d_u_coord = md_alloc_gpu(1, ud, sizeof(int)); @@ -431,7 +431,7 @@ static bool grid_ready(struct grid_s* d, const void* ref) if (NULL != d->bt) { - long bd[1] = { d->T * d->R }; + bart_dim_t bd[1] = { d->T * d->R }; d->d_bt = md_alloc_gpu(1, bd, CFL_SIZE); md_copy(1, bd, d->d_bt, d->bt, CFL_SIZE); @@ -454,9 +454,9 @@ static bool grid_ready(struct grid_s* d, const void* ref) /* BART's transform along `flags` in place, in the grid's convention, or its * adjoint: fftuc and ifftuc centred, fft and ifft otherwise. */ -static void grid_ft(const struct grid_s* d, const long dims[DIMS], unsigned long flags, complex float* x, bool adjoint) +static void grid_ft(const struct grid_s* d, const bart_dim_t dims[DIMS], bart_flags_t flags, complex float* x, bool adjoint) { - if (0UL == flags) + if (0 == flags) return; if (d->centred) @@ -478,9 +478,9 @@ static void grid_in(struct grid_s* d, complex float* bank, complex float* volume return; } - long wx = d->cim_dims[READ_DIM]; + bart_dim_t wx = d->cim_dims[READ_DIM]; - long hdims[DIMS]; + bart_dim_t hdims[DIMS]; md_select_dims(DIMS, FFT_FLAGS, hdims, d->cim_dims); bartorch_cuda_pad_map(d->dom_dims[READ_DIM], wx, d->vol / wx, d->wave_off, hybrid, src, map); @@ -498,9 +498,9 @@ static void grid_out(struct grid_s* d, complex float* dst, complex float* volume return; } - long wx = d->cim_dims[READ_DIM]; + bart_dim_t wx = d->cim_dims[READ_DIM]; - long hdims[DIMS]; + bart_dim_t hdims[DIMS]; md_select_dims(DIMS, FFT_FLAGS, hdims, d->cim_dims); md_clear(DIMS, hdims, hybrid, CFL_SIZE); @@ -515,29 +515,29 @@ static void grid_out(struct grid_s* d, complex float* dst, complex float* volume * `coeff_step` after the one before, and a coil's sensitivity `map_step` * after the one before in `map` (or no map). */ static void grid_fused(struct grid_s* d, complex float* dst, const complex float* src, - long coils, long coil_step, long coeff_step, const complex float* map, long map_step) + bart_dim_t coils, bart_dim_t coil_step, bart_dim_t coeff_step, const complex float* map, bart_dim_t map_step) { #ifdef USE_CUDA - long vd[1] = { d->vol }; - long bd[1] = { d->R * d->batch * MAX(1, d->kept) }; + bart_dim_t vd[1] = { d->vol }; + bart_dim_t bd[1] = { d->R * d->batch * MAX(1, d->kept) }; complex float* volume = md_alloc_gpu(1, vd, CFL_SIZE); complex float* bank = md_alloc_gpu(1, bd, CFL_SIZE); complex float* hybrid = d->wave ? md_alloc_gpu(1, vd, CFL_SIZE) : NULL; - long per = d->batch * d->kept; + bart_dim_t per = d->batch * d->kept; - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_step; - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) grid_in(d, bank + r * per, volume, hybrid, src + c * coil_step + r * coeff_step, m); if (NULL != d->dkernel) bartorch_cuda_contract_grid(d->kept, d->batch, (int)d->R, bank, d->dkernel); - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) grid_out(d, dst + c * coil_step + r * coeff_step, volume, hybrid, bank + r * per, m); } @@ -559,7 +559,7 @@ static void grid_fused(struct grid_s* d, complex float* dst, const complex float * has been transformed by its front already, so the grid never transforms it * again for a table -- where cuFFT's callbacks would (a plane of one axis * takes the readout along), the table is served on the host instead. */ -static unsigned long sampled_flags(const struct grid_s* d) +static bart_flags_t sampled_flags(const struct grid_s* d) { return d->wave ? (d->flags & ~READ_FLAG) : d->flags; } @@ -574,13 +574,13 @@ static bool sampled_fusable(const struct grid_s* d) * function -- and the adjoint back. Without a wave both are copies. */ static void wave_psf_host(const struct grid_s* d, complex float* k, bool conj) { - long pdims[DIMS]; + bart_dim_t pdims[DIMS]; md_select_dims(DIMS, FFT_FLAGS, pdims, d->cim_dims); - long kstr[DIMS]; + bart_stride_t kstr[DIMS]; md_calc_strides(DIMS, kstr, d->cim_dims, CFL_SIZE); - long pstr[DIMS]; + bart_stride_t pstr[DIMS]; md_calc_strides(DIMS, pstr, pdims, CFL_SIZE); for (int i = 0; i < DIMS; i++) @@ -629,7 +629,7 @@ static void wave_back_host(const struct grid_s* d, complex float* dst, complex f * axis alone takes a second), so the table is transformed along its samples * where the two differ: into k-space forward, out of it for the adjoint, and * the other way round for samples in image space along the readout. */ -static void sampled_readout(const struct grid_s* d, const long dims[DIMS], complex float* table, bool forward) +static void sampled_readout(const struct grid_s* d, const bart_dim_t dims[DIMS], complex float* table, bool forward) { bool transformed = (0 != MD_IS_SET(sampled_flags(d), READ_DIM)); @@ -646,32 +646,32 @@ static void sampled_readout(const struct grid_s* d, const long dims[DIMS], compl * readout, contracted with the basis for its frame. */ static void sampled_gather(const struct grid_s* d, complex float* out, const complex float* k) { - long cstr[DIMS]; + bart_stride_t cstr[DIMS]; md_calc_strides(DIMS, cstr, d->cim_dims, 1); - long ostr[DIMS]; + bart_stride_t ostr[DIMS]; md_calc_strides(DIMS, ostr, d->out_dims, 1); - long C = d->cim_dims[COIL_DIM]; + bart_dim_t C = d->cim_dims[COIL_DIM]; #pragma omp parallel for collapse(2) - for (long c = 0; c < C; c++) { - for (long e = 0; e < d->E; e++) { + for (bart_dim_t c = 0; c < C; c++) { + for (bart_dim_t e = 0; e < d->E; e++) { - long t = e / d->S; - long u = d->entry_u[e]; - long base = (e - t * d->S) * ostr[PHS2_DIM] + c * ostr[COIL_DIM] + t * ostr[TE_DIM]; + bart_dim_t t = e / d->S; + bart_dim_t u = d->entry_u[e]; + bart_dim_t base = (e - t * d->S) * ostr[PHS2_DIM] + c * ostr[COIL_DIM] + t * ostr[TE_DIM]; - for (long x = 0; x < d->X; x++) { + for (bart_dim_t x = 0; x < d->X; x++) { complex float acc = 0.; if (0 <= u) { - long at = x * cstr[READ_DIM] + d->u_coord[2 * u + 1] * cstr[PHS1_DIM] + bart_dim_t at = x * cstr[READ_DIM] + d->u_coord[2 * u + 1] * cstr[PHS1_DIM] + d->u_coord[2 * u] * cstr[PHS2_DIM] + c * cstr[COIL_DIM]; - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) acc += ((NULL == d->bt) ? 1.f : d->bt[t * d->R + r]) * k[at + r * cstr[COEFF_DIM]]; } @@ -687,31 +687,31 @@ static void sampled_scatter(const struct grid_s* d, complex float* k, const comp { md_clear(DIMS, d->cim_dims, k, CFL_SIZE); - long cstr[DIMS]; + bart_stride_t cstr[DIMS]; md_calc_strides(DIMS, cstr, d->cim_dims, 1); - long ostr[DIMS]; + bart_stride_t ostr[DIMS]; md_calc_strides(DIMS, ostr, d->out_dims, 1); - long C = d->cim_dims[COIL_DIM]; + bart_dim_t C = d->cim_dims[COIL_DIM]; #pragma omp parallel for collapse(2) - for (long c = 0; c < C; c++) { - for (long u = 0; u < d->U; u++) { + for (bart_dim_t c = 0; c < C; c++) { + for (bart_dim_t u = 0; u < d->U; u++) { - for (long x = 0; x < d->X; x++) { + for (bart_dim_t x = 0; x < d->X; x++) { - long at = x * cstr[READ_DIM] + d->u_coord[2 * u + 1] * cstr[PHS1_DIM] + bart_dim_t at = x * cstr[READ_DIM] + d->u_coord[2 * u + 1] * cstr[PHS1_DIM] + d->u_coord[2 * u] * cstr[PHS2_DIM] + c * cstr[COIL_DIM]; - for (long r = 0; r < d->R; r++) { + for (bart_dim_t r = 0; r < d->R; r++) { complex float acc = 0.; - for (long q = d->csr_start[u]; q < d->csr_start[u + 1]; q++) { + for (bart_dim_t q = d->csr_start[u]; q < d->csr_start[u + 1]; q++) { - long e = d->csr[q]; - long t = e / d->S; + bart_dim_t e = d->csr[q]; + bart_dim_t t = e / d->S; complex float v = out[x * ostr[PHS1_DIM] + (e - t * d->S) * ostr[PHS2_DIM] + c * ostr[COIL_DIM] + t * ostr[TE_DIM]]; @@ -787,18 +787,18 @@ static void grid_axes(const struct grid_s* d, struct bartorch_grid_axes* ax) * centring on and whose write gathers the kept places, then the table read * off the gathered spectrum on the card. */ static void sampled_fused_forward(struct grid_s* d, complex float* dst, const complex float* src, - long coils, long coil_step, long coeff_step, const complex float* map, long map_step) + bart_dim_t coils, bart_dim_t coil_step, bart_dim_t coeff_step, const complex float* map, bart_dim_t map_step) { - long vd[1] = { d->vol }; - long bd[1] = { d->R * d->batch * MAX(1, d->kept) }; + bart_dim_t vd[1] = { d->vol }; + bart_dim_t bd[1] = { d->R * d->batch * MAX(1, d->kept) }; - long td[DIMS]; + bart_dim_t td[DIMS]; md_select_dims(DIMS, ~COIL_FLAG, td, d->out_dims); - long tstr[DIMS]; + bart_stride_t tstr[DIMS]; md_calc_strides(DIMS, tstr, td, CFL_SIZE); - long ostr[DIMS]; + bart_stride_t ostr[DIMS]; md_calc_strides(DIMS, ostr, d->out_dims, CFL_SIZE); complex float* volume = md_alloc_gpu(1, vd, CFL_SIZE); @@ -809,13 +809,13 @@ static void sampled_fused_forward(struct grid_s* d, complex float* dst, const co struct bartorch_grid_axes ax; grid_axes(d, &ax); - long per = d->batch * d->kept; + bart_dim_t per = d->batch * d->kept; - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_step; - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) grid_in(d, bank + r * per, volume, hybrid, src + c * coil_step + r * coeff_step, m); bartorch_cuda_bank_to_table(d->E, d->X, d->S, (int)d->R, d->kept, per, d->d_entry_u, d->d_u_coord, @@ -823,7 +823,7 @@ static void sampled_fused_forward(struct grid_s* d, complex float* dst, const co sampled_readout(d, td, table, true); - md_copy2(DIMS, td, ostr, dst + c * (ostr[COIL_DIM] / (long)CFL_SIZE), tstr, table, CFL_SIZE); + md_copy2(DIMS, td, ostr, dst + c * (ostr[COIL_DIM] / (bart_stride_t)CFL_SIZE), tstr, table, CFL_SIZE); } md_free(table); @@ -841,18 +841,18 @@ static void sampled_fused_forward(struct grid_s* d, complex float* dst, const co * spectrum on the card, and per coefficient one transform back whose read * scatters it and whose write takes the conjugates off into the image. */ static void sampled_fused_adjoint(struct grid_s* d, complex float* dst, const complex float* src, - long coils, long coil_step, long coeff_step, const complex float* map, long map_step) + bart_dim_t coils, bart_dim_t coil_step, bart_dim_t coeff_step, const complex float* map, bart_dim_t map_step) { - long vd[1] = { d->vol }; - long bd[1] = { d->R * d->batch * MAX(1, d->kept) }; + bart_dim_t vd[1] = { d->vol }; + bart_dim_t bd[1] = { d->R * d->batch * MAX(1, d->kept) }; - long td[DIMS]; + bart_dim_t td[DIMS]; md_select_dims(DIMS, ~COIL_FLAG, td, d->out_dims); - long tstr[DIMS]; + bart_stride_t tstr[DIMS]; md_calc_strides(DIMS, tstr, td, CFL_SIZE); - long ostr[DIMS]; + bart_stride_t ostr[DIMS]; md_calc_strides(DIMS, ostr, d->out_dims, CFL_SIZE); complex float* volume = md_alloc_gpu(1, vd, CFL_SIZE); @@ -863,13 +863,13 @@ static void sampled_fused_adjoint(struct grid_s* d, complex float* dst, const co struct bartorch_grid_axes ax; grid_axes(d, &ax); - long per = d->batch * d->kept; + bart_dim_t per = d->batch * d->kept; - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_step; - md_copy2(DIMS, td, tstr, table, ostr, src + c * (ostr[COIL_DIM] / (long)CFL_SIZE), CFL_SIZE); + md_copy2(DIMS, td, tstr, table, ostr, src + c * (ostr[COIL_DIM] / (bart_stride_t)CFL_SIZE), CFL_SIZE); sampled_readout(d, td, table, false); @@ -878,7 +878,7 @@ static void sampled_fused_adjoint(struct grid_s* d, complex float* dst, const co bartorch_cuda_table_to_bank(d->U, d->X, d->S, (int)d->R, d->kept, per, d->d_csr_start, d->d_csr, d->d_u_coord, d->dmask, d->dprefix, &ax, d->d_bt, bank, table); - for (long r = 0; r < d->R; r++) + for (bart_dim_t r = 0; r < d->R; r++) grid_out(d, dst + c * coil_step + r * coeff_step, volume, hybrid, bank + r * per, m); } @@ -903,7 +903,7 @@ static void grid_forward(const linop_data_t* _d, complex float* dst, const compl #ifdef USE_CUDA if (grid_ready(d, dst) && cuda_ondevice(src) && sampled_fusable(d)) { - long coils = d->cim_dims[COIL_DIM]; + bart_dim_t coils = d->cim_dims[COIL_DIM]; sampled_fused_forward(d, dst, src, coils, d->img_vol, d->img_vol * coils, NULL, 0); return; @@ -925,7 +925,7 @@ static void grid_adjoint(const linop_data_t* _d, complex float* dst, const compl #ifdef USE_CUDA if (grid_ready(d, dst) && cuda_ondevice(src) && sampled_fusable(d)) { - long coils = d->cim_dims[COIL_DIM]; + bart_dim_t coils = d->cim_dims[COIL_DIM]; md_clear(DIMS, d->dom_dims, dst, CFL_SIZE); sampled_fused_adjoint(d, dst, src, coils, d->img_vol, d->img_vol * coils, NULL, 0); @@ -945,7 +945,7 @@ static void grid_normal(const linop_data_t* _d, complex float* dst, const comple if (grid_ready(d, dst) && cuda_ondevice(src)) { - long coils = d->cim_dims[COIL_DIM]; + bart_dim_t coils = d->cim_dims[COIL_DIM]; md_clear(DIMS, d->dom_dims, dst, CFL_SIZE); grid_fused(d, dst, src, coils, d->img_vol, d->img_vol * coils, NULL, 0); @@ -1008,11 +1008,11 @@ int bartorch_grid_folds(const struct linop_s* op, const void* ref) /* The normal of a slab of coils, each with its sensitivity from `map`, * against the image `src`, added to the image `dst`. */ void bartorch_grid_normal_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map) + const int64_t map_strs[DIMS], const complex float* map) { struct grid_s* d = CAST_DOWN(grid_s, linop_get_data(op)); - grid_fused(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (long)CFL_SIZE); + grid_fused(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE); } /* Whether `op` is a sampled-only Cartesian transform whose forward and adjoint @@ -1027,12 +1027,12 @@ int bartorch_grid_folds_samples(const struct linop_s* op, const void* ref) /* The samples of a slab of coils, each with its sensitivity from `map`, of the * image `src`, into the slab's samples `dst`. */ void bartorch_grid_forward_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map) + const int64_t map_strs[DIMS], const complex float* map) { struct grid_s* d = CAST_DOWN(grid_s, linop_get_data(op)); #ifdef USE_CUDA - sampled_fused_forward(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (long)CFL_SIZE); + sampled_fused_forward(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE); #else (void)d; (void)dst; (void)src; (void)map_strs; (void)map; error("bartorch: the Cartesian callbacks run on a card\n"); @@ -1041,12 +1041,12 @@ void bartorch_grid_forward_sense(const struct linop_s* op, complex float* dst, c /* The adjoint of the slab's samples `src`, added to the image `dst`. */ void bartorch_grid_adjoint_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map) + const int64_t map_strs[DIMS], const complex float* map) { struct grid_s* d = CAST_DOWN(grid_s, linop_get_data(op)); #ifdef USE_CUDA - sampled_fused_adjoint(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (long)CFL_SIZE); + sampled_fused_adjoint(d, dst, src, d->cim_dims[COIL_DIM], 0, d->img_vol, map, map_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE); #else (void)d; (void)dst; (void)src; (void)map_strs; (void)map; error("bartorch: the Cartesian callbacks run on a card\n"); @@ -1057,7 +1057,7 @@ void bartorch_grid_adjoint_sense(const struct linop_s* op, complex float* dst, c * dense forward, the normal as BART's chain, and what the callbacks need. * See grid_transform_create. */ /* BART's transform as a linop in the convention asked for, or its adjoint. */ -static struct linop_s* grid_ft_create(const long dims[DIMS], unsigned long flags, bool centred, bool adjoint) +static struct linop_s* grid_ft_create(const bart_dim_t dims[DIMS], bart_flags_t flags, bool centred, bool adjoint) { if (centred) return adjoint ? linop_ifftc_create(DIMS, dims, flags) : linop_fftc_create(DIMS, dims, flags); @@ -1065,29 +1065,29 @@ static struct linop_s* grid_ft_create(const long dims[DIMS], unsigned long flags return adjoint ? linop_ifft_create(DIMS, dims, flags) : linop_fft_create(DIMS, dims, flags); } -static struct grid_s* grid_state(const long cim_dims[DIMS], - const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz, bool centred) +static struct grid_s* grid_state(const bart_dim_t cim_dims[DIMS], + const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz, bool centred) { - unsigned long fft_flags = FFT_FLAGS & md_nontriv_dims(DIMS, cim_dims); + bart_flags_t fft_flags = FFT_FLAGS & md_nontriv_dims(DIMS, cim_dims); if (NULL != pattern) { /* The kernel is laid out over the spatial axes and the frames and * nothing else, so a pattern that differs between coils or sets * of maps is not one this can collapse. */ - if (0UL != (md_nontriv_dims(DIMS, pat_dims) & ~(FFT_FLAGS | TE_FLAG))) + if (0 != (md_nontriv_dims(DIMS, pat_dims) & ~(FFT_FLAGS | TE_FLAG))) error("bartorch: a Cartesian pattern varies along the spatial axes and the frames only\n"); for (int a = 0; a < 3; a++) if ((1 != pat_dims[a]) && (pat_dims[a] != cim_dims[a])) - error("bartorch: the pattern is %ld along axis %d, where the images are %ld\n", + error("bartorch: the pattern is %" PRId64 " along axis %d, where the images are %" PRId64 "\n", pat_dims[a], a, cim_dims[a]); - long frames = (NULL == basis) ? 1 : bas_dims[TE_DIM]; + bart_dim_t frames = (NULL == basis) ? 1 : bas_dims[TE_DIM]; if ((1 != pat_dims[TE_DIM]) && (frames != pat_dims[TE_DIM])) - error("bartorch: the pattern has %ld frames and the basis %ld\n", pat_dims[TE_DIM], frames); + error("bartorch: the pattern has %" PRId64 " frames and the basis %" PRId64 "\n", pat_dims[TE_DIM], frames); } PTR_ALLOC(struct grid_s, d); @@ -1099,14 +1099,14 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], struct linop_s* fwd = grid_ft_create(cim_dims, fft_flags, centred, false); - long R = 1; + bart_dim_t R = 1; if (NULL != basis) { R = bas_dims[COEFF_DIM]; if (R != cim_dims[COEFF_DIM]) - error("bartorch: the basis has %ld coefficients and the image %ld\n", R, cim_dims[COEFF_DIM]); + error("bartorch: the basis has %" PRId64 " coefficients and the image %" PRId64 "\n", R, cim_dims[COEFF_DIM]); d->out_dims[COEFF_DIM] = 1; d->out_dims[TE_DIM] = bas_dims[TE_DIM]; @@ -1117,7 +1117,7 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], if (NULL != pattern) fwd = linop_chain_FF(fwd, linop_sampling_create(d->out_dims, pat_dims, pattern)); - unsigned long flags = (NULL == pattern) ? 0UL : (varying(pat_dims) & fft_flags); + bart_flags_t flags = (NULL == pattern) ? 0 : (varying(pat_dims) & fft_flags); /* cuFFT links no callbacks into a transform along a single axis -- its * plan answers CUFFT_INTERNAL_ERROR -- so a pattern that varies along one @@ -1125,9 +1125,9 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], * Along that axis the transform and its inverse cancel, and so do the * centring and its conjugate, so the normal is the same one; the pattern * the kernel and the kept places are read from is repeated along it. */ - long kpat_dims[DIMS]; + bart_dim_t kpat_dims[DIMS]; complex float* repeated = NULL; - long promoted = 1; + bart_dim_t promoted = 1; if (NULL != pattern) md_copy_dims(DIMS, kpat_dims, pat_dims); @@ -1144,11 +1144,11 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], kpat_dims[extra] = cim_dims[extra]; - long istrs[DIMS]; + bart_stride_t istrs[DIMS]; md_calc_strides(DIMS, istrs, pat_dims, CFL_SIZE); istrs[extra] = 0; - long ostrs[DIMS]; + bart_stride_t ostrs[DIMS]; md_calc_strides(DIMS, ostrs, kpat_dims, CFL_SIZE); complex float* host = md_alloc(DIMS, pat_dims, CFL_SIZE); @@ -1166,7 +1166,7 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], const complex float* kpattern = (NULL != repeated) ? repeated : pattern; - long kdims[DIMS]; + bart_dim_t kdims[DIMS]; complex float* K = grid_kernel(kdims, kpat_dims, kpattern, bas_dims, basis); /* The uncentred transform is unnormalized, so along the axis taken in @@ -1186,7 +1186,7 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], /* The coefficients go out on TE, where the multiply-accumulate * can put them, and a reshape reads them back on COEFF; the two * layouts are the same memory. */ - long mixed_dims[DIMS]; + bart_dim_t mixed_dims[DIMS]; md_copy_dims(DIMS, mixed_dims, cim_dims); mixed_dims[COEFF_DIM] = 1; mixed_dims[TE_DIM] = R; @@ -1218,11 +1218,11 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], } if (0 == toeplitz) - d->flags = 0UL; + d->flags = 0; struct linop_s* nrm = core; - if (0UL != d->flags) { + if (0 != d->flags) { nrm = linop_chain_FF(linop_chain_FF(grid_ft_create(cim_dims, d->flags, centred, false), core), grid_ft_create(cim_dims, d->flags, centred, true)); @@ -1263,7 +1263,7 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], d->d_csr = NULL; d->d_bt = NULL; - debug_printf(DP_DEBUG1, "Cartesian slab: %ld coefficients, normal transforms axes %lx of %lx, %ld of %ld places kept\n", + debug_printf(DP_DEBUG1, "Cartesian slab: %" PRId64 " coefficients, normal transforms axes %" PRIx64 " of %" PRIx64 ", %" PRId64 " of %" PRId64 " places kept\n", R, d->flags, fft_flags, d->kept, d->plane); return PTR_PASS(d); @@ -1272,13 +1272,13 @@ static struct grid_s* grid_state(const long cim_dims[DIMS], /* The slab transform over coil images of `cim_dims`, or over every coil when * the loop does not run. `pattern` and `basis` may each be NULL. Without * `toeplitz` the normal is the two applications, as BART derives it. */ -const struct linop_s* grid_transform_create(const long cim_dims[DIMS], - const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz) +const struct linop_s* grid_transform_create(const bart_dim_t cim_dims[DIMS], + const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz) { struct grid_s* d = grid_state(cim_dims, pat_dims, pattern, bas_dims, basis, toeplitz, true); - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; md_copy_dims(DIMS, out_dims, d->out_dims); return linop_create(DIMS, out_dims, DIMS, cim_dims, CAST_UP(d), @@ -1297,38 +1297,38 @@ const struct linop_s* grid_transform_create(const long cim_dims[DIMS], * does in the two applications. Forward, the table is read off the gathered * spectrum and transformed along its readout where that is asked for; that * is a transform of the table, not of the volume. */ -static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, int components, - const long* positions, const long bas_dims[DIMS], const complex float* basis, bool centred) +static struct grid_s* sampled_state(const bart_dim_t cim_dims[DIMS], bart_dim_t T, bart_dim_t S, int components, + const bart_dim_t* positions, const bart_dim_t bas_dims[DIMS], const complex float* basis, bool centred) { - long ny = cim_dims[PHS1_DIM]; - long nz = cim_dims[PHS2_DIM]; + bart_dim_t ny = cim_dims[PHS1_DIM]; + bart_dim_t nz = cim_dims[PHS2_DIM]; if (((1 < nz) ? 2 : 1) != components) error("bartorch: positions of %d indices for an image of %d phase-encode axes\n", components, (1 < nz) ? 2 : 1); if ((NULL == basis) ? (1 != T) : (bas_dims[TE_DIM] != T)) - error("bartorch: positions over %ld frames, and a basis of %ld\n", T, (NULL == basis) ? 1L : bas_dims[TE_DIM]); + error("bartorch: positions over %" PRId64 " frames, and a basis of %" PRId64 "\n", T, (NULL == basis) ? 1 : bas_dims[TE_DIM]); - long plane = ny * nz; - long E = T * S; + bart_dim_t plane = ny * nz; + bart_dim_t E = T * S; int* place_u = xmalloc((size_t)plane * sizeof(int)); float* counts = xmalloc((size_t)(T * plane) * sizeof(float)); int* entry_u = xmalloc((size_t)MAX(1, E) * sizeof(int)); - for (long p = 0; p < plane; p++) + for (bart_dim_t p = 0; p < plane; p++) place_u[p] = -1; - for (long i = 0; i < T * plane; i++) + for (bart_dim_t i = 0; i < T * plane; i++) counts[i] = 0.f; - long U = 0; - long valid = 0; + bart_dim_t U = 0; + bart_dim_t valid = 0; - for (long e = 0; e < E; e++) { + for (bart_dim_t e = 0; e < E; e++) { - long zc = (2 == components) ? positions[e * components] : 0; - long yc = positions[e * components + components - 1]; + bart_dim_t zc = (2 == components) ? positions[e * components] : 0; + bart_dim_t yc = positions[e * components + components - 1]; if ((-1 == zc) || (-1 == yc)) { @@ -1337,9 +1337,9 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i } if ((zc < 0) || (zc >= nz) || (yc < 0) || (yc >= ny)) - error("bartorch: a phase encode lies outside the %ld x %ld plane\n", nz, ny); + error("bartorch: a phase encode lies outside the %" PRId64 " x %" PRId64 " plane\n", nz, ny); - long p = yc + ny * zc; + bart_dim_t p = yc + ny * zc; if (-1 == place_u[p]) place_u[p] = (int)U++; @@ -1354,7 +1354,7 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i int* csr = xmalloc((size_t)MAX(1, valid) * sizeof(int)); int* fill = xmalloc((size_t)MAX(1, U) * sizeof(int)); - for (long p = 0; p < plane; p++) { + for (bart_dim_t p = 0; p < plane; p++) { if (0 > place_u[p]) continue; @@ -1363,27 +1363,27 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i u_coord[2 * place_u[p] + 1] = (int)(p % ny); } - for (long u = 0; u <= U; u++) + for (bart_dim_t u = 0; u <= U; u++) csr_start[u] = 0; - for (long e = 0; e < E; e++) + for (bart_dim_t e = 0; e < E; e++) if (0 <= entry_u[e]) csr_start[entry_u[e] + 1]++; - for (long u = 0; u < U; u++) { + for (bart_dim_t u = 0; u < U; u++) { csr_start[u + 1] += csr_start[u]; fill[u] = csr_start[u]; } - for (long e = 0; e < E; e++) + for (bart_dim_t e = 0; e < E; e++) if (0 <= entry_u[e]) csr[fill[entry_u[e]]++] = (int)e; xfree(fill); xfree(place_u); - long pat_dims[DIMS]; + bart_dim_t pat_dims[DIMS]; md_singleton_dims(DIMS, pat_dims); pat_dims[PHS1_DIM] = ny; pat_dims[PHS2_DIM] = nz; @@ -1391,7 +1391,7 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i complex float* pattern = md_alloc(DIMS, pat_dims, CFL_SIZE); - for (long i = 0; i < T * plane; i++) + for (bart_dim_t i = 0; i < T * plane; i++) pattern[i] = sqrtf(counts[i]); xfree(counts); @@ -1424,8 +1424,8 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i d->bt = xmalloc((size_t)(T * d->R) * sizeof(complex float)); - for (long t = 0; t < T; t++) - for (long r = 0; r < d->R; r++) + for (bart_dim_t t = 0; t < T; t++) + for (bart_dim_t r = 0; r < d->R; r++) d->bt[t * d->R + r] = B[t + T * r]; md_free(B); @@ -1436,14 +1436,14 @@ static struct grid_s* sampled_state(const long cim_dims[DIMS], long T, long S, i d->out_dims[PHS2_DIM] = S; d->out_dims[TE_DIM] = T; - debug_printf(DP_DEBUG1, "Cartesian samples: %ld frames x %ld shots x %ld readout at %ld places\n", + debug_printf(DP_DEBUG1, "Cartesian samples: %" PRId64 " frames x %" PRId64 " shots x %" PRId64 " readout at %" PRId64 " places\n", T, S, d->X, U); return d; } -const struct linop_s* grid_sampled_create(const long cim_dims[DIMS], long T, long S, int components, - const long* positions, const long bas_dims[DIMS], const complex float* basis, +const struct linop_s* grid_sampled_create(const bart_dim_t cim_dims[DIMS], bart_dim_t T, bart_dim_t S, int components, + const bart_dim_t* positions, const bart_dim_t bas_dims[DIMS], const complex float* basis, int kspace_readout, int toeplitz) { struct grid_s* d = sampled_state(cim_dims, T, S, components, positions, bas_dims, basis, true); @@ -1451,7 +1451,7 @@ const struct linop_s* grid_sampled_create(const long cim_dims[DIMS], long T, lon d->toeplitz = (0 != toeplitz); d->kspace_readout = (0 != kspace_readout); - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; md_copy_dims(DIMS, out_dims, d->out_dims); return linop_create(DIMS, out_dims, DIMS, cim_dims, CAST_UP(d), @@ -1460,17 +1460,17 @@ const struct linop_s* grid_sampled_create(const long cim_dims[DIMS], long T, lon /* A wave over the grid `d` holds: the domain, the offset of the zero-fill, * and the point spread function over the grid's spatial axes. */ -static void wave_setup(struct grid_s* d, const long dom_dims[DIMS], const complex float* psf) +static void wave_setup(struct grid_s* d, const bart_dim_t dom_dims[DIMS], const complex float* psf) { - long wx = d->cim_dims[READ_DIM]; - long sx = dom_dims[READ_DIM]; + bart_dim_t wx = d->cim_dims[READ_DIM]; + bart_dim_t sx = dom_dims[READ_DIM]; d->wave = true; md_copy_dims(DIMS, d->dom_dims, dom_dims); d->img_vol = md_calc_size(3, dom_dims); d->wave_off = (wx / 2 > sx / 2) ? (wx / 2 - sx / 2) : (sx / 2 - wx / 2); - long pdims[DIMS]; + bart_dim_t pdims[DIMS]; md_select_dims(DIMS, FFT_FLAGS, pdims, d->cim_dims); d->psf = md_alloc(DIMS, pdims, CFL_SIZE); @@ -1505,11 +1505,11 @@ static void wave_normal_chain(struct grid_s* d) * closed-form normal puts its kernel between the phase-encode transforms, so * it takes a pattern the same all along the readout -- the readout is not * the transform's to cancel -- and any other normal is the two applications. */ -const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, const complex float* psf, int centred, - const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz) +const struct linop_s* wave_transform_create(const bart_dim_t dom_dims[DIMS], bart_dim_t wx, const complex float* psf, int centred, + const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz) { - long cim_dims[DIMS]; + bart_dim_t cim_dims[DIMS]; md_copy_dims(DIMS, cim_dims, dom_dims); cim_dims[READ_DIM] = wx; @@ -1519,7 +1519,7 @@ const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, wave_setup(d, dom_dims, psf); - unsigned long pe = FFT_FLAGS & ~READ_FLAG & md_nontriv_dims(DIMS, cim_dims); + bart_flags_t pe = FFT_FLAGS & ~READ_FLAG & md_nontriv_dims(DIMS, cim_dims); struct linop_s* fwd = linop_chain_FF(wave_front(d), grid_ft_create(cim_dims, pe, d->centred, false)); @@ -1534,7 +1534,7 @@ const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, wave_normal_chain(d); - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; md_copy_dims(DIMS, out_dims, d->out_dims); return linop_create(DIMS, out_dims, DIMS, d->dom_dims, CAST_UP(d), @@ -1543,11 +1543,11 @@ const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, /* The same over sampled-only k-space: a table of phase encodes per frame with * the oversampled readout along each, as the front leaves it. */ -const struct linop_s* wave_sampled_create(const long dom_dims[DIMS], long wx, const complex float* psf, int centred, - long T, long S, int components, const long* positions, - const long bas_dims[DIMS], const complex float* basis, int toeplitz) +const struct linop_s* wave_sampled_create(const bart_dim_t dom_dims[DIMS], bart_dim_t wx, const complex float* psf, int centred, + bart_dim_t T, bart_dim_t S, int components, const bart_dim_t* positions, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz) { - long cim_dims[DIMS]; + bart_dim_t cim_dims[DIMS]; md_copy_dims(DIMS, cim_dims, dom_dims); cim_dims[READ_DIM] = wx; @@ -1560,7 +1560,7 @@ const struct linop_s* wave_sampled_create(const long dom_dims[DIMS], long wx, co wave_normal_chain(d); - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; md_copy_dims(DIMS, out_dims, d->out_dims); return linop_create(DIMS, out_dims, DIMS, d->dom_dims, CAST_UP(d), diff --git a/src/csrc/ops/grid.cuh b/src/csrc/ops/grid.cuh index 4b9f3e38..887d16fb 100644 --- a/src/csrc/ops/grid.cuh +++ b/src/csrc/ops/grid.cuh @@ -67,14 +67,14 @@ __device__ static inline unsigned int grid_index(const struct grid_info* g, unsi /* Where cuFFT offset `off` sits in the gathered spectrum, or -1 where the * pattern does not keep its place. */ -__device__ static inline long grid_kept(const struct grid_info* g, unsigned int off) +__device__ static inline int64_t grid_kept(const struct grid_info* g, unsigned int off) { unsigned int batch = off / g->plane; unsigned int place = off - batch * g->plane; - long j = kept_at(g->mask, g->prefix, (long)place); + int64_t j = kept_at(g->mask, g->prefix, (int64_t)place); - return (0 > j) ? -1 : (long)batch * g->L + j; + return (0 > j) ? -1 : (int64_t)batch * g->L + j; } #endif diff --git a/src/csrc/ops/iter.c b/src/csrc/ops/iter.c index 799e8ac6..bd437dd6 100644 --- a/src/csrc/ops/iter.c +++ b/src/csrc/ops/iter.c @@ -100,9 +100,9 @@ struct bartorch_prox_s { int xform; }; -int bartorch_prox_create(const char* kind, long xflags, long jflags, float lambda, int k, +int bartorch_prox_create(const char* kind, uint64_t xflags, uint64_t jflags, float lambda, int k, int llr_blk, const char* wavelet, int shift_mode, - const long* img_dims, bartorch_prox** out) + const int64_t* img_dims, bartorch_prox** out) { int xform; @@ -113,8 +113,8 @@ int bartorch_prox_create(const char* kind, long xflags, long jflags, float lambd (void)opt_reg_init(&ropts); ropts.regs[0].xform = xform; - ropts.regs[0].xflags = (unsigned long)xflags; - ropts.regs[0].jflags = (unsigned long)jflags; + ropts.regs[0].xflags = xflags; + ropts.regs[0].jflags = jflags; ropts.regs[0].lambda = lambda; ropts.regs[0].k = k; ropts.regs[0].graph_file = NULL; @@ -123,9 +123,9 @@ int bartorch_prox_create(const char* kind, long xflags, long jflags, float lambd const struct operator_p_s* prox_ops[NUM_REGS] = { NULL }; const struct linop_s* trafos[NUM_REGS] = { NULL }; - const long (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; + const bart_dim_t (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; - long dims[DIMS]; + bart_dim_t dims[DIMS]; md_copy_dims(DIMS, dims, img_dims); opt_reg_configure(DIMS, dims, &ropts, prox_ops, trafos, sdims, @@ -150,10 +150,10 @@ int bartorch_prox_create(const char* kind, long xflags, long jflags, float lambd } int bartorch_prox_set_create(int n, const char* const* kinds, - const long* xflags, const long* jflags, const float* lambda, const int* k, + const uint64_t* xflags, const uint64_t* jflags, const float* lambda, const int* k, int llr_blk, const char* wavelet, int shift_mode, - const float* alpha, const float* gamma, const long* img_dims, - int max_out, bartorch_prox** out, int* count, long* svars) + const float* alpha, const float* gamma, const int64_t* img_dims, + int max_out, bartorch_prox** out, int* count, int64_t* svars) { if ((0 > n) || (NUM_REGS < n) || (NULL == out) || (NULL == count) || (NULL == svars)) return -1; @@ -169,8 +169,8 @@ int bartorch_prox_set_create(int n, const char* const* kinds, return -4; ropts.regs[i].xform = xform; - ropts.regs[i].xflags = (unsigned long)xflags[i]; - ropts.regs[i].jflags = (unsigned long)jflags[i]; + ropts.regs[i].xflags = xflags[i]; + ropts.regs[i].jflags = jflags[i]; ropts.regs[i].lambda = lambda[i]; ropts.regs[i].k = k[i]; ropts.regs[i].graph_file = NULL; @@ -189,9 +189,9 @@ int bartorch_prox_set_create(int n, const char* const* kinds, const struct operator_p_s* prox_ops[NUM_REGS] = { NULL }; const struct linop_s* trafos[NUM_REGS] = { NULL }; - const long (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; + const bart_dim_t (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; - long dims[DIMS]; + bart_dim_t dims[DIMS]; md_copy_dims(DIMS, dims, img_dims); opt_reg_configure(DIMS, dims, &ropts, prox_ops, trafos, sdims, @@ -232,7 +232,7 @@ int bartorch_prox_set_create(int n, const char* const* kinds, * Returns the rank, or a negative code. `dims` holds `N` entries and is * filled with ones past the rank. */ -int bartorch_prox_domain(const bartorch_prox* h, int N, long* dims) +int bartorch_prox_domain(const bartorch_prox* h, int N, int64_t* dims) { if ((NULL == h) || (NULL == h->op) || (NULL == dims) || (1 > N)) return -1; @@ -383,7 +383,7 @@ void bartorch_prox_free(bartorch_prox* h) struct counting_monitor { struct iter_monitor_s super; - long count; + bart_dim_t count; }; static void counting_monitor_fun(struct iter_monitor_s* monitor, const struct vec_iter_s* /*ops*/, const float* /*x*/) @@ -403,7 +403,7 @@ struct maxeigen_data { operator_data_t super; float lambda; - long size; + bart_dim_t size; const struct linop_s* model_op; }; @@ -510,7 +510,7 @@ int bartorch_maxeigen(const bartorch_linop* handle, const bartorch_linop* precon int bartorch_solve(const bartorch_linop* handle, const char* algorithm, - const char* const* reg_kinds, const long* reg_xflags, const long* reg_jflags, + const char* const* reg_kinds, const uint64_t* reg_xflags, const uint64_t* reg_jflags, const float* reg_lambda, const int* reg_k, const bartorch_prox* const* reg_ops, int n_reg, float cclambda, int maxiter, float step, int eigen, int hogwild, @@ -522,7 +522,7 @@ int bartorch_solve(const bartorch_linop* handle, const bartorch_linop* precond, int llr_blk, const char* wavelet, int shift_mode, const float* alpha, const float* gamma, - void* x, const void* y, long* iterations) + void* x, const void* y, int64_t* iterations) { const struct linop_s* model_op = bartorch_linop_unwrap(handle); @@ -582,8 +582,8 @@ int bartorch_solve(const bartorch_linop* handle, return -4; ropts.regs[i].xform = xform; - ropts.regs[i].xflags = (unsigned long)reg_xflags[i]; - ropts.regs[i].jflags = (unsigned long)reg_jflags[i]; + ropts.regs[i].xflags = reg_xflags[i]; + ropts.regs[i].jflags = reg_jflags[i]; ropts.regs[i].lambda = reg_lambda[i]; ropts.regs[i].k = reg_k[i]; ropts.regs[i].graph_file = NULL; @@ -595,10 +595,10 @@ int bartorch_solve(const bartorch_linop* handle, const struct operator_p_s* thresh_ops[NUM_REGS] = { NULL }; const struct linop_s* trafos[NUM_REGS] = { NULL }; - long img_dims[DIMS]; + bart_dim_t img_dims[DIMS]; md_copy_dims(DIMS, img_dims, linop_domain(model_op)->dims); - long ksp_dims[DIMS]; + bart_dim_t ksp_dims[DIMS]; md_copy_dims(DIMS, ksp_dims, linop_codomain(model_op)->dims); /* Three terms are not one proximal operator on the image: total @@ -619,7 +619,7 @@ int bartorch_solve(const bartorch_linop* handle, if (extending) { - const long (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; + const bart_dim_t (*sdims[NUM_REGS])[DIMS + 1] = { NULL }; /* `--alpha` and `--gamma` are one pair each for the whole set, and * only these three terms read them; `opt_reg_init` has already put @@ -720,8 +720,8 @@ int bartorch_solve(const bartorch_linop* handle, * of that vector out and the encoding is chained onto it, which is what * `pics.c` does. The solve then walks the long vector and the image is * the front of the answer. */ - long total = md_calc_size(DIMS, img_dims); - long ext = total + ropts.svars; + bart_dim_t total = md_calc_size(DIMS, img_dims); + bart_dim_t ext = total + ropts.svars; complex float* xbig = NULL; @@ -785,7 +785,7 @@ int bartorch_solve(const bartorch_linop* handle, return 0; } -float bartorch_scaling_norm(long size, const void* image, float rescale, int compat, float p) +float bartorch_scaling_norm(int64_t size, const void* image, float rescale, int compat, float p) { /* On the host, and on a copy: the estimate is read off order statistics * and BART sorts the array it is handed to get them. */ diff --git a/src/csrc/ops/kernels.cu b/src/csrc/ops/kernels.cu index b587c8e4..1f004e21 100644 --- a/src/csrc/ops/kernels.cu +++ b/src/csrc/ops/kernels.cu @@ -38,15 +38,15 @@ __global__ static void kern_phase_map_in(struct phase_conf c, cuFloatComplex* ds int startZ = threadIdx.z + blockDim.z * blockIdx.z; int strideZ = blockDim.z * gridDim.z; - for (long z = startZ; z < c.dims[2]; z += strideZ) - for (long y = startY; y < c.dims[1]; y += strideY) - for (long x = startX; x < c.dims[0]; x += strideX) { + for (bart_dim_t z = startZ; z < c.dims[2]; z += strideZ) + for (bart_dim_t y = startY; y < c.dims[1]; y += strideY) + for (bart_dim_t x = startX; x < c.dims[0]; x += strideX) { - long idx = x + c.dims[0] * (y + c.dims[1] * z); + bart_dim_t idx = x + c.dims[0] * (y + c.dims[1] * z); cuFloatComplex w = cuCmulf(map[idx], phase_at(c, x, y, z, false)); - for (long i = 0; i < c.batch; i++) + for (bart_dim_t i = 0; i < c.batch; i++) dst[idx + i * c.tot] = cuCmulf(src[idx + i * c.tot], w); } } @@ -62,20 +62,20 @@ __global__ static void kern_phase_map_out(struct phase_conf c, cuFloatComplex* d int startZ = threadIdx.z + blockDim.z * blockIdx.z; int strideZ = blockDim.z * gridDim.z; - for (long z = startZ; z < c.dims[2]; z += strideZ) - for (long y = startY; y < c.dims[1]; y += strideY) - for (long x = startX; x < c.dims[0]; x += strideX) { + for (bart_dim_t z = startZ; z < c.dims[2]; z += strideZ) + for (bart_dim_t y = startY; y < c.dims[1]; y += strideY) + for (bart_dim_t x = startX; x < c.dims[0]; x += strideX) { - long idx = x + c.dims[0] * (y + c.dims[1] * z); + bart_dim_t idx = x + c.dims[0] * (y + c.dims[1] * z); cuFloatComplex w = cuCmulf(cuConjf(map[idx]), phase_at(c, x, y, z, true)); - for (long i = 0; i < c.batch; i++) + for (bart_dim_t i = 0; i < c.batch; i++) dst[idx + i * c.tot] = cuCaddf(dst[idx + i * c.tot], cuCmulf(src[idx + i * c.tot], w)); } } -extern "C" void bartorch_cuda_phase_map_in(int N, const long dims[], const float shift[3], float scale, +extern "C" void bartorch_cuda_phase_map_in(int N, const bart_dim_t dims[], const float shift[3], float scale, _Complex float* dst, const _Complex float* src, const _Complex float* map) { struct phase_conf c = phase_setup(N, dims, shift, scale); @@ -88,7 +88,7 @@ extern "C" void bartorch_cuda_phase_map_in(int N, const long dims[], const float CUDA_KERNEL_ERROR; } -extern "C" void bartorch_cuda_phase_map_out(int N, const long dims[], const float shift[3], float scale, +extern "C" void bartorch_cuda_phase_map_out(int N, const bart_dim_t dims[], const float shift[3], float scale, _Complex float* dst, const _Complex float* src, const _Complex float* map) { struct phase_conf c = phase_setup(N, dims, shift, scale); @@ -104,43 +104,43 @@ extern "C" void bartorch_cuda_phase_map_out(int N, const long dims[], const floa /* Gather and scatter over the places the samples reach, found through the * mask and counts of coset.cuh. A scatter writes zeros everywhere else, which * is what the transform that follows it needs to see there. */ -__global__ static void kern_gather(long V, const unsigned int* mask, const int* prefix, +__global__ static void kern_gather(bart_dim_t V, const unsigned int* mask, const int* prefix, cuFloatComplex* dst, const cuFloatComplex* src) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; - for (long i = start; i < V; i += stride) { + for (bart_dim_t i = start; i < V; i += stride) { - long j = kept_at(mask, prefix, i); + bart_dim_t j = kept_at(mask, prefix, i); if (0 <= j) dst[j] = src[i]; } } -__global__ static void kern_scatter(long V, const unsigned int* mask, const int* prefix, +__global__ static void kern_scatter(bart_dim_t V, const unsigned int* mask, const int* prefix, cuFloatComplex* dst, const cuFloatComplex* src) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; - for (long i = start; i < V; i += stride) { + for (bart_dim_t i = start; i < V; i += stride) { - long j = kept_at(mask, prefix, i); + bart_dim_t j = kept_at(mask, prefix, i); dst[i] = (0 <= j) ? src[j] : make_cuFloatComplex(0.f, 0.f); } } -static dim3 grid_for(long n) +static dim3 grid_for(bart_dim_t n) { - long blocks = (n + 255) / 256; + bart_dim_t blocks = (n + 255) / 256; return dim3((unsigned int)((blocks < 65535) ? blocks : 65535)); } -extern "C" void bartorch_cuda_gather(long V, const unsigned int* mask, const int* prefix, +extern "C" void bartorch_cuda_gather(bart_dim_t V, const unsigned int* mask, const int* prefix, _Complex float* dst, const _Complex float* src) { kern_gather<<>>(V, mask, prefix, (cuFloatComplex*)dst, (const cuFloatComplex*)src); @@ -148,7 +148,7 @@ extern "C" void bartorch_cuda_gather(long V, const unsigned int* mask, const int CUDA_KERNEL_ERROR; } -extern "C" void bartorch_cuda_scatter(long V, const unsigned int* mask, const int* prefix, +extern "C" void bartorch_cuda_scatter(bart_dim_t V, const unsigned int* mask, const int* prefix, _Complex float* dst, const _Complex float* src) { kern_scatter<<>>(V, mask, prefix, (cuFloatComplex*)dst, (const cuFloatComplex*)src); @@ -188,7 +188,7 @@ __device__ static inline cuFloatComplex ifftmod_at(int n, int j) * what the centred transforms of a padded kernel put on around their * transforms. Every axis after the third is a batch the same factor * multiplies. */ -__global__ static void kern_modulate(unsigned int d0, unsigned int d1, unsigned int d2, long rest, +__global__ static void kern_modulate(unsigned int d0, unsigned int d1, unsigned int d2, bart_dim_t rest, int g0, int g1, int g2, int o0, int o1, int o2, float scale, cuFloatComplex* x) { unsigned int vol = d0 * d1 * d2; @@ -204,15 +204,15 @@ __global__ static void kern_modulate(unsigned int d0, unsigned int d1, unsigned f = make_cuFloatComplex(scale * f.x, scale * f.y); - for (long r = 0; r < rest; r++) + for (bart_dim_t r = 0; r < rest; r++) x[i + r * vol] = cuCmulf(x[i + r * vol], f); } } -extern "C" void bartorch_cuda_modulate(const long dims[3], long rest, const long grid[3], const long off[3], +extern "C" void bartorch_cuda_modulate(const bart_dim_t dims[3], bart_dim_t rest, const bart_dim_t grid[3], const bart_stride_t off[3], float scale, _Complex float* x) { - long vol = dims[0] * dims[1] * dims[2]; + bart_dim_t vol = dims[0] * dims[1] * dims[2]; kern_modulate<<>>((unsigned int)dims[0], (unsigned int)dims[1], (unsigned int)dims[2], rest, (int)grid[0], (int)grid[1], (int)grid[2], @@ -232,12 +232,12 @@ extern "C" void bartorch_cuda_modulate(const long dims[3], long rest, const long enum { CONTRACT_MAX = 16 }; template -__global__ static void kern_contract_upper_real(long L, int R, cuFloatComplex* bank, const P* mat) +__global__ static void kern_contract_upper_real(bart_dim_t L, int R, cuFloatComplex* bank, const P* mat) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; - for (long l = start; l < L; l += stride) { + for (bart_dim_t l = start; l < L; l += stride) { cuFloatComplex in[CONTRACT_MAX]; @@ -253,7 +253,7 @@ __global__ static void kern_contract_upper_real(long L, int R, cuFloatComplex* b int lo = (r < c) ? r : c; int hi = (r < c) ? c : r; - float m = widen(mat[(long)(lo + hi * (hi + 1) / 2) * L + l]); + float m = widen(mat[(bart_dim_t)(lo + hi * (hi + 1) / 2) * L + l]); re += m * in[c].x; im += m * in[c].y; @@ -264,7 +264,7 @@ __global__ static void kern_contract_upper_real(long L, int R, cuFloatComplex* b } } -extern "C" int bartorch_cuda_contract_upper_real(long L, int R, _Complex float* bank, const float* mat) +extern "C" int bartorch_cuda_contract_upper_real(bart_dim_t L, int R, _Complex float* bank, const float* mat) { if ((R < 1) || (R > CONTRACT_MAX)) return -1; @@ -277,7 +277,7 @@ extern "C" int bartorch_cuda_contract_upper_real(long L, int R, _Complex float* } /* The same, against a function kept in bfloat16. */ -extern "C" int bartorch_cuda_contract_upper_real_bf16(long L, int R, _Complex float* bank, const void* mat) +extern "C" int bartorch_cuda_contract_upper_real_bf16(bart_dim_t L, int R, _Complex float* bank, const void* mat) { if ((R < 1) || (R > CONTRACT_MAX)) return -1; @@ -296,15 +296,15 @@ extern "C" int bartorch_cuda_contract_upper_real_bf16(long L, int R, _Complex fl * kernel and written back over itself. Entry (r, c) of the kernel at place l * is `K[(l R + r) R + c]`: the kernel varies over the transformed plane only, * so every batch reads the same one. */ -__global__ static void kern_contract_grid(long L, long B, int R, cuFloatComplex* bank, const cuFloatComplex* K) +__global__ static void kern_contract_grid(bart_dim_t L, bart_dim_t B, int R, cuFloatComplex* bank, const cuFloatComplex* K) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = L * B; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = L * B; - for (long i = start; i < n; i += stride) { + for (bart_dim_t i = start; i < n; i += stride) { - long l = i % L; + bart_dim_t l = i % L; cuFloatComplex in[CONTRACT_MAX]; @@ -323,7 +323,7 @@ __global__ static void kern_contract_grid(long L, long B, int R, cuFloatComplex* } } -extern "C" int bartorch_cuda_contract_grid(long L, long B, int R, _Complex float* bank, const _Complex float* K) +extern "C" int bartorch_cuda_contract_grid(bart_dim_t L, bart_dim_t B, int R, _Complex float* bank, const _Complex float* K) { if ((R < 1) || (R > CONTRACT_MAX)) return -1; @@ -350,60 +350,60 @@ struct bartorch_grid_axes { /* Where coordinates `c` (x, y, z) sit in the spectrum, or -1, and the * centring there. */ -__device__ static inline long grid_spectrum_at(const struct bartorch_grid_axes* ax, long L, - const unsigned int* mask, const int* prefix, const long c[3], cuFloatComplex* mod) +__device__ static inline bart_dim_t grid_spectrum_at(const struct bartorch_grid_axes* ax, bart_dim_t L, + const unsigned int* mask, const int* prefix, const bart_dim_t c[3], cuFloatComplex* mod) { - long place = 0; - long batch = 0; + bart_dim_t place = 0; + bart_dim_t batch = 0; cuFloatComplex m = make_cuFloatComplex(1.f, 0.f); for (int a = 0; a < 3; a++) { if (0 != ax->pstr[a]) { - place += c[a] * (long)ax->pstr[a]; + place += c[a] * (bart_dim_t)ax->pstr[a]; m = cuCmulf(m, ax->mod[a][c[a]]); } else if (0 != ax->bstr[a]) { - batch += c[a] * (long)ax->bstr[a]; + batch += c[a] * (bart_dim_t)ax->bstr[a]; } } *mod = m; - long j = kept_at(mask, prefix, place); + bart_dim_t j = kept_at(mask, prefix, place); return (0 > j) ? -1 : batch * L + j; } /* table[e X + x] = centring * sum_r B[t R + r] bank[r per + at(x, place of e)], * with t = e / S, zero for a padding entry. B NULL is one coefficient. */ -__global__ static void kern_bank_to_table(long E, long X, long S, int R, long L, long per, +__global__ static void kern_bank_to_table(bart_dim_t E, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* entry_u, const int* u_coord, const unsigned int* mask, const int* prefix, struct bartorch_grid_axes ax, const cuFloatComplex* B, const cuFloatComplex* bank, cuFloatComplex* table) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = E * X; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = E * X; - for (long i = start; i < n; i += stride) { + for (bart_dim_t i = start; i < n; i += stride) { - long e = i / X; - long u = entry_u[e]; + bart_dim_t e = i / X; + bart_dim_t u = entry_u[e]; cuFloatComplex acc = make_cuFloatComplex(0.f, 0.f); if (0 <= u) { - long c[3] = { i - e * X, u_coord[2 * u + 1], u_coord[2 * u] }; + bart_dim_t c[3] = { i - e * X, u_coord[2 * u + 1], u_coord[2 * u] }; cuFloatComplex m; - long at = grid_spectrum_at(&ax, L, mask, prefix, c, &m); + bart_dim_t at = grid_spectrum_at(&ax, L, mask, prefix, c, &m); if (0 <= at) { - long t = e / S; + bart_dim_t t = e / S; for (int r = 0; r < R; r++) { @@ -423,21 +423,21 @@ __global__ static void kern_bank_to_table(long E, long X, long S, int R, long L, /* The adjoint: bank[r per + at(x, u)] = conj(centring) * sum over the entries * e of place u of conj(B[t R + r]) table[e X + x]. Each (u, x) is its own * place of the spectrum, so no two threads write the same one. */ -__global__ static void kern_table_to_bank(long U, long X, long S, int R, long L, long per, +__global__ static void kern_table_to_bank(bart_dim_t U, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* csr_start, const int* csr, const int* u_coord, const unsigned int* mask, const int* prefix, struct bartorch_grid_axes ax, const cuFloatComplex* B, cuFloatComplex* bank, const cuFloatComplex* table) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = U * X; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = U * X; - for (long i = start; i < n; i += stride) { + for (bart_dim_t i = start; i < n; i += stride) { - long u = i / X; - long c[3] = { i - u * X, u_coord[2 * u + 1], u_coord[2 * u] }; + bart_dim_t u = i / X; + bart_dim_t c[3] = { i - u * X, u_coord[2 * u + 1], u_coord[2 * u] }; cuFloatComplex m; - long at = grid_spectrum_at(&ax, L, mask, prefix, c, &m); + bart_dim_t at = grid_spectrum_at(&ax, L, mask, prefix, c, &m); if (0 > at) continue; @@ -448,9 +448,9 @@ __global__ static void kern_table_to_bank(long U, long X, long S, int R, long L, cuFloatComplex acc = make_cuFloatComplex(0.f, 0.f); - for (long k = csr_start[u]; k < csr_start[u + 1]; k++) { + for (bart_dim_t k = csr_start[u]; k < csr_start[u + 1]; k++) { - long e = csr[k]; + bart_dim_t e = csr[k]; cuFloatComplex v = table[e * X + c[0]]; acc = cuCaddf(acc, (NULL == B) ? v : cuCmulf(cuConjf(B[(e / S) * R + r]), v)); @@ -461,7 +461,7 @@ __global__ static void kern_table_to_bank(long U, long X, long S, int R, long L, } } -extern "C" void bartorch_cuda_bank_to_table(long E, long X, long S, int R, long L, long per, +extern "C" void bartorch_cuda_bank_to_table(bart_dim_t E, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* entry_u, const int* u_coord, const unsigned int* mask, const int* prefix, const struct bartorch_grid_axes* ax, const _Complex float* B, const _Complex float* bank, _Complex float* table) { @@ -471,7 +471,7 @@ extern "C" void bartorch_cuda_bank_to_table(long E, long X, long S, int R, long CUDA_KERNEL_ERROR; } -extern "C" void bartorch_cuda_table_to_bank(long U, long X, long S, int R, long L, long per, +extern "C" void bartorch_cuda_table_to_bank(bart_dim_t U, bart_dim_t X, bart_dim_t S, int R, bart_dim_t L, bart_dim_t per, const int* csr_start, const int* csr, const int* u_coord, const unsigned int* mask, const int* prefix, const struct bartorch_grid_axes* ax, const _Complex float* B, _Complex float* bank, const _Complex float* table) { @@ -485,17 +485,17 @@ extern "C" void bartorch_cuda_table_to_bank(long U, long X, long S, int R, long * sensitivity, zero-filled along the readout from `sx` to `wx` about the * centre, and its adjoint. The image is laid out as BART lays it, readout * fastest; `map` NULL is one. */ -__global__ static void kern_pad_map(long sx, long wx, long rest, long off, +__global__ static void kern_pad_map(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, cuFloatComplex* dst, const cuFloatComplex* src, const cuFloatComplex* map) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = wx * rest; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = wx * rest; - for (long i = start; i < n; i += stride) { + for (bart_dim_t i = start; i < n; i += stride) { - long r = i / wx; - long x = i - r * wx - off; + bart_dim_t r = i / wx; + bart_dim_t x = i - r * wx - off; if ((0 > x) || (x >= sx)) { @@ -503,22 +503,22 @@ __global__ static void kern_pad_map(long sx, long wx, long rest, long off, continue; } - long j = x + r * sx; + bart_dim_t j = x + r * sx; dst[i] = (NULL == map) ? src[j] : cuCmulf(src[j], map[j]); } } -__global__ static void kern_crop_mapc_add(long sx, long wx, long rest, long off, +__global__ static void kern_crop_mapc_add(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, cuFloatComplex* dst, const cuFloatComplex* src, const cuFloatComplex* map) { - long start = threadIdx.x + (long)blockDim.x * blockIdx.x; - long stride = (long)blockDim.x * gridDim.x; - long n = sx * rest; + bart_dim_t start = threadIdx.x + (bart_dim_t)blockDim.x * blockIdx.x; + bart_stride_t stride = (bart_dim_t)blockDim.x * gridDim.x; + bart_dim_t n = sx * rest; - for (long j = start; j < n; j += stride) { + for (bart_dim_t j = start; j < n; j += stride) { - long r = j / sx; + bart_dim_t r = j / sx; cuFloatComplex v = src[(j - r * sx) + off + r * wx]; if (NULL != map) @@ -528,7 +528,7 @@ __global__ static void kern_crop_mapc_add(long sx, long wx, long rest, long off, } } -extern "C" void bartorch_cuda_pad_map(long sx, long wx, long rest, long off, +extern "C" void bartorch_cuda_pad_map(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, _Complex float* dst, const _Complex float* src, const _Complex float* map) { kern_pad_map<<>>(sx, wx, rest, off, @@ -537,7 +537,7 @@ extern "C" void bartorch_cuda_pad_map(long sx, long wx, long rest, long off, CUDA_KERNEL_ERROR; } -extern "C" void bartorch_cuda_crop_mapc_add(long sx, long wx, long rest, long off, +extern "C" void bartorch_cuda_crop_mapc_add(bart_dim_t sx, bart_dim_t wx, bart_dim_t rest, bart_stride_t off, _Complex float* dst, const _Complex float* src, const _Complex float* map) { kern_crop_mapc_add<<>>(sx, wx, rest, off, diff --git a/src/csrc/ops/ops.c b/src/csrc/ops/ops.c index 37a5aef5..dee19392 100644 --- a/src/csrc/ops/ops.c +++ b/src/csrc/ops/ops.c @@ -129,7 +129,7 @@ static bartorch_nlop* wrap_nlop(const struct nlop_s* op) return h; } -static void copy_dims(const struct iovec_s* iov, int N, long* dims) +static void copy_dims(const struct iovec_s* iov, int N, bart_dim_t* dims) { for (int i = 0; i < N; i++) dims[i] = (i < iov->N) ? iov->dims[i] : 1; @@ -186,7 +186,7 @@ static void cb_lin_del(const linop_data_t* _d) struct linop_callback_args { - int ON; const long* odims; int IN; const long* idims; + int ON; const bart_dim_t* odims; int IN; const bart_dim_t* idims; bartorch_apply_fn forward; bartorch_apply_fn adjoint; bartorch_apply_fn normal; void* ctx; bartorch_release_fn release; bartorch_linop* result; @@ -209,7 +209,7 @@ static int linop_callback_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_callback(int ON, const long* odims, int IN, const long* idims, +bartorch_linop* bartorch_linop_callback(int ON, const int64_t* odims, int IN, const int64_t* idims, bartorch_apply_fn forward, bartorch_apply_fn adjoint, bartorch_apply_fn normal, void* ctx, bartorch_release_fn release) { @@ -220,7 +220,7 @@ bartorch_linop* bartorch_linop_callback(int ON, const long* odims, int IN, const /* --- BART's own linear operators ------------------------------------------- */ -struct linop_fft_args { int N; const long* dims; unsigned long flags; int inverse; int centered; bartorch_linop* result; }; +struct linop_fft_args { int N; const bart_dim_t* dims; bart_flags_t flags; int inverse; int centered; bartorch_linop* result; }; static int linop_fft_worker(void* p) { @@ -236,13 +236,13 @@ static int linop_fft_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_fft(int N, const long* dims, unsigned long flags, int inverse, int centered) +bartorch_linop* bartorch_linop_fft(int N, const int64_t* dims, uint64_t flags, int inverse, int centered) { struct linop_fft_args a = { N, dims, flags, inverse, centered, NULL }; return (0 == guarded(linop_fft_worker, &a)) ? a.result : NULL; } -struct linop_cdiag_args { int N; const long* dims; unsigned long flags; const void* diag; bartorch_linop* result; }; +struct linop_cdiag_args { int N; const bart_dim_t* dims; bart_flags_t flags; const void* diag; bartorch_linop* result; }; static int linop_cdiag_worker(void* p) { @@ -251,13 +251,13 @@ static int linop_cdiag_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_cdiag(int N, const long* dims, unsigned long flags, const void* diag) +bartorch_linop* bartorch_linop_cdiag(int N, const int64_t* dims, uint64_t flags, const void* diag) { struct linop_cdiag_args a = { N, dims, flags, diag, NULL }; return (0 == guarded(linop_cdiag_worker, &a)) ? a.result : NULL; } -struct linop_fmac_args { int N; const long* odims; const long* idims; const long* tdims; const void* tensor; bartorch_linop* result; }; +struct linop_fmac_args { int N; const bart_dim_t* odims; const bart_dim_t* idims; const bart_dim_t* tdims; const void* tensor; bartorch_linop* result; }; static int linop_fmac_worker(void* p) { @@ -266,13 +266,13 @@ static int linop_fmac_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_fmac(int N, const long* odims, const long* idims, const long* tdims, const void* tensor) +bartorch_linop* bartorch_linop_fmac(int N, const int64_t* odims, const int64_t* idims, const int64_t* tdims, const void* tensor) { struct linop_fmac_args a = { N, odims, idims, tdims, tensor, NULL }; return (0 == guarded(linop_fmac_worker, &a)) ? a.result : NULL; } -struct linop_sampling_args { const long* dims; const long* pat_dims; const void* pattern; bartorch_linop* result; }; +struct linop_sampling_args { const bart_dim_t* dims; const bart_dim_t* pat_dims; const void* pattern; bartorch_linop* result; }; static int linop_sampling_worker(void* p) { @@ -281,13 +281,13 @@ static int linop_sampling_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_sampling(const long* dims, const long* pat_dims, const void* pattern) +bartorch_linop* bartorch_linop_sampling(const int64_t* dims, const int64_t* pat_dims, const void* pattern) { struct linop_sampling_args a = { dims, pat_dims, pattern, NULL }; return (0 == guarded(linop_sampling_worker, &a)) ? a.result : NULL; } -struct linop_set_diag_args { const bartorch_linop* h; int N; const long* ddims; const void* diag; }; +struct linop_set_diag_args { const bartorch_linop* h; int N; const bart_dim_t* ddims; const void* diag; }; static int linop_set_diag_worker(void* p) { @@ -303,7 +303,7 @@ static int linop_set_diag_worker(void* p) return 0; } -int bartorch_linop_set_diagonal(const bartorch_linop* h, int N, const long* ddims, const void* diag) +int bartorch_linop_set_diagonal(const bartorch_linop* h, int N, const int64_t* ddims, const void* diag) { struct linop_set_diag_args a = { h, N, ddims, diag }; return guarded(linop_set_diag_worker, &a); @@ -311,9 +311,9 @@ int bartorch_linop_set_diagonal(const bartorch_linop* h, int N, const long* ddim struct linop_nufft_args { - int N; const long* ksp_dims; const long* cim_dims; const long* traj_dims; const void* traj; - const long* wgh_dims; const void* weights; - const long* bas_dims; const void* basis; + int N; const bart_dim_t* ksp_dims; const bart_dim_t* cim_dims; const bart_dim_t* traj_dims; const void* traj; + const bart_dim_t* wgh_dims; const void* weights; + const bart_dim_t* bas_dims; const void* basis; int toeplitz; float os; float width; bartorch_linop* result; }; @@ -334,7 +334,7 @@ static int linop_nufft_worker(void* p) /* Weights and a subspace basis belong to the operator rather than to * something chained onto it: the normal is a point spread function over * both, which a chain could not be. */ - long wgh_dims[a->N]; + bart_dim_t wgh_dims[a->N]; if (NULL == a->weights) md_singleton_dims(a->N, wgh_dims); @@ -346,9 +346,9 @@ static int linop_nufft_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_nufft(int N, const long* ksp_dims, const long* cim_dims, const long* traj_dims, - const void* traj, const long* wgh_dims, const void* weights, - const long* bas_dims, const void* basis, int toeplitz, float os, float width) +bartorch_linop* bartorch_linop_nufft(int N, const int64_t* ksp_dims, const int64_t* cim_dims, const int64_t* traj_dims, + const void* traj, const int64_t* wgh_dims, const void* weights, + const int64_t* bas_dims, const void* basis, int toeplitz, float os, float width) { struct linop_nufft_args a = { N, ksp_dims, cim_dims, traj_dims, traj, wgh_dims, weights, bas_dims, basis, toeplitz, os, width, NULL }; @@ -369,9 +369,9 @@ struct blocks_s { linop_data_t super; const struct linop_s* op; - long n; - long isize; - long osize; + bart_dim_t n; + bart_dim_t isize; + bart_dim_t osize; }; static DEF_TYPEID(blocks_s); @@ -380,7 +380,7 @@ static void blocks_forward(const linop_data_t* _d, complex float* dst, const com { const auto d = CAST_DOWN(blocks_s, _d); - for (long k = 0; k < d->n; k++) + for (bart_dim_t k = 0; k < d->n; k++) linop_forward_unchecked(d->op, dst + k * d->osize, src + k * d->isize); } @@ -388,7 +388,7 @@ static void blocks_adjoint(const linop_data_t* _d, complex float* dst, const com { const auto d = CAST_DOWN(blocks_s, _d); - for (long k = 0; k < d->n; k++) + for (bart_dim_t k = 0; k < d->n; k++) linop_adjoint_unchecked(d->op, dst + k * d->isize, src + k * d->osize); } @@ -396,7 +396,7 @@ static void blocks_normal(const linop_data_t* _d, complex float* dst, const comp { const auto d = CAST_DOWN(blocks_s, _d); - for (long k = 0; k < d->n; k++) + for (bart_dim_t k = 0; k < d->n; k++) linop_normal_unchecked(d->op, dst + k * d->isize, src + k * d->isize); } @@ -410,7 +410,7 @@ static void blocks_del(const linop_data_t* _d) struct linop_blocks_args { - const bartorch_linop* block; int N; const long* odims; const long* idims; long n; + const bartorch_linop* block; int N; const bart_dim_t* odims; const bart_dim_t* idims; bart_dim_t n; bartorch_linop* result; }; @@ -421,11 +421,11 @@ static int linop_blocks_worker(void* p) auto dom = linop_domain(a->block->op); auto cod = linop_codomain(a->block->op); - long isize = md_calc_size(dom->N, dom->dims); - long osize = md_calc_size(cod->N, cod->dims); + bart_dim_t isize = md_calc_size(dom->N, dom->dims); + bart_dim_t osize = md_calc_size(cod->N, cod->dims); if ((a->n < 1) || (md_calc_size(a->N, a->idims) != a->n * isize) || (md_calc_size(a->N, a->odims) != a->n * osize)) - error("bartorch: %ld blocks of an operator from %ld to %ld elements are not %ld to %ld elements\n", + error("bartorch: %" PRId64 " blocks of an operator from %" PRId64 " to %" PRId64 " elements are not %" PRId64 " to %" PRId64 " elements\n", a->n, isize, osize, md_calc_size(a->N, a->idims), md_calc_size(a->N, a->odims)); PTR_ALLOC(struct blocks_s, d); @@ -441,7 +441,7 @@ static int linop_blocks_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_blocks(const bartorch_linop* block, int N, const long* odims, const long* idims, long n) +bartorch_linop* bartorch_linop_blocks(const bartorch_linop* block, int N, const int64_t* odims, const int64_t* idims, int64_t n) { struct linop_blocks_args a = { block, N, odims, idims, n, NULL }; return (0 == guarded(linop_blocks_worker, &a)) ? a.result : NULL; @@ -452,7 +452,7 @@ bartorch_linop* bartorch_linop_blocks(const bartorch_linop* block, int N, const * hands the caller's buffers straight through. An operator built on BART's * roles for a torch layout is given the plain reversal of the torch shapes * this way, so it chains with every other operator built on that reversal. */ -struct linop_reshaped_args { const bartorch_linop* op; int N; const long* odims; const long* idims; bartorch_linop* result; }; +struct linop_reshaped_args { const bartorch_linop* op; int N; const bart_dim_t* odims; const bart_dim_t* idims; bartorch_linop* result; }; static int linop_reshaped_worker(void* p) { @@ -469,7 +469,7 @@ static int linop_reshaped_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_reshaped(const bartorch_linop* op, int N, const long* odims, const long* idims) +bartorch_linop* bartorch_linop_reshaped(const bartorch_linop* op, int N, const int64_t* odims, const int64_t* idims) { struct linop_reshaped_args a = { op, N, odims, idims, NULL }; return (0 == guarded(linop_reshaped_worker, &a)) ? a.result : NULL; @@ -520,8 +520,8 @@ static int linop_with_normal_worker(void* p) /* `linop_from_ops` asserts this, and an assertion here would take the * process rather than the call. */ - if (!md_check_equal_dims(dom->N, dom->dims, nrm_dom->dims, ~0UL) - || !md_check_equal_dims(dom->N, dom->dims, nrm_cod->dims, ~0UL)) + if (!md_check_equal_dims(dom->N, dom->dims, nrm_dom->dims, ~UINT64_C(0)) + || !md_check_equal_dims(dom->N, dom->dims, nrm_cod->dims, ~UINT64_C(0))) error("bartorch: a normal operator maps the domain to itself\n"); /* `linop_from_ops` takes its own reference to each of these. */ @@ -639,7 +639,7 @@ bartorch_linop* bartorch_linop_normal_op(const bartorch_linop* a) return (0 == guarded(linop_normal_op_worker, &args)) ? args.result : NULL; } -struct linop_scale_args { int N; const long* dims; float re; float im; bartorch_linop* result; }; +struct linop_scale_args { int N; const bart_dim_t* dims; float re; float im; bartorch_linop* result; }; static int linop_scale_worker(void* p) { @@ -649,13 +649,13 @@ static int linop_scale_worker(void* p) } /* The scale is passed as two floats: a complex float is not in the ABI. */ -bartorch_linop* bartorch_linop_scale(int N, const long* dims, float re, float im) +bartorch_linop* bartorch_linop_scale(int N, const int64_t* dims, float re, float im) { struct linop_scale_args args = { N, dims, re, im, NULL }; return (0 == guarded(linop_scale_worker, &args)) ? args.result : NULL; } -struct linop_dims_args { int N; const long* dims; bartorch_linop* result; }; +struct linop_dims_args { int N; const bart_dim_t* dims; bartorch_linop* result; }; static int linop_zconj_worker(void* p) { @@ -664,7 +664,7 @@ static int linop_zconj_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_zconj(int N, const long* dims) +bartorch_linop* bartorch_linop_zconj(int N, const int64_t* dims) { struct linop_dims_args args = { N, dims, NULL }; return (0 == guarded(linop_zconj_worker, &args)) ? args.result : NULL; @@ -677,13 +677,13 @@ static int linop_identity_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_identity(int N, const long* dims) +bartorch_linop* bartorch_linop_identity(int N, const int64_t* dims) { struct linop_dims_args args = { N, dims, NULL }; return (0 == guarded(linop_identity_worker, &args)) ? args.result : NULL; } -struct linop_null_args { int NO; const long* odims; int NI; const long* idims; bartorch_linop* result; }; +struct linop_null_args { int NO; const bart_dim_t* odims; int NI; const bart_dim_t* idims; bartorch_linop* result; }; static int linop_null_worker(void* p) { @@ -692,7 +692,7 @@ static int linop_null_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_null(int NO, const long* odims, int NI, const long* idims) +bartorch_linop* bartorch_linop_null(int NO, const int64_t* odims, int NI, const int64_t* idims) { struct linop_null_args args = { NO, odims, NI, idims, NULL }; return (0 == guarded(linop_null_worker, &args)) ? args.result : NULL; @@ -741,7 +741,7 @@ double bartorch_linop_maxeigen(const bartorch_linop* a) * reversed and padded to DIMS by the host, so nothing here reorders anything. */ -struct linop_flagged_args { int N; const long* dims; unsigned long flags; const void* data; bartorch_linop* result; }; +struct linop_flagged_args { int N; const bart_dim_t* dims; bart_flags_t flags; const void* data; bartorch_linop* result; }; static int linop_rdiag_worker(void* p) { @@ -752,13 +752,13 @@ static int linop_rdiag_worker(void* p) /* md_zrmul: real parts by real parts and imaginary by imaginary, which is a * diagonal on each of the two components and not a real-valued diagonal. */ -bartorch_linop* bartorch_linop_rdiag(int N, const long* dims, unsigned long flags, const void* diag) +bartorch_linop* bartorch_linop_rdiag(int N, const int64_t* dims, uint64_t flags, const void* diag) { struct linop_flagged_args a = { N, dims, flags, diag, NULL }; return (0 == guarded(linop_rdiag_worker, &a)) ? a.result : NULL; } -struct linop_matrix_args { int N; const long* odims; const long* idims; const long* mdims; const void* matrix; bartorch_linop* result; }; +struct linop_matrix_args { int N; const bart_dim_t* odims; const bart_dim_t* idims; const bart_dim_t* mdims; const void* matrix; bartorch_linop* result; }; static int linop_matrix_worker(void* p) { @@ -767,13 +767,13 @@ static int linop_matrix_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_matrix(int N, const long* odims, const long* idims, const long* mdims, const void* matrix) +bartorch_linop* bartorch_linop_matrix(int N, const int64_t* odims, const int64_t* idims, const int64_t* mdims, const void* matrix) { struct linop_matrix_args a = { N, odims, idims, mdims, matrix, NULL }; return (0 == guarded(linop_matrix_worker, &a)) ? a.result : NULL; } -struct linop_conv_args { int N; unsigned long flags; int ctype; int cmode; const long* odims; const long* idims; const long* kdims; const void* kernel; bartorch_linop* result; }; +struct linop_conv_args { int N; bart_flags_t flags; int ctype; int cmode; const bart_dim_t* odims; const bart_dim_t* idims; const bart_dim_t* kdims; const void* kernel; bartorch_linop* result; }; static int linop_conv_worker(void* p) { @@ -783,14 +783,14 @@ static int linop_conv_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_conv(int N, unsigned long flags, int ctype, int cmode, - const long* odims, const long* idims, const long* kdims, const void* kernel) +bartorch_linop* bartorch_linop_conv(int N, uint64_t flags, int ctype, int cmode, + const int64_t* odims, const int64_t* idims, const int64_t* kdims, const void* kernel) { struct linop_conv_args a = { N, flags, ctype, cmode, odims, idims, kdims, kernel, NULL }; return (0 == guarded(linop_conv_worker, &a)) ? a.result : NULL; } -struct linop_grad_args { int N; const long* dims; int d; unsigned long flags; bartorch_linop* result; }; +struct linop_grad_args { int N; const bart_dim_t* dims; int d; bart_flags_t flags; bartorch_linop* result; }; static int linop_grad_worker(void* p) { @@ -801,7 +801,7 @@ static int linop_grad_worker(void* p) /* The finite differences along the flagged axes, stacked along d, which must * be a dimension the input has only one of. */ -bartorch_linop* bartorch_linop_grad(int N, const long* dims, int d, unsigned long flags) +bartorch_linop* bartorch_linop_grad(int N, const int64_t* dims, int d, uint64_t flags) { struct linop_grad_args a = { N, dims, d, flags, NULL }; return (0 == guarded(linop_grad_worker, &a)) ? a.result : NULL; @@ -814,9 +814,9 @@ static int linop_zreal_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_zreal(int N, const long* dims) +bartorch_linop* bartorch_linop_zreal(int N, const int64_t* dims) { - struct linop_flagged_args a = { N, dims, 0UL, NULL, NULL }; + struct linop_flagged_args a = { N, dims, 0, NULL, NULL }; return (0 == guarded(linop_zreal_worker, &a)) ? a.result : NULL; } @@ -828,7 +828,7 @@ static int linop_sum_worker(void* p) } /* The one BART operator with a closed-form pseudo-inverse. */ -bartorch_linop* bartorch_linop_sum(int N, const long* dims, unsigned long flags) +bartorch_linop* bartorch_linop_sum(int N, const int64_t* dims, uint64_t flags) { struct linop_flagged_args a = { N, dims, flags, NULL, NULL }; return (0 == guarded(linop_sum_worker, &a)) ? a.result : NULL; @@ -844,7 +844,7 @@ static int linop_scaled_sum_worker(void* p) /* The sum divided by the square root of how many were summed, so that its * normal is an orthogonal projection. That is the operator BART's closed-form * pseudo-inverse is written for. */ -bartorch_linop* bartorch_linop_scaled_sum(int N, const long* dims, unsigned long flags) +bartorch_linop* bartorch_linop_scaled_sum(int N, const int64_t* dims, uint64_t flags) { struct linop_flagged_args a = { N, dims, flags, NULL, NULL }; return (0 == guarded(linop_scaled_sum_worker, &a)) ? a.result : NULL; @@ -857,7 +857,7 @@ static int linop_avg_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_avg(int N, const long* dims, unsigned long flags) +bartorch_linop* bartorch_linop_avg(int N, const int64_t* dims, uint64_t flags) { struct linop_flagged_args a = { N, dims, flags, NULL, NULL }; return (0 == guarded(linop_avg_worker, &a)) ? a.result : NULL; @@ -871,13 +871,13 @@ static int linop_repmat_worker(void* p) } /* dims is the codomain: the domain is it with the flagged axes set to one. */ -bartorch_linop* bartorch_linop_repmat(int N, const long* odims, unsigned long flags) +bartorch_linop* bartorch_linop_repmat(int N, const int64_t* odims, uint64_t flags) { struct linop_flagged_args a = { N, odims, flags, NULL, NULL }; return (0 == guarded(linop_repmat_worker, &a)) ? a.result : NULL; } -struct linop_hankel_args { int N; const long* dims; int dim; int window_dim; int window; bartorch_linop* result; }; +struct linop_hankel_args { int N; const bart_dim_t* dims; int dim; int window_dim; int window; bartorch_linop* result; }; static int linop_hankel_worker(void* p) { @@ -889,7 +889,7 @@ static int linop_hankel_worker(void* p) /* A sliding window along dim, laid out along window_dim -- which the input * must have only one of. BART builds it as a strided view, so nothing is * copied to make the windows overlap. */ -bartorch_linop* bartorch_linop_hankel(int N, const long* dims, int dim, int window_dim, int window) +bartorch_linop* bartorch_linop_hankel(int N, const int64_t* dims, int dim, int window_dim, int window) { struct linop_hankel_args a = { N, dims, dim, window_dim, window, NULL }; return (0 == guarded(linop_hankel_worker, &a)) ? a.result : NULL; @@ -902,13 +902,13 @@ static int linop_flip_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_flip(int N, const long* dims, unsigned long flags) +bartorch_linop* bartorch_linop_flip(int N, const int64_t* dims, uint64_t flags) { struct linop_flagged_args a = { N, dims, flags, NULL, NULL }; return (0 == guarded(linop_flip_worker, &a)) ? a.result : NULL; } -struct linop_two_shapes_args { int NO; const long* odims; int NI; const long* idims; const long* pos; bartorch_linop* result; }; +struct linop_two_shapes_args { int NO; const bart_dim_t* odims; int NI; const bart_dim_t* idims; const bart_dim_t* pos; bartorch_linop* result; }; static int linop_reshape_worker(void* p) { @@ -917,7 +917,7 @@ static int linop_reshape_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_reshape(int NO, const long* odims, int NI, const long* idims) +bartorch_linop* bartorch_linop_reshape(int NO, const int64_t* odims, int NI, const int64_t* idims) { struct linop_two_shapes_args a = { NO, odims, NI, idims, NULL, NULL }; return (0 == guarded(linop_reshape_worker, &a)) ? a.result : NULL; @@ -932,7 +932,7 @@ static int linop_resize_worker(void* p) /* Centred: what BART's `resize -c` does, cropping or zero-filling about the * middle of each axis rather than the corner. */ -bartorch_linop* bartorch_linop_resize(int N, const long* odims, const long* idims) +bartorch_linop* bartorch_linop_resize(int N, const int64_t* odims, const int64_t* idims) { struct linop_two_shapes_args a = { N, odims, N, idims, NULL, NULL }; return (0 == guarded(linop_resize_worker, &a)) ? a.result : NULL; @@ -945,13 +945,13 @@ static int linop_extract_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_extract(int N, const long* pos, const long* odims, const long* idims) +bartorch_linop* bartorch_linop_extract(int N, const int64_t* pos, const int64_t* odims, const int64_t* idims) { struct linop_two_shapes_args a = { N, odims, N, idims, pos, NULL }; return (0 == guarded(linop_extract_worker, &a)) ? a.result : NULL; } -struct linop_transpose_args { int N; int a; int b; const long* dims; bartorch_linop* result; }; +struct linop_transpose_args { int N; int a; int b; const bart_dim_t* dims; bartorch_linop* result; }; static int linop_transpose_worker(void* p) { @@ -960,13 +960,13 @@ static int linop_transpose_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_transpose(int N, int a, int b, const long* dims) +bartorch_linop* bartorch_linop_transpose(int N, int a, int b, const int64_t* dims) { struct linop_transpose_args t = { N, a, b, dims, NULL }; return (0 == guarded(linop_transpose_worker, &t)) ? t.result : NULL; } -struct linop_permute_args { int N; const int* order; const long* idims; bartorch_linop* result; }; +struct linop_permute_args { int N; const int* order; const bart_dim_t* idims; bartorch_linop* result; }; static int linop_permute_worker(void* p) { @@ -976,13 +976,13 @@ static int linop_permute_worker(void* p) } /* order is BART's: the output's dimension i is the input's order[i]. */ -bartorch_linop* bartorch_linop_permute(int N, const int* order, const long* idims) +bartorch_linop* bartorch_linop_permute(int N, const int* order, const int64_t* idims) { struct linop_permute_args a = { N, order, idims, NULL }; return (0 == guarded(linop_permute_worker, &a)) ? a.result : NULL; } -struct linop_shift_args { int N; const long* dims; int dim; long shift; int pad; bartorch_linop* result; }; +struct linop_shift_args { int N; const bart_dim_t* dims; int dim; bart_dim_t shift; int pad; bartorch_linop* result; }; static int linop_shift_worker(void* p) { @@ -991,21 +991,21 @@ static int linop_shift_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_shift(int N, const long* dims, int dim, long shift, int pad) +bartorch_linop* bartorch_linop_shift(int N, const int64_t* dims, int dim, int64_t shift, int pad) { struct linop_shift_args a = { N, dims, dim, shift, pad, NULL }; return (0 == guarded(linop_shift_worker, &a)) ? a.result : NULL; } -struct linop_padding_args { int N; const long* dims; int pad; const long* before; const long* after; bartorch_linop* result; }; +struct linop_padding_args { int N; const bart_dim_t* dims; int pad; const bart_dim_t* before; const bart_dim_t* after; bartorch_linop* result; }; static int linop_padding_worker(void* p) { struct linop_padding_args* a = p; /* BART takes these non-const and does not write through them. */ - long before[DIMS]; - long after[DIMS]; + bart_dim_t before[DIMS]; + bart_dim_t after[DIMS]; md_copy_dims(a->N, before, a->before); md_copy_dims(a->N, after, a->after); @@ -1013,7 +1013,7 @@ static int linop_padding_worker(void* p) return 0; } -bartorch_linop* bartorch_linop_padding(int N, const long* dims, int pad, const long* before, const long* after) +bartorch_linop* bartorch_linop_padding(int N, const int64_t* dims, int pad, const int64_t* before, const int64_t* after) { struct linop_padding_args a = { N, dims, pad, before, after, NULL }; return (0 == guarded(linop_padding_worker, &a)) ? a.result : NULL; @@ -1052,14 +1052,14 @@ int bartorch_linop_pseudo_inv(const bartorch_linop* h, float lambda, void* dst, return guarded(linop_pinv_worker, &args); } -int bartorch_linop_domain(const bartorch_linop* h, int N, long* dims) +int bartorch_linop_domain(const bartorch_linop* h, int N, int64_t* dims) { const struct iovec_s* iov = linop_domain(h->op); copy_dims(iov, N, dims); return iov->N; } -int bartorch_linop_codomain(const bartorch_linop* h, int N, long* dims) +int bartorch_linop_codomain(const bartorch_linop* h, int N, int64_t* dims) { const struct iovec_s* iov = linop_codomain(h->op); copy_dims(iov, N, dims); @@ -1161,7 +1161,7 @@ static void cb_nl_del(const nlop_data_t* _d) struct nlop_callback_args { - int ON; const long* odims; int IN; const long* idims; + int ON; const bart_dim_t* odims; int IN; const bart_dim_t* idims; bartorch_apply_fn forward; bartorch_apply_fn derivative; bartorch_apply_fn adjoint; void* ctx; bartorch_release_fn release; bartorch_nlop* result; @@ -1236,8 +1236,8 @@ static void cbg_del(const nlop_data_t* _d) struct nlop_callback_generic_args { - int OO; int ON; const long* odims; - int II; int IN; const long* idims; + int OO; int ON; const bart_dim_t* odims; + int II; int IN; const bart_dim_t* idims; bartorch_generic_apply_fn forward; bartorch_pair_apply_fn derivative; bartorch_pair_apply_fn adjoint; @@ -1261,8 +1261,8 @@ static int nlop_callback_generic_worker(void* p) /* The shapes arrive flat, one argument after another; BART wants them * as arrays of arrays, which is the same memory read differently. */ - const long (*od)[a->ON] = (const long (*)[a->ON])a->odims; - const long (*id)[a->IN] = (const long (*)[a->IN])a->idims; + const bart_dim_t (*od)[a->ON] = (const bart_dim_t (*)[a->ON])a->odims; + const bart_dim_t (*id)[a->IN] = (const bart_dim_t (*)[a->IN])a->idims; /* One derivative and one adjoint for every (input, output) pair. The * two shims below dispatch on the pair themselves, so every entry is @@ -1285,8 +1285,8 @@ static int nlop_callback_generic_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const long* odims, - int II, int IN, const long* idims, +bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const int64_t* odims, + int II, int IN, const int64_t* idims, bartorch_generic_apply_fn forward, bartorch_pair_apply_fn derivative, bartorch_pair_apply_fn adjoint, @@ -1304,7 +1304,7 @@ bartorch_nlop* bartorch_nlop_callback_generic(int OO, int ON, const long* odims, return (0 == guarded(nlop_callback_generic_worker, &args)) ? args.result : NULL; } -bartorch_nlop* bartorch_nlop_callback(int ON, const long* odims, int IN, const long* idims, +bartorch_nlop* bartorch_nlop_callback(int ON, const int64_t* odims, int IN, const int64_t* idims, bartorch_apply_fn forward, bartorch_apply_fn derivative, bartorch_apply_fn adjoint, void* ctx, bartorch_release_fn release) { @@ -1343,14 +1343,14 @@ bartorch_nlop* bartorch_nlop_chain(const bartorch_nlop* a, const bartorch_nlop* return (0 == guarded(nlop_chain_worker, &args)) ? args.result : NULL; } -int bartorch_nlop_domain(const bartorch_nlop* h, int N, long* dims) +int bartorch_nlop_domain(const bartorch_nlop* h, int N, int64_t* dims) { const struct iovec_s* iov = nlop_domain(h->op); copy_dims(iov, N, dims); return iov->N; } -int bartorch_nlop_codomain(const bartorch_nlop* h, int N, long* dims) +int bartorch_nlop_codomain(const bartorch_nlop* h, int N, int64_t* dims) { const struct iovec_s* iov = nlop_codomain(h->op); copy_dims(iov, N, dims); @@ -1366,7 +1366,7 @@ static int nlop_apply_worker(void* p) const struct iovec_s* cod = nlop_codomain(a->h->op); switch (a->mode) { - case 0: nlop_generic_apply_select_derivative_unchecked(a->h->op, 2, (void*[]){ a->dst, (void*)a->src }, ~0UL, ~0UL); break; + case 0: nlop_generic_apply_select_derivative_unchecked(a->h->op, 2, (void*[]){ a->dst, (void*)a->src }, ~UINT64_C(0), ~UINT64_C(0)); break; case 1: nlop_derivative(a->h->op, cod->N, cod->dims, a->dst, dom->N, dom->dims, a->src); break; default: nlop_adjoint(a->h->op, dom->N, dom->dims, a->dst, cod->N, cod->dims, a->src); break; } @@ -1412,7 +1412,7 @@ int bartorch_nlop_outputs(const bartorch_nlop* h) return (NULL == h) ? -1 : nlop_get_nr_out_args(h->op); } -int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, long* dims) +int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, int64_t* dims) { if ((NULL == h) || (NULL == dims) || (0 > i) || (i >= nlop_get_nr_in_args(h->op))) return -1; @@ -1427,7 +1427,7 @@ int bartorch_nlop_input_domain(const bartorch_nlop* h, int i, int N, long* dims) return iov->N; } -int bartorch_nlop_output_codomain(const bartorch_nlop* h, int o, int N, long* dims) +int bartorch_nlop_output_codomain(const bartorch_nlop* h, int o, int N, int64_t* dims) { if ((NULL == h) || (NULL == dims) || (0 > o) || (o >= nlop_get_nr_out_args(h->op))) return -1; @@ -1463,7 +1463,7 @@ struct nlop_generic_args { const bartorch_nlop* h; int nargs; void** args; }; static int nlop_generic_worker(void* p) { struct nlop_generic_args* a = p; - nlop_generic_apply_select_derivative_unchecked(a->h->op, a->nargs, a->args, ~0UL, ~0UL); + nlop_generic_apply_select_derivative_unchecked(a->h->op, a->nargs, a->args, ~UINT64_C(0), ~UINT64_C(0)); return 0; } @@ -1517,7 +1517,7 @@ bartorch_linop* bartorch_nlop_derivative_linop(const bartorch_nlop* h, int o, in * operators side by side, an output tied back to an input, two inputs made * one, and the reorderings that make those usable. */ -struct nlop_reshape_args { const bartorch_nlop* a; int at; int N; const long* dims; int output; bartorch_nlop* result; }; +struct nlop_reshape_args { const bartorch_nlop* a; int at; int N; const bart_dim_t* dims; int output; bartorch_nlop* result; }; static int nlop_reshape_worker(void* p) { @@ -1532,7 +1532,7 @@ static int nlop_reshape_worker(void* p) return 0; } -static bartorch_nlop* nlop_reshape(const bartorch_nlop* a, int at, int N, const long* dims, int output) +static bartorch_nlop* nlop_reshape(const bartorch_nlop* a, int at, int N, const bart_dim_t* dims, int output) { if ((NULL == a) || (NULL == dims) || (0 >= N)) return NULL; @@ -1542,12 +1542,12 @@ static bartorch_nlop* nlop_reshape(const bartorch_nlop* a, int at, int N, const return (0 == guarded(nlop_reshape_worker, &args)) ? args.result : NULL; } -bartorch_nlop* bartorch_nlop_reshape_in(const bartorch_nlop* a, int i, int N, const long* dims) +bartorch_nlop* bartorch_nlop_reshape_in(const bartorch_nlop* a, int i, int N, const int64_t* dims) { return nlop_reshape(a, i, N, dims, 0); } -bartorch_nlop* bartorch_nlop_reshape_out(const bartorch_nlop* a, int o, int N, const long* dims) +bartorch_nlop* bartorch_nlop_reshape_out(const bartorch_nlop* a, int o, int N, const int64_t* dims) { return nlop_reshape(a, o, N, dims, 1); } @@ -1700,7 +1700,7 @@ static int nlop_flatten_worker(void* p) * checks the rank it was given against the operator's own -- so a * flattened model handed to a solver asserts unless the vector is * restated the way the rest of the library states it. */ - long dims[DIMS]; + bart_dim_t dims[DIMS]; md_singleton_dims(DIMS, dims); dims[0] = md_calc_size(nlop_generic_domain(op, 0)->N, nlop_generic_domain(op, 0)->dims); @@ -1757,12 +1757,12 @@ bartorch_nlop* bartorch_nlop_del_out(const bartorch_nlop* x, int o) struct nlop_dims_args { int N; - const long* dims; - const long* dims2; - const long* dims3; + const bart_dim_t* dims; + const bart_dim_t* dims2; + const bart_dim_t* dims3; float a; float b; - unsigned long flags; + bart_flags_t flags; bartorch_nlop* result; }; @@ -1773,12 +1773,12 @@ static int nlop_tenmul_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_tenmul(int N, const long* odims, const long* idims1, const long* idims2) +bartorch_nlop* bartorch_nlop_tenmul(int N, const int64_t* odims, const int64_t* idims1, const int64_t* idims2) { if ((NULL == odims) || (NULL == idims1) || (NULL == idims2)) return NULL; - struct nlop_dims_args v = { N, odims, idims1, idims2, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, odims, idims1, idims2, 0., 0., 0, NULL }; return (0 == guarded(nlop_tenmul_worker, &v)) ? v.result : NULL; } @@ -1792,12 +1792,12 @@ static int nlop_zdiv_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zdiv(int N, const long* dims, float eps) +bartorch_nlop* bartorch_nlop_zdiv(int N, const int64_t* dims, float eps) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0, NULL }; return (0 == guarded(nlop_zdiv_worker, &v)) ? v.result : NULL; } @@ -1809,12 +1809,12 @@ static int nlop_zaxpbz_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zaxpbz(int N, const long* dims, float a, float b) +bartorch_nlop* bartorch_nlop_zaxpbz(int N, const int64_t* dims, float a, float b) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, a, b, ~0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, a, b, ~UINT64_C(0), NULL }; return (0 == guarded(nlop_zaxpbz_worker, &v)) ? v.result : NULL; } @@ -1826,12 +1826,12 @@ static int nlop_zexp_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zexp(int N, const long* dims) +bartorch_nlop* bartorch_nlop_zexp(int N, const int64_t* dims) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0, NULL }; return (0 == guarded(nlop_zexp_worker, &v)) ? v.result : NULL; } @@ -1843,12 +1843,12 @@ static int nlop_zlog_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zlog(int N, const long* dims) +bartorch_nlop* bartorch_nlop_zlog(int N, const int64_t* dims) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0, NULL }; return (0 == guarded(nlop_zlog_worker, &v)) ? v.result : NULL; } @@ -1862,12 +1862,12 @@ static int nlop_zinv_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zinv(int N, const long* dims, float eps) +bartorch_nlop* bartorch_nlop_zinv(int N, const int64_t* dims, float eps) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0, NULL }; return (0 == guarded(nlop_zinv_worker, &v)) ? v.result : NULL; } @@ -1879,12 +1879,12 @@ static int nlop_zsqrt_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zsqrt(int N, const long* dims) +bartorch_nlop* bartorch_nlop_zsqrt(int N, const int64_t* dims) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0, NULL }; return (0 == guarded(nlop_zsqrt_worker, &v)) ? v.result : NULL; } @@ -1896,12 +1896,12 @@ static int nlop_zspow_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zspow(int N, const long* dims, float re, float im) +bartorch_nlop* bartorch_nlop_zspow(int N, const int64_t* dims, float re, float im) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, re, im, 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, re, im, 0, NULL }; return (0 == guarded(nlop_zspow_worker, &v)) ? v.result : NULL; } @@ -1913,12 +1913,12 @@ static int nlop_zsadd_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zsadd(int N, const long* dims, float re, float im) +bartorch_nlop* bartorch_nlop_zsadd(int N, const int64_t* dims, float re, float im) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, re, im, 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, re, im, 0, NULL }; return (0 == guarded(nlop_zsadd_worker, &v)) ? v.result : NULL; } @@ -1930,12 +1930,12 @@ static int nlop_zabs_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zabs(int N, const long* dims) +bartorch_nlop* bartorch_nlop_zabs(int N, const int64_t* dims) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, 0., 0., 0, NULL }; return (0 == guarded(nlop_zabs_worker, &v)) ? v.result : NULL; } @@ -1947,12 +1947,12 @@ static int nlop_smo_abs_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_smo_abs(int N, const long* dims, float eps) +bartorch_nlop* bartorch_nlop_smo_abs(int N, const int64_t* dims, float eps) { if (NULL == dims) return NULL; - struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0ul, NULL }; + struct nlop_dims_args v = { N, dims, NULL, NULL, eps, 0., 0, NULL }; return (0 == guarded(nlop_smo_abs_worker, &v)) ? v.result : NULL; } @@ -1966,7 +1966,7 @@ static int nlop_zrss_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zrss(int N, const long* dims, unsigned long flags, float eps) +bartorch_nlop* bartorch_nlop_zrss(int N, const int64_t* dims, uint64_t flags, float eps) { if (NULL == dims) return NULL; @@ -1983,7 +1983,7 @@ static int nlop_zss_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_zss(int N, const long* dims, unsigned long flags) +bartorch_nlop* bartorch_nlop_zss(int N, const int64_t* dims, uint64_t flags) { if (NULL == dims) return NULL; @@ -1998,7 +1998,7 @@ bartorch_nlop* bartorch_nlop_zss(int N, const long* dims, unsigned long flags) * `nlop_const_create` copies, so the caller's buffer is free after the call. * Combined and linked, this is how an input is pinned to a value. */ -struct nlop_const_args { int N; const long* dims; const void* val; bartorch_nlop* result; }; +struct nlop_const_args { int N; const bart_dim_t* dims; const void* val; bartorch_nlop* result; }; static int nlop_const_worker(void* p) { @@ -2007,7 +2007,7 @@ static int nlop_const_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_const(int N, const long* dims, const void* val) +bartorch_nlop* bartorch_nlop_const(int N, const int64_t* dims, const void* val) { if ((NULL == dims) || (NULL == val)) return NULL; @@ -2017,7 +2017,7 @@ bartorch_nlop* bartorch_nlop_const(int N, const long* dims, const void* val) return (0 == guarded(nlop_const_worker, &v)) ? v.result : NULL; } -struct nlop_set_const_args { const bartorch_nlop* a; int i; int N; const long* dims; const void* val; bartorch_nlop* result; }; +struct nlop_set_const_args { const bartorch_nlop* a; int i; int N; const bart_dim_t* dims; const void* val; bartorch_nlop* result; }; static int nlop_set_input_const_worker(void* p) { @@ -2026,7 +2026,7 @@ static int nlop_set_input_const_worker(void* p) return 0; } -bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int N, const long* dims, const void* val) +bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int N, const int64_t* dims, const void* val) { if ((NULL == a) || (NULL == dims) || (NULL == val) || (0 > i) || (i >= nlop_get_nr_in_args(a->op))) @@ -2038,7 +2038,7 @@ bartorch_nlop* bartorch_nlop_set_input_const(const bartorch_nlop* a, int i, int } -struct nlop_norm_inv_args { const bartorch_nlop* normal; int maxiter; float tol; float l2lambda; long batch; bartorch_nlop* result; }; +struct nlop_norm_inv_args { const bartorch_nlop* normal; int maxiter; float tol; float l2lambda; bart_dim_t batch; bartorch_nlop* result; }; static int nlop_norm_inv_worker(void* p) { @@ -2062,12 +2062,12 @@ static int nlop_norm_inv_worker(void* p) /* Every axis of the vector takes its own weight, which is what * `model_net.c` asks for and what lets `lambda` arrive as a tensor. */ - v->result = wrap_nlop(norm_inv_lambda_create(&conf, v->normal->op, ~0UL)); + v->result = wrap_nlop(norm_inv_lambda_create(&conf, v->normal->op, ~UINT64_C(0))); return 0; } -bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* normal, int maxiter, float tol, float l2lambda, long batch) +bartorch_nlop* bartorch_nlop_norm_inv_lambda(const bartorch_nlop* normal, int maxiter, float tol, float l2lambda, int64_t batch) { if ((NULL == normal) || (1 > batch)) return NULL; @@ -2114,20 +2114,20 @@ void bartorch_nlop_free(bartorch_nlop* h) struct noir_create_args { int N; - const long* ksp_dims; - const long* cim_dims; - const long* img_dims; - const long* kco_dims; - const long* col_dims; - const long* pat_dims; + const bart_dim_t* ksp_dims; + const bart_dim_t* cim_dims; + const bart_dim_t* img_dims; + const bart_dim_t* kco_dims; + const bart_dim_t* col_dims; + const bart_dim_t* pat_dims; const void* pattern; - const long* trj_dims; + const bart_dim_t* trj_dims; const void* traj; - const long* wgh_dims; + const bart_dim_t* wgh_dims; const void* weights; - const long* bas_dims; + const bart_dim_t* bas_dims; const void* basis; - const long* msk_dims; + const bart_dim_t* msk_dims; const void* mask; int noncart; int optimized; @@ -2174,15 +2174,15 @@ static int noir_create_worker(void* p) } bartorch_noir* bartorch_noir_create(int N, - const long* ksp_dims, const long* cim_dims, const long* img_dims, - const long* kco_dims, const long* col_dims, - const long* pat_dims, const void* pattern, - const long* trj_dims, const void* traj, - const long* wgh_dims, const void* weights, - const long* bas_dims, const void* basis, - const long* msk_dims, const void* mask, + const int64_t* ksp_dims, const int64_t* cim_dims, const int64_t* img_dims, + const int64_t* kco_dims, const int64_t* col_dims, + const int64_t* pat_dims, const void* pattern, + const int64_t* trj_dims, const void* traj, + const int64_t* wgh_dims, const void* weights, + const int64_t* bas_dims, const void* basis, + const int64_t* msk_dims, const void* mask, int noncart, int optimized, int toeplitz, - unsigned long fft_flags, unsigned long wght_flags, + uint64_t fft_flags, uint64_t wght_flags, int rvc, int sos, float a, float b, float c, float oversampling_coils, int ret_os_coils) { @@ -2288,12 +2288,12 @@ bartorch_linop* bartorch_noir_transform(const bartorch_noir* h) return noir_linop(h, 3); } -int bartorch_noir_dims(const bartorch_noir* h, int which, int N, long* dims) +int bartorch_noir_dims(const bartorch_noir* h, int which, int N, int64_t* dims) { if ((NULL == h) || (NULL == dims) || (N < h->model.N)) return -1; - const long* src = NULL; + const bart_dim_t* src = NULL; switch (which) { case 0: src = h->model.ksp_dims; break; @@ -2357,8 +2357,8 @@ static int irgnm_worker(void* p) const struct iovec_s* dom = nlop_domain(a->F->op); const struct iovec_s* cod = nlop_codomain(a->F->op); - long N = 2 * md_calc_size(dom->N, dom->dims); - long M = 2 * md_calc_size(cod->N, cod->dims); + bart_dim_t N = 2 * md_calc_size(dom->N, dom->dims); + bart_dim_t M = 2 * md_calc_size(cod->N, cod->dims); iter4_irgnm(CAST_UP(&conf), a->F->op, N, a->x, a->xref, M, a->y, NULL, (struct iter_op_s){ NULL, NULL }); return 0; @@ -2404,8 +2404,8 @@ static int irgnm2_worker(void* p) const struct iovec_s* dom = nlop_domain(a->F->op); const struct iovec_s* cod = nlop_codomain(a->F->op); - long N = 2 * md_calc_size(dom->N, dom->dims); - long M = 2 * md_calc_size(cod->N, cod->dims); + bart_dim_t N = 2 * md_calc_size(dom->N, dom->dims); + bart_dim_t M = 2 * md_calc_size(cod->N, cod->dims); iter4_irgnm2(CAST_UP(&conf), a->F->op, N, a->x, a->xref, M, a->y, NULL, (struct iter_op_s){ NULL, NULL }); diff --git a/src/csrc/ops/paired.cu b/src/csrc/ops/paired.cu index 2fcb277d..4c98b5f1 100644 --- a/src/csrc/ops/paired.cu +++ b/src/csrc/ops/paired.cu @@ -196,7 +196,7 @@ struct XArgs { const unsigned* mask; const int* prefix; const cplx* tx; /* 2 x N: each set's phase along x */ - long L; + bart_dim_t L; }; /* The pass along x of both sets of the pair: per line of four coefficients, @@ -307,7 +307,7 @@ __global__ void __launch_bounds__(Shape::XF::max_threads_per_block) fused(XAr if (word & bit) { - const long j = a.prefix[g >> 5] + __popc(word & (bit - 1)); + const bart_dim_t j = a.prefix[g >> 5] + __popc(word & (bit - 1)); #pragma unroll for (unsigned r = 0; r < R; r++) @@ -316,7 +316,7 @@ __global__ void __launch_bounds__(Shape::XF::max_threads_per_block) fused(XAr const unsigned lo = (r < c) ? r : c; const unsigned hi = (r < c) ? c : r; - const float m = widen(static_cast(a.psf[sx])[(long)(lo + hi * (hi + 1) / 2) * a.L + j]); + const float m = widen(static_cast(a.psf[sx])[(bart_dim_t)(lo + hi * (hi + 1) / 2) * a.L + j]); o[r].x += m * v[c].x; o[r].y += m * v[c].y; @@ -377,7 +377,7 @@ struct Call { int bf16; const unsigned* mask; const int* prefix; - long L; + bart_dim_t L; const cplx* tab; /* the pair's tables: x of each set, y, z */ float scale; }; @@ -496,7 +496,7 @@ const Entry table[] = { BARTORCH_PAIRED_SIZES(BARTORCH_PAIRED_ENTRY) }; /* A set's phase along one axis, as `phase_setup` in coset.cuh builds it -- * the shift, the centring, the fftmod folded in -- in double precision. */ -void axis_phase(long d, float shift, cplx* out) +void axis_phase(bart_dim_t d, float shift, cplx* out) { double s = shift; @@ -506,13 +506,13 @@ void axis_phase(long d, float shift, cplx* out) double slope = 2. * M_PI * s / d; double offset = -slope * d / 2.; - long centre = d / 2; + bart_dim_t centre = d / 2; double half = (double)centre / d; slope += 2. * M_PI * half; offset -= 2. * M_PI * half * centre / 2.; - for (long i = 0; i < d; i++) { + for (bart_dim_t i = 0; i < d; i++) { double v = offset + i * slope; out[i].x = (float)cos(v); @@ -533,7 +533,7 @@ struct bartorch_paired { /* The pair kernels for a grid of `dims`, with the sets' shifts as * `bartorch_psf_shift` gives them (set i and i + 1, i even, differ only along * x); NULL where there are none or the card cannot run them. */ -extern "C" struct bartorch_paired* bartorch_paired_create(const long dims[3], int coeffs, int sets, const float (*shifts)[3]) +extern "C" struct bartorch_paired* bartorch_paired_create(const bart_dim_t dims[3], int coeffs, int sets, const float (*shifts)[3]) { if ((R != (unsigned)coeffs) || (8 != sets) || (dims[0] != dims[1]) || (dims[0] != dims[2])) return NULL; @@ -541,7 +541,7 @@ extern "C" struct bartorch_paired* bartorch_paired_create(const long dims[3], in const Entry* entry = NULL; for (const Entry& e : table) - if ((long)e.n == dims[0]) + if ((bart_dim_t)e.n == dims[0]) entry = &e; if ((NULL == entry) || (0 != entry->prepare())) @@ -551,7 +551,7 @@ extern "C" struct bartorch_paired* bartorch_paired_create(const long dims[3], in if ((shifts[i][1] != shifts[i + 1][1]) || (shifts[i][2] != shifts[i + 1][2])) return NULL; - const long n = dims[0]; + const bart_dim_t n = dims[0]; const int pairs = sets / 2; cplx* host = (cplx*)xmalloc(sizeof(cplx) * 4 * n * pairs); @@ -583,7 +583,7 @@ extern "C" struct bartorch_paired* bartorch_paired_create(const long dims[3], in cudaMemcpy(p->tables, host, sizeof(cplx) * 4 * n * pairs, cudaMemcpyHostToDevice); xfree(host); - debug_printf(DP_DEBUG1, "bartorch: paired kernels for %ld^3\n", n); + debug_printf(DP_DEBUG1, "bartorch: paired kernels for %" PRId64 "^3\n", n); return p; } @@ -623,7 +623,7 @@ extern "C" void bartorch_paired_in(const struct bartorch_paired* p, int k, } extern "C" void bartorch_paired_fused(const struct bartorch_paired* p, int k, _Complex float* scratch, - const void* psf0, const void* psf1, int bf16, const unsigned int* mask, const int* prefix, long L) + const void* psf0, const void* psf1, int bf16, const unsigned int* mask, const int* prefix, bart_dim_t L) { Call c; diff --git a/src/csrc/ops/sense.c b/src/csrc/ops/sense.c index d5e91f43..58c16322 100644 --- a/src/csrc/ops/sense.c +++ b/src/csrc/ops/sense.c @@ -67,7 +67,7 @@ extern void bartorch_nufft_coset_normal(const struct linop_s* op, complex float* extern int bartorch_nufft_coset_folds(const struct linop_s* op); extern void bartorch_nufft_coset_normal_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, int last); + const bart_stride_t map_strs[], const complex float* map, int last); extern void bartorch_nufft_coset_end(const struct linop_s* op); /* Provided by grid.c: a Cartesian transform's normal, run through cuFFT's @@ -75,27 +75,27 @@ extern void bartorch_nufft_coset_end(const struct linop_s* op); extern int bartorch_grid_folds(const struct linop_s* op, const void* ref); extern int bartorch_grid_folds_samples(const struct linop_s* op, const void* ref); extern void bartorch_grid_forward_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map); + const bart_stride_t map_strs[DIMS], const complex float* map); extern void bartorch_grid_adjoint_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map); + const bart_stride_t map_strs[DIMS], const complex float* map); extern void bartorch_grid_normal_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[DIMS], const complex float* map); + const bart_stride_t map_strs[DIMS], const complex float* map); #ifdef USE_CUDA /* csrc/kernels.cu: a volume times BART's inverse fftmod along its first three axes. */ -extern void bartorch_cuda_modulate(const long dims[3], long rest, const long grid[3], const long off[3], +extern void bartorch_cuda_modulate(const bart_dim_t dims[3], bart_dim_t rest, const bart_dim_t grid[3], const bart_stride_t off[3], float scale, complex float* x); #endif -extern struct linop_s* bart_sense_init(unsigned long shared_img_flags, const long max_dims[DIMS], - unsigned long sens_flags, const complex float* sens); +extern struct linop_s* bart_sense_init(bart_flags_t shared_img_flags, const bart_dim_t max_dims[DIMS], + bart_flags_t sens_flags, const complex float* sens); -extern const struct linop_s* bart_sense_nc_init(const long max_dims[DIMS], const long map_dims[DIMS], const complex float* maps, - const long ksp_dims[DIMS], - const long traj_dims[DIMS], const complex float* traj, const struct nufft_conf_s* conf, - const long wgs_dims[DIMS], const complex float* weights, - const long basis_dims[DIMS], const complex float* basis, - const struct linop_s** fft_opp, unsigned long shared_img_dims); +extern const struct linop_s* bart_sense_nc_init(const bart_dim_t max_dims[DIMS], const bart_dim_t map_dims[DIMS], const complex float* maps, + const bart_dim_t ksp_dims[DIMS], + const bart_dim_t traj_dims[DIMS], const complex float* traj, const struct nufft_conf_s* conf, + const bart_dim_t wgs_dims[DIMS], const complex float* weights, + const bart_dim_t basis_dims[DIMS], const complex float* basis, + const struct linop_s** fft_opp, bart_flags_t shared_img_dims); /* How many coils a slab holds. Zero leaves the operators as BART builds * them, every coil at once, which is the fastest and the largest. */ @@ -119,7 +119,7 @@ int bartorch_sense_fold_maps(void) * (bartorch_encoding_count). bartorch_sense_counter is the first three of * them under the slab loop's own name. */ enum { SN_COUNTS = BARTORCH_ENCODING_ITEMS + 1 }; -static long sense_counters[SN_COUNTS]; +static bart_dim_t sense_counters[SN_COUNTS]; static void counted(int which) { @@ -137,7 +137,7 @@ int bartorch_sense_coil_batch(void) return coil_batch; } -long bartorch_encoding_counter(int which) +int64_t bartorch_encoding_counter(int which) { return ((0 <= which) && (which < SN_COUNTS)) ? sense_counters[which] : 0; } @@ -148,7 +148,7 @@ void bartorch_encoding_reset_counters(void) sense_counters[i] = 0; } -long bartorch_sense_counter(int which) +int64_t bartorch_sense_counter(int which) { return bartorch_encoding_counter(((BARTORCH_ENCODING_BUILT == which) || (BARTORCH_ENCODING_FOLDED == which)) ? which : BARTORCH_ENCODING_CHAINED); @@ -163,37 +163,37 @@ struct sense_s { linop_data_t super; - long batch; /* coils in a slab */ + bart_dim_t batch; /* coils in a slab */ bool fold; /* apply the maps inside the transform of the normal */ - long coils; /* coils in all */ + bart_dim_t coils; /* coils in all */ /* One slab: the dimensions the sensitivities contract over, the coil * images they produce, and what the transform answers. */ - long slab_dims[DIMS]; - long cim_dims[DIMS]; - long out_dims[DIMS]; - long img_dims[DIMS]; - - long map_dims[DIMS]; - long slab_map_strs[DIMS]; /* a slab of maps, densely */ - long cim_strs[DIMS]; - long out_strs[DIMS]; - long img_strs[DIMS]; - long map_strs[DIMS]; + bart_dim_t slab_dims[DIMS]; + bart_dim_t cim_dims[DIMS]; + bart_dim_t out_dims[DIMS]; + bart_dim_t img_dims[DIMS]; + + bart_dim_t map_dims[DIMS]; + bart_stride_t slab_map_strs[DIMS]; /* a slab of maps, densely */ + bart_stride_t cim_strs[DIMS]; + bart_stride_t out_strs[DIMS]; + bart_stride_t img_strs[DIMS]; + bart_stride_t map_strs[DIMS]; /* The whole of what the caller sees, and the stride that steps a slab * along the coil axis of it and of the sensitivities. */ - long full_out_dims[DIMS]; - long full_out_strs[DIMS]; - long out_slab_offset; - long map_slab_offset; + bart_dim_t full_out_dims[DIMS]; + bart_stride_t full_out_strs[DIMS]; + bart_stride_t out_slab_offset; + bart_stride_t map_slab_offset; /* Where the coils lie in the samples. BART's tools keep them on * COIL_DIM, so a slab is a stride along it; the torch layout puts them * slowest, so the whole is one coil's samples, contiguous, one block * after another. */ bool coils_slowest; - long block_dims[DIMS]; + bart_dim_t block_dims[DIMS]; /* The sensitivities, and whether they are ours to free: the Cartesian * operator scales and modulates a copy, the non-Cartesian one reads @@ -208,9 +208,9 @@ struct sense_s { * unitary spectrum back to the grid it was taken on needs no scaling, * so what comes back is the map band-limited and nothing else. */ const complex float* kernels; - long kern_dims[DIMS]; - long kern_strs[DIMS]; - long kern_slab_offset; + bart_dim_t kern_dims[DIMS]; + bart_stride_t kern_strs[DIMS]; + bart_stride_t kern_slab_offset; /* The transform for one slab: a Fourier transform on a grid, a NUFFT * off one. */ @@ -226,43 +226,43 @@ struct sense_s { * its blocks, and the planes the slab's transform works on are gathered * out of a coil image transformed along z and put back into one. The * normal then takes the transform along z too. NULL is every position. */ - long stack_count; - long* stack_positions; - long plane_elems; - long sub_dims[DIMS]; + bart_dim_t stack_count; + bart_dim_t* stack_positions; + bart_dim_t plane_elems; + bart_dim_t sub_dims[DIMS]; /* A contraction whose image weight differs between sets: each term's * image weight goes on before the sensitivities contract the sets and its * sample weight after the transform, all inside the slab. Zero terms * leaves any contraction around the slab instead. */ - long terms; + bart_dim_t terms; struct multiplace_array_s* term_image; struct multiplace_array_s* term_sample; - long term_image_strs[DIMS]; - long term_sample_strs[DIMS]; - long term_image_step; - long term_sample_step; - long term_image_item_step; /* how far an item steps into a term's weights, or 0 */ - long term_sample_item_step; + bart_stride_t term_image_strs[DIMS]; + bart_stride_t term_sample_strs[DIMS]; + bart_dim_t term_image_step; + bart_dim_t term_sample_step; + bart_dim_t term_image_item_step; /* how far an item steps into a term's weights, or 0 */ + bart_dim_t term_sample_item_step; /* Items that each have a trajectory of their own: `slab` is the first of * `item_slabs`, which is NULL with one item. `img_dims` and a coil's * samples are one item's, and the whole is the items one after another, * so an item is an offset into both. */ - long items; + bart_dim_t items; const struct linop_s** item_slabs; - long item_image_step; - long item_sample_step; - long whole_img_dims[DIMS]; + bart_dim_t item_image_step; + bart_dim_t item_sample_step; + bart_dim_t whole_img_dims[DIMS]; }; static DEF_TYPEID(sense_s); /* Where a slab starts, as an index into an array laid out over every coil. * Strides are in bytes and this indexes complex floats. */ -static long slab_at(long stride, long coil) +static bart_dim_t slab_at(bart_stride_t stride, bart_dim_t coil) { - return coil * stride / (long)CFL_SIZE; + return coil * stride / (bart_stride_t)CFL_SIZE; } /* Whether a slab has to be brought to where the arithmetic is. @@ -288,9 +288,9 @@ static bool assembled(const struct sense_s* d, const void* ref) * Kernels are laid out, padded back on to the image grid and transformed, * which is the map they were taken from with everything above the kernel's * own band removed. A bank on the host is copied across as it stands. */ -static void fetch_slab(const struct sense_s* d, long coil, complex float* into) +static void fetch_slab(const struct sense_s* d, bart_dim_t coil, complex float* into) { - long mdims[DIMS]; + bart_dim_t mdims[DIMS]; md_copy_dims(DIMS, mdims, d->map_dims); mdims[COIL_DIM] = d->batch; @@ -301,11 +301,11 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) return; } - long kdims[DIMS]; + bart_dim_t kdims[DIMS]; md_copy_dims(DIMS, kdims, d->kern_dims); kdims[COIL_DIM] = d->batch; - long kstrs[DIMS]; + bart_stride_t kstrs[DIMS]; md_calc_strides(DIMS, kstrs, kdims, CFL_SIZE); complex float* k = md_alloc_sameplace(DIMS, kdims, CFL_SIZE, into); @@ -320,7 +320,7 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) * planes through it, and only the third is the whole grid. A centred * unitary transform is one transform per axis whichever way it is taken, * so this is the same map. */ - long sdims[DIMS]; + bart_dim_t sdims[DIMS]; md_copy_dims(DIMS, sdims, kdims); /* Each axis's centred unitary transform is a modulation, a transform, the @@ -331,19 +331,19 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) * map at the end -- where the last axis alone would otherwise make three * passes over the whole grid. */ bool modulated = false; - long grid[3]; - long off[3]; + bart_dim_t grid[3]; + bart_stride_t off[3]; float scale = 1.f; #ifdef USE_CUDA - modulated = cuda_ondevice(into) && (md_calc_size(3, mdims) < (1L << 31)); + modulated = cuda_ondevice(into) && (md_calc_size(3, mdims) < (INT64_C(1) << 31)); for (int a = 0; a < 3; a++) { bool fft = MD_IS_SET(FFT_FLAGS, a) && (1 < mdims[a]); grid[a] = fft ? mdims[a] : 1; - off[a] = labs(mdims[a] / 2 - kdims[a] / 2); + off[a] = llabs(mdims[a] / 2 - kdims[a] / 2); if (fft) scale /= sqrtf((float)mdims[a]); @@ -359,7 +359,7 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) for (int a = 0; a < 3; a++) { - long ndims[DIMS]; + bart_dim_t ndims[DIMS]; md_copy_dims(DIMS, ndims, sdims); ndims[a] = mdims[a]; @@ -380,12 +380,12 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) md_copy_dims(DIMS, sdims, ndims); } - assert(md_check_equal_dims(DIMS, sdims, mdims, ~0UL)); + assert(md_check_equal_dims(DIMS, sdims, mdims, ~UINT64_C(0))); #ifdef USE_CUDA if (modulated) { - long zero[3] = { 0, 0, 0 }; + bart_dim_t zero[3] = { 0, 0, 0 }; bartorch_cuda_modulate(mdims, md_calc_size(DIMS - 3, mdims + 3), grid, zero, scale, into); } @@ -399,7 +399,7 @@ static void fetch_slab(const struct sense_s* d, long coil, complex float* into) /* Somewhere to put a slab, when one has to be put together. */ static complex float* slab_buffer(const struct sense_s* d, const void* ref) { - long mdims[DIMS]; + bart_dim_t mdims[DIMS]; md_copy_dims(DIMS, mdims, d->map_dims); mdims[COIL_DIM] = d->batch; @@ -418,8 +418,8 @@ static void stream_wait(void) { } #endif /* What one slab does, whichever way the operator is being applied. */ -typedef void (*slab_fn)(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* ctx); +typedef void (*slab_fn)(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* ctx); /* Slab buffers kept from one walk over the coils to the next. * @@ -430,7 +430,7 @@ typedef void (*slab_fn)(const struct sense_s* d, long coil, const complex float* struct slab_walk { complex float* buf[2]; - long holds[2]; /* the first coil each buffer holds, or -1 */ + bart_dim_t holds[2]; /* the first coil each buffer holds, or -1 */ bool reverse; }; @@ -450,7 +450,7 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo { if (!assembled(d, ref)) { - for (long c = 0; c < d->coils; c += d->batch) + for (bart_dim_t c = 0; c < d->coils; c += d->batch) fn(d, c, d->maps + slab_at(d->map_slab_offset, c), d->map_strs, c + d->batch >= d->coils, ctx); return; @@ -462,7 +462,7 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo if (NULL == w->buf[0]) w->buf[0] = slab_buffer(d, ref); - const long* mstrs = d->slab_map_strs; + const bart_stride_t* mstrs = d->slab_map_strs; bool overlap = (1 < stream_count()) && (d->batch < d->coils); @@ -471,10 +471,10 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo overlap = overlap && (NULL != w->buf[1]); - long slabs = (d->coils + d->batch - 1) / d->batch; - long order[slabs]; + bart_dim_t slabs = (d->coils + d->batch - 1) / d->batch; + bart_dim_t order[slabs]; - for (long t = 0; t < slabs; t++) + for (bart_dim_t t = 0; t < slabs; t++) order[t] = (w->reverse ? slabs - 1 - t : t) * d->batch; /* The buffer the first slab is in, if the walk before left it there. */ @@ -487,9 +487,9 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo w->holds[0] = order[0]; } - for (long t = 0; t < slabs; t++) { + for (bart_dim_t t = 0; t < slabs; t++) { - long c = order[t]; + bart_dim_t c = order[t]; bool last = (t + 1 == slabs); if (!overlap) { @@ -504,7 +504,7 @@ static void drive_slabs(const struct sense_s* d, const void* ref, slab_fn fn, vo continue; } - long next = last ? -1 : order[t + 1]; + bart_dim_t next = last ? -1 : order[t + 1]; /* Armed here rather than once: BART forgets which level owns * the streams as soon as anything asks for one from below it, @@ -575,14 +575,14 @@ static bool crosses(const void* ptr) return !bartorch_on_device(ptr) && (0 <= bartorch_cuda_device()); } -static complex float* onto_card(const long dims[DIMS], const complex float* ptr, bool filled) +static complex float* onto_card(const bart_dim_t dims[DIMS], const complex float* ptr, bool filled) { #ifdef USE_CUDA if (crosses(ptr)) { complex float* on = md_alloc_gpu(DIMS, dims, CFL_SIZE); - if (filled && (0 != bartorch_cuda_copy_pageable(on, ptr, md_calc_size(DIMS, dims) * (long)CFL_SIZE))) + if (filled && (0 != bartorch_cuda_copy_pageable(on, ptr, md_calc_size(DIMS, dims) * (bart_stride_t)CFL_SIZE))) md_copy(DIMS, dims, on, ptr, CFL_SIZE); return on; @@ -593,12 +593,12 @@ static complex float* onto_card(const long dims[DIMS], const complex float* ptr, return (complex float*)ptr; } -static void off_card(const long dims[DIMS], complex float* ptr, complex float* on, bool filled) +static void off_card(const bart_dim_t dims[DIMS], complex float* ptr, complex float* on, bool filled) { if (on == ptr) return; - if (filled && (0 != bartorch_cuda_copy_pageable(ptr, on, md_calc_size(DIMS, dims) * (long)CFL_SIZE))) + if (filled && (0 != bartorch_cuda_copy_pageable(ptr, on, md_calc_size(DIMS, dims) * (bart_stride_t)CFL_SIZE))) md_copy(DIMS, dims, ptr, on, CFL_SIZE); md_free(on); @@ -619,10 +619,10 @@ struct slab_ctx { /* Where there are several items: the one a slab function works on, what * each does, and how far an item steps the source and the destination. */ - long item; + bart_dim_t item; slab_fn fn; - long src_step; - long dst_step; + bart_dim_t src_step; + bart_dim_t dst_step; }; /* The transform an item's slab function applies. */ @@ -636,38 +636,38 @@ static const struct linop_s* slab_of(const struct sense_s* d, const struct slab_ * coefficients -- steps over the planes, a position after another. */ static void planes_take(const struct sense_s* d, complex float* sub, const complex float* cim) { - long z = d->cim_dims[PHS2_DIM]; - long outer = md_calc_size(DIMS, d->cim_dims) / (z * d->plane_elems); + bart_dim_t z = d->cim_dims[PHS2_DIM]; + bart_dim_t outer = md_calc_size(DIMS, d->cim_dims) / (z * d->plane_elems); - for (long o = 0; o < outer; o++) - for (long j = 0; j < d->stack_count; j++) + for (bart_dim_t o = 0; o < outer; o++) + for (bart_dim_t j = 0; j < d->stack_count; j++) md_copy(1, MD_DIMS(d->plane_elems), sub + (o * d->stack_count + j) * d->plane_elems, cim + (o * z + d->stack_positions[j]) * d->plane_elems, CFL_SIZE); } static void planes_put(const struct sense_s* d, complex float* cim, const complex float* sub) { - long z = d->cim_dims[PHS2_DIM]; - long outer = md_calc_size(DIMS, d->cim_dims) / (z * d->plane_elems); + bart_dim_t z = d->cim_dims[PHS2_DIM]; + bart_dim_t outer = md_calc_size(DIMS, d->cim_dims) / (z * d->plane_elems); md_clear(DIMS, d->cim_dims, cim, CFL_SIZE); - for (long o = 0; o < outer; o++) - for (long j = 0; j < d->stack_count; j++) + for (bart_dim_t o = 0; o < outer; o++) + for (bart_dim_t j = 0; j < d->stack_count; j++) md_copy(1, MD_DIMS(d->plane_elems), cim + (o * z + d->stack_positions[j]) * d->plane_elems, sub + (o * d->stack_count + j) * d->plane_elems, CFL_SIZE); } /* Each item in turn under one slab of coils. */ -static void each_item(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void each_item(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { struct slab_ctx* c = _c; complex float* dst = c->dst; const complex float* src = c->src; - for (long t = 0; t < d->items; t++) { + for (bart_dim_t t = 0; t < d->items; t++) { c->item = t; c->dst = dst + t * c->dst_step; @@ -682,7 +682,7 @@ static void each_item(const struct sense_s* d, long coil, const complex float* m } /* A slab's samples into their place in the whole, and back out of it. */ -static void put_samples(const struct sense_s* d, long coil, complex float* dst, const complex float* out) +static void put_samples(const struct sense_s* d, bart_dim_t coil, complex float* dst, const complex float* out) { if (!d->coils_slowest) { @@ -695,15 +695,15 @@ static void put_samples(const struct sense_s* d, long coil, complex float* dst, * The destination is walked by the whole's own strides rather than by a * block's, so an axis the whole carries above the coils lands where it * belongs instead of inside the block. */ - long coil_step = d->out_strs[COIL_DIM] / (long)CFL_SIZE; + bart_dim_t coil_step = d->out_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE; - for (long b = 0; b < d->batch; b++) + for (bart_dim_t b = 0; b < d->batch; b++) md_copy2(DIMS, d->block_dims, d->full_out_strs, dst + slab_at(d->out_slab_offset, coil + b), d->out_strs, out + b * coil_step, CFL_SIZE); } -static void take_samples(const struct sense_s* d, long coil, complex float* out, const complex float* src) +static void take_samples(const struct sense_s* d, bart_dim_t coil, complex float* out, const complex float* src) { if (!d->coils_slowest) { @@ -712,15 +712,15 @@ static void take_samples(const struct sense_s* d, long coil, complex float* out, return; } - long coil_step = d->out_strs[COIL_DIM] / (long)CFL_SIZE; + bart_dim_t coil_step = d->out_strs[COIL_DIM] / (bart_stride_t)CFL_SIZE; - for (long b = 0; b < d->batch; b++) + for (bart_dim_t b = 0; b < d->batch; b++) md_copy2(DIMS, d->block_dims, d->out_strs, out + b * coil_step, d->full_out_strs, src + slab_at(d->out_slab_offset, coil + b), CFL_SIZE); } -static void forward_slab(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void forward_slab(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -743,8 +743,8 @@ static void forward_slab(const struct sense_s* d, long coil, const complex float put_samples(d, coil, c->dst, c->out); } -static void adjoint_slab(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void adjoint_slab(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -767,8 +767,8 @@ static void adjoint_slab(const struct sense_s* d, long coil, const complex float /* The same for a sampled-only Cartesian transform on a card: the sensitivity * goes on as a coefficient is read into the transform and comes off as the * adjoint writes into the image, so no coil image is made. */ -static void forward_slab_gridded(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void forward_slab_gridded(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -778,8 +778,8 @@ static void forward_slab_gridded(const struct sense_s* d, long coil, const compl put_samples(d, coil, c->dst, c->out); } -static void adjoint_slab_gridded(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void adjoint_slab_gridded(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -789,8 +789,8 @@ static void adjoint_slab_gridded(const struct sense_s* d, long coil, const compl bartorch_grid_adjoint_sense(d->slab, c->dst, c->out, mstrs, map); } -static void normal_slab(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; (void)last; @@ -821,8 +821,8 @@ static void normal_slab(const struct sense_s* d, long coil, const complex float* * off into the answer. Clearing what the convolution accumulates into is a * pass over the coil images, which is what walking the sets outside costs -- * against the whole function crossing again, which is what it saves. */ -static void normal_slab_coset(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab_coset(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; struct slab_ctx* c = _c; @@ -854,8 +854,8 @@ static void normal_slab_coset(const struct sense_s* d, long coil, const complex * image the map would have been multiplied into nor the one the answer would * have landed in is ever made. At 256^3 over four coefficients each of those * is half a gigabyte. */ -static void normal_slab_folded(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab_folded(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; struct slab_ctx* c = _c; @@ -866,8 +866,8 @@ static void normal_slab_folded(const struct sense_s* d, long coil, const complex /* The same for a Cartesian transform: the sensitivity goes on as a * coefficient is read into the transform and comes off as it is written * into the answer, so here too no coil image is made. */ -static void normal_slab_gridded(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab_gridded(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; (void)last; @@ -879,14 +879,14 @@ static void normal_slab_gridded(const struct sense_s* d, long coil, const comple /* The terms of a contraction the slab loop applies: the image weight before the * sensitivities, the sample weight after the transform, the terms' samples * added up in `acc`. */ -static void terms_forward(const struct sense_s* d, const complex float* map, const long* mstrs, struct slab_ctx* c) +static void terms_forward(const struct sense_s* d, const complex float* map, const bart_stride_t* mstrs, struct slab_ctx* c) { const complex float* image = multiplace_read(d->term_image, c->src); const complex float* sample = multiplace_read(d->term_sample, c->src); md_clear(DIMS, d->out_dims, c->acc, CFL_SIZE); - for (long l = 0; l < d->terms; l++) { + for (bart_dim_t l = 0; l < d->terms; l++) { md_zmul2(DIMS, d->img_dims, d->img_strs, c->img, d->img_strs, c->src, d->term_image_strs, image + l * d->term_image_step + c->item * d->term_image_item_step); @@ -902,13 +902,13 @@ static void terms_forward(const struct sense_s* d, const complex float* map, con } } -static void terms_adjoint(const struct sense_s* d, const complex float* map, const long* mstrs, +static void terms_adjoint(const struct sense_s* d, const complex float* map, const bart_stride_t* mstrs, struct slab_ctx* c, const complex float* samples) { const complex float* image = multiplace_read(d->term_image, c->dst); const complex float* sample = multiplace_read(d->term_sample, c->dst); - for (long l = 0; l < d->terms; l++) { + for (bart_dim_t l = 0; l < d->terms; l++) { md_zmulc2(DIMS, d->out_dims, d->out_strs, c->out, d->out_strs, samples, d->term_sample_strs, sample + l * d->term_sample_step + c->item * d->term_sample_item_step); @@ -923,8 +923,8 @@ static void terms_adjoint(const struct sense_s* d, const complex float* map, con } } -static void forward_slab_terms(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void forward_slab_terms(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -933,8 +933,8 @@ static void forward_slab_terms(const struct sense_s* d, long coil, const complex put_samples(d, coil, c->dst, c->acc); } -static void adjoint_slab_terms(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void adjoint_slab_terms(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)last; struct slab_ctx* c = _c; @@ -943,8 +943,8 @@ static void adjoint_slab_terms(const struct sense_s* d, long coil, const complex terms_adjoint(d, map, mstrs, c, c->acc); } -static void normal_slab_terms(const struct sense_s* d, long coil, const complex float* map, - const long* mstrs, bool last, void* _c) +static void normal_slab_terms(const struct sense_s* d, bart_dim_t coil, const complex float* map, + const bart_stride_t* mstrs, bool last, void* _c) { (void)coil; (void)last; @@ -1010,7 +1010,7 @@ static void sense_adjoint(const linop_data_t* _d, complex float* dst, const comp complex float* dst_on = onto_card(d->whole_img_dims, dst, false); /* The caller's pages are faulted in while the card works (cuda.c). */ - void* faulting = (dst_on != dst) ? bartorch_host_prefault_begin(dst, md_calc_size(DIMS, d->whole_img_dims) * (long)CFL_SIZE) : NULL; + void* faulting = (dst_on != dst) ? bartorch_host_prefault_begin(dst, md_calc_size(DIMS, d->whole_img_dims) * (bart_stride_t)CFL_SIZE) : NULL; bool terms = (0 != d->terms); bool gridded = !terms && d->fold && (1 == d->slab_dims[MAPS_DIM]) @@ -1088,7 +1088,7 @@ static void sense_normal(const linop_data_t* _d, complex float* dst, const compl && (0 != bartorch_grid_folds(d->slab, dst_on)); /* The caller's pages are faulted in while the card works (cuda.c). */ - void* faulting = (dst_on != dst) ? bartorch_host_prefault_begin(dst, md_calc_size(DIMS, d->whole_img_dims) * (long)CFL_SIZE) : NULL; + void* faulting = (dst_on != dst) ? bartorch_host_prefault_begin(dst, md_calc_size(DIMS, d->whole_img_dims) * (bart_stride_t)CFL_SIZE) : NULL; struct slab_ctx c = { @@ -1134,7 +1134,7 @@ static void sense_normal(const linop_data_t* _d, complex float* dst, const compl * the item rather than once for every coil. */ struct slab_walk walk = { { NULL, NULL }, { -1, -1 }, false }; - for (long t = 0; t < d->items; t++) { + for (bart_dim_t t = 0; t < d->items; t++) { const struct linop_s* slab = (NULL == d->item_slabs) ? d->slab : d->item_slabs[t]; @@ -1195,7 +1195,7 @@ static void sense_del(const linop_data_t* _d) } else { - for (long t = 0; t < d->items; t++) + for (bart_dim_t t = 0; t < d->items; t++) linop_free(d->item_slabs[t]); xfree(d->item_slabs); @@ -1213,10 +1213,10 @@ static void sense_del(const linop_data_t* _d) * They can when nothing else is laid out along them: a pattern or a basis * that varies across coils would have to be sliced with them, and the * sensitivities have to carry the coils the images do. */ -static bool sliceable(long batch, const long max_dims[DIMS], const long map_dims[DIMS], const long out_dims[DIMS], - unsigned long shared_img_flags) +static bool sliceable(bart_dim_t batch, const bart_dim_t max_dims[DIMS], const bart_dim_t map_dims[DIMS], const bart_dim_t out_dims[DIMS], + bart_flags_t shared_img_flags) { - long coils = max_dims[COIL_DIM]; + bart_dim_t coils = max_dims[COIL_DIM]; if ((0 == batch) || (coils < 2)) return false; @@ -1241,9 +1241,9 @@ static bool sliceable(long batch, const long max_dims[DIMS], const long map_dims * one asked for -- the one asked for whenever it divides the coils, and one * when nothing else does. */ -static long slab_size(long coils, long want) +static bart_dim_t slab_size(bart_dim_t coils, bart_dim_t want) { - for (long n = MIN(want, coils); n > 1; n--) + for (bart_dim_t n = MIN(want, coils); n > 1; n--) if (0 == coils % n) return n; @@ -1253,8 +1253,8 @@ static long slab_size(long coils, long want) /* The parts of the operator that do not depend on which transform it is. * `batch` and `fold` are the form's, so that nothing about one build is read * from the settings BART's own tools leave behind. */ -static struct sense_s* sense_slabs(long batch, bool fold, const long max_dims[DIMS], const long map_dims[DIMS], - const long out_dims[DIMS], unsigned long shared_img_flags, bool keep_sets) +static struct sense_s* sense_slabs(bart_dim_t batch, bool fold, const bart_dim_t max_dims[DIMS], const bart_dim_t map_dims[DIMS], + const bart_dim_t out_dims[DIMS], bart_flags_t shared_img_flags, bool keep_sets) { PTR_ALLOC(struct sense_s, d); SET_TYPEID(sense_s, d); @@ -1286,7 +1286,7 @@ static struct sense_s* sense_slabs(long batch, bool fold, const long max_dims[DI * something after the transform needs them: a k-space factor that * differs between sets is summed over on the far side instead, so the * coil images keep them and the transform runs once per set. */ - md_select_dims(DIMS, keep_sets ? ~0UL : ~MAPS_FLAG, d->cim_dims, d->slab_dims); + md_select_dims(DIMS, keep_sets ? ~UINT64_C(0) : ~MAPS_FLAG, d->cim_dims, d->slab_dims); md_select_dims(DIMS, ~COIL_FLAG & ~shared_img_flags, d->img_dims, max_dims); md_calc_strides(DIMS, d->cim_strs, d->cim_dims, CFL_SIZE); @@ -1303,7 +1303,7 @@ static struct sense_s* sense_slabs(long batch, bool fold, const long max_dims[DI md_calc_strides(DIMS, d->map_strs, map_dims, CFL_SIZE); d->map_slab_offset = d->map_strs[COIL_DIM]; - long slab_map_dims[DIMS]; + bart_dim_t slab_map_dims[DIMS]; md_copy_dims(DIMS, slab_map_dims, map_dims); slab_map_dims[COIL_DIM] = d->batch; md_calc_strides(DIMS, d->slab_map_strs, slab_map_dims, CFL_SIZE); @@ -1322,7 +1322,7 @@ static struct sense_s* sense_slabs(long batch, bool fold, const long max_dims[DI * than around it because one bank cannot serve every item. They are still * part of what a slab answers, so they stay in the block; what they must not * do is push the coils above them. */ -static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned long outer, const long* item_dims) +static void sense_output_from(struct sense_s* d, bool coils_slowest, bart_flags_t outer, const bart_dim_t* item_dims) { auto cod = linop_codomain(d->slab); @@ -1344,11 +1344,11 @@ static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned lo d->block_dims[COIL_DIM] = 1; /* A block is one item's; a coil holds the items one after another. */ - long whole_dims[DIMS]; + bart_dim_t whole_dims[DIMS]; md_copy_dims(DIMS, whole_dims, d->block_dims); if (NULL != item_dims) - md_max_dims(DIMS, ~0UL, whole_dims, whole_dims, item_dims); + md_max_dims(DIMS, ~UINT64_C(0), whole_dims, whole_dims, item_dims); d->item_sample_step = md_calc_size(DIMS, d->block_dims); @@ -1357,7 +1357,7 @@ static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned lo * where a block has nothing beyond it, a later one where encoding axes * lie there, and never above an outer axis, which the torch layout puts * slower than the coils. */ - long inner_dims[DIMS]; + bart_dim_t inner_dims[DIMS]; md_select_dims(DIMS, ~outer, inner_dims, whole_dims); int last = DIMS - 1; @@ -1372,7 +1372,7 @@ static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned lo /* The coils were placed above every inner axis; an outer one that is not * above them too would be read in the wrong order rather than refused. */ - unsigned long at_or_below = (1UL << (coil_axis + 1)) - 1UL; + bart_flags_t at_or_below = (UINT64_C(1) << (coil_axis + 1)) - 1; if (0 != (outer & md_nontriv_dims(DIMS, whole_dims) & at_or_below)) error("bartorch: a batch the sensitivities vary along lies slower than the coils, " @@ -1386,7 +1386,7 @@ static void sense_output_from(struct sense_s* d, bool coils_slowest, unsigned lo static struct linop_s* sense_operator(struct sense_s* d) { - debug_printf(DP_DEBUG1, "SENSE over %ld coils, %ld at a time\n", d->coils, d->batch); + debug_printf(DP_DEBUG1, "SENSE over %" PRId64 " coils, %" PRId64 " at a time\n", d->coils, d->batch); counted(BARTORCH_ENCODING_BUILT); @@ -1400,22 +1400,22 @@ static void chained(void) } /* y = F S x, on a grid. */ -struct linop_s* sense_init(unsigned long shared_img_flags, const long max_dims[DIMS], - unsigned long sens_flags, const complex float* sens) +struct linop_s* sense_init(bart_flags_t shared_img_flags, const bart_dim_t max_dims[DIMS], + bart_flags_t sens_flags, const complex float* sens) { - long map_dims[DIMS]; - long ksp_dims[DIMS]; + bart_dim_t map_dims[DIMS]; + bart_dim_t ksp_dims[DIMS]; md_select_dims(DIMS, sens_flags, map_dims, max_dims); md_select_dims(DIMS, ~MAPS_FLAG, ksp_dims, max_dims); - if (!sliceable((long)coil_batch, max_dims, map_dims, ksp_dims, shared_img_flags)) { + if (!sliceable((bart_dim_t)coil_batch, max_dims, map_dims, ksp_dims, shared_img_flags)) { chained(); return bart_sense_init(shared_img_flags, max_dims, sens_flags, sens); } - struct sense_s* d = sense_slabs((long)coil_batch, 0 != fold_maps, max_dims, map_dims, ksp_dims, shared_img_flags, false); + struct sense_s* d = sense_slabs((bart_dim_t)coil_batch, 0 != fold_maps, max_dims, map_dims, ksp_dims, shared_img_flags, false); /* The scaling and the modulation `maps_create` folds into the * sensitivities, kept here because the loop reads them many times. */ @@ -1424,29 +1424,29 @@ struct linop_s* sense_init(unsigned long shared_img_flags, const long max_dims[D fftmod(DIMS, map_dims, FFT_FLAGS, d->owned, d->owned); d->maps = d->owned; - long slab_ksp_dims[DIMS]; + bart_dim_t slab_ksp_dims[DIMS]; md_copy_dims(DIMS, slab_ksp_dims, ksp_dims); slab_ksp_dims[COIL_DIM] = d->batch; d->slab = linop_fft_create(DIMS, slab_ksp_dims, FFT_FLAGS); - sense_output_from(d, false, 0UL, NULL); + sense_output_from(d, false, 0, NULL); return sense_operator(d); } /* y = A S x, off one. */ -const struct linop_s* sense_nc_init(const long max_dims[DIMS], const long map_dims[DIMS], const complex float* maps, - const long ksp_dims[DIMS], - const long traj_dims[DIMS], const complex float* traj, const struct nufft_conf_s* _conf, - const long wgs_dims[DIMS], const complex float* weights, - const long basis_dims[DIMS], const complex float* basis, - const struct linop_s** fft_opp, unsigned long shared_img_dims) +const struct linop_s* sense_nc_init(const bart_dim_t max_dims[DIMS], const bart_dim_t map_dims[DIMS], const complex float* maps, + const bart_dim_t ksp_dims[DIMS], + const bart_dim_t traj_dims[DIMS], const complex float* traj, const struct nufft_conf_s* _conf, + const bart_dim_t wgs_dims[DIMS], const complex float* weights, + const bart_dim_t basis_dims[DIMS], const complex float* basis, + const struct linop_s** fft_opp, bart_flags_t shared_img_dims) { - long ksp_dims2[DIMS]; + bart_dim_t ksp_dims2[DIMS]; md_copy_dims(DIMS, ksp_dims2, ksp_dims); ksp_dims2[COEFF_DIM] = max_dims[COEFF_DIM]; - bool sliced = sliceable((long)coil_batch, max_dims, map_dims, ksp_dims2, shared_img_dims) + bool sliced = sliceable((bart_dim_t)coil_batch, max_dims, map_dims, ksp_dims2, shared_img_dims) && ((NULL == weights) || (1 == wgs_dims[COIL_DIM])) && ((NULL == basis) || (1 == basis_dims[COIL_DIM])); @@ -1457,9 +1457,9 @@ const struct linop_s* sense_nc_init(const long max_dims[DIMS], const long map_di wgs_dims, weights, basis_dims, basis, fft_opp, shared_img_dims); } - struct sense_s* d = sense_slabs((long)coil_batch, 0 != fold_maps, max_dims, map_dims, ksp_dims2, shared_img_dims, false); + struct sense_s* d = sense_slabs((bart_dim_t)coil_batch, 0 != fold_maps, max_dims, map_dims, ksp_dims2, shared_img_dims, false); - long slab_ksp_dims[DIMS]; + bart_dim_t slab_ksp_dims[DIMS]; md_copy_dims(DIMS, slab_ksp_dims, ksp_dims2); slab_ksp_dims[COIL_DIM] = d->batch; @@ -1468,7 +1468,7 @@ const struct linop_s* sense_nc_init(const long max_dims[DIMS], const long map_di (weights ? wgs_dims : NULL), weights, (basis ? basis_dims : NULL), basis, *_conf); - sense_output_from(d, false, 0UL, NULL); + sense_output_from(d, false, 0, NULL); /* The caller reads the point spread function off this and imports one * into it; a slab's transform carries the same one, because a point @@ -1482,7 +1482,7 @@ const struct linop_s* sense_nc_init(const long max_dims[DIMS], const long map_di /* Hold the sensitivities the way the caller has them: as maps the loop reads * where they lie, or as the kernels it inflates a slab at a time. */ -static void sense_hold(struct sense_s* d, const long sens_dims[DIMS], const complex float* sens, int kernels) +static void sense_hold(struct sense_s* d, const bart_dim_t sens_dims[DIMS], const complex float* sens, int kernels) { if (0 == kernels) { @@ -1511,18 +1511,18 @@ static void kernels_need_the_loop(void) * its pattern and basis and the normal that transforms only the axes the * pattern varies along, the same over a table of sampled phase encodes, and * the wave in either arrangement. */ -extern const struct linop_s* grid_transform_create(const long cim_dims[DIMS], - const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz); -extern const struct linop_s* grid_sampled_create(const long cim_dims[DIMS], long T, long S, int components, - const long* positions, const long bas_dims[DIMS], const complex float* basis, +extern const struct linop_s* grid_transform_create(const bart_dim_t cim_dims[DIMS], + const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz); +extern const struct linop_s* grid_sampled_create(const bart_dim_t cim_dims[DIMS], bart_dim_t T, bart_dim_t S, int components, + const bart_dim_t* positions, const bart_dim_t bas_dims[DIMS], const complex float* basis, int kspace_readout, int toeplitz); -extern const struct linop_s* wave_transform_create(const long dom_dims[DIMS], long wx, const complex float* psf, - int centred, const long pat_dims[DIMS], const complex float* pattern, - const long bas_dims[DIMS], const complex float* basis, int toeplitz); -extern const struct linop_s* wave_sampled_create(const long dom_dims[DIMS], long wx, const complex float* psf, - int centred, long T, long S, int components, const long* positions, - const long bas_dims[DIMS], const complex float* basis, int toeplitz); +extern const struct linop_s* wave_transform_create(const bart_dim_t dom_dims[DIMS], bart_dim_t wx, const complex float* psf, + int centred, const bart_dim_t pat_dims[DIMS], const complex float* pattern, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz); +extern const struct linop_s* wave_sampled_create(const bart_dim_t dom_dims[DIMS], bart_dim_t wx, const complex float* psf, + int centred, bart_dim_t T, bart_dim_t S, int components, const bart_dim_t* positions, + const bart_dim_t bas_dims[DIMS], const complex float* basis, int toeplitz); /* `slab` with the form's contraction around it, sum_l diag(b_l) slab * diag(c_l): each term a copy of the weights on the side they multiply, all @@ -1541,15 +1541,15 @@ static const struct linop_s* contracted(const struct bartorch_encoding* f, const const complex float* sample = f->segment_sample; const complex float* image = f->segment_image; - long sample_step = md_calc_size(DIMS, f->segment_sample_dims); - long image_step = md_calc_size(DIMS, f->segment_image_dims); + bart_dim_t sample_step = md_calc_size(DIMS, f->segment_sample_dims); + bart_dim_t image_step = md_calc_size(DIMS, f->segment_image_dims); - unsigned long sample_flags = md_nontriv_dims(DIMS, f->segment_sample_dims); - unsigned long image_flags = md_nontriv_dims(DIMS, f->segment_image_dims); + bart_flags_t sample_flags = md_nontriv_dims(DIMS, f->segment_sample_dims); + bart_flags_t image_flags = md_nontriv_dims(DIMS, f->segment_image_dims); const struct linop_s* sum = NULL; - for (long l = 0; l < f->segments; l++) { + for (bart_dim_t l = 0; l < f->segments; l++) { const struct linop_s* term = linop_chain_FF(linop_chain_FF( linop_cdiag_create(DIMS, dom->dims, image_flags, image + l * image_step), @@ -1584,9 +1584,9 @@ static void hold_terms(struct sense_s* d, const struct bartorch_encoding* f) if (NULL != f->item_dims) { - unsigned long item_flags = md_nontriv_dims(DIMS, f->item_dims); - unsigned long on_image = item_flags & md_nontriv_dims(DIMS, f->segment_image_dims); - unsigned long on_sample = item_flags & md_nontriv_dims(DIMS, f->segment_sample_dims); + bart_flags_t item_flags = md_nontriv_dims(DIMS, f->item_dims); + bart_flags_t on_image = item_flags & md_nontriv_dims(DIMS, f->segment_image_dims); + bart_flags_t on_sample = item_flags & md_nontriv_dims(DIMS, f->segment_sample_dims); if (((0 != on_image) && (item_flags != on_image)) || ((0 != on_sample) && (item_flags != on_sample))) error("bartorch: weights before the sensitivities vary along every item axis or none\n"); @@ -1598,8 +1598,8 @@ static void hold_terms(struct sense_s* d, const struct bartorch_encoding* f) d->term_sample_item_step = d->term_sample_step / md_calc_size(DIMS, f->item_dims); } - long image[1] = { d->terms * d->term_image_step }; - long sample[1] = { d->terms * d->term_sample_step }; + bart_dim_t image[1] = { d->terms * d->term_image_step }; + bart_dim_t sample[1] = { d->terms * d->term_sample_step }; d->term_image = multiplace_move(1, image, CFL_SIZE, f->segment_image); d->term_sample = multiplace_move(1, sample, CFL_SIZE, f->segment_sample); @@ -1633,7 +1633,7 @@ static const struct linop_s* summed_over_sets(const struct bartorch_encoding* f, * it -- the coils, the slab loop, the streaming, the contraction -- is the * same code whichever transform it is. */ -static const struct linop_s* form_transform(const struct bartorch_encoding* f, const long cim_dims[DIMS], +static const struct linop_s* form_transform(const struct bartorch_encoding* f, const bart_dim_t cim_dims[DIMS], const struct nufft_conf_s* conf) { switch (f->transform) { @@ -1688,8 +1688,8 @@ static const struct linop_s* form_transform(const struct bartorch_encoding* f, c * the coils, or one whose coils the loop cannot slice, and the counter says * so rather than leaving it to be guessed from a timing. */ -static const struct linop_s* form_chain(const struct bartorch_encoding* f, const long max_dims[DIMS], - const long map_dims[DIMS], const long cim_dims[DIMS], const struct nufft_conf_s* conf) +static const struct linop_s* form_chain(const struct bartorch_encoding* f, const bart_dim_t max_dims[DIMS], + const bart_dim_t map_dims[DIMS], const bart_dim_t cim_dims[DIMS], const struct nufft_conf_s* conf) { if (0 != f->kernels) kernels_need_the_loop(); @@ -1698,19 +1698,19 @@ static const struct linop_s* form_chain(const struct bartorch_encoding* f, const if (BARTORCH_ENCODING_NUFFT == f->transform) return bart_sense_nc_init(max_dims, map_dims, f->sens, f->ksp_dims, f->traj_dims, f->traj, conf, - f->wgh_dims, f->weights, f->bas_dims, f->basis, NULL, 0UL); + f->wgh_dims, f->weights, f->bas_dims, f->basis, NULL, 0); if (0 != f->modulated) { /* The flags `pics` gives it, so that what comes back is the * operator the tool builds and not one like it -- which is * what a caller asking for this convention is after. */ - unsigned long map_flags = FFT_FLAGS | SENS_FLAGS | md_nontriv_dims(DIMS, f->sens_dims); + bart_flags_t map_flags = FFT_FLAGS | SENS_FLAGS | md_nontriv_dims(DIMS, f->sens_dims); - return bart_sense_init(0UL, max_dims, map_flags, f->sens); + return bart_sense_init(0, max_dims, map_flags, f->sens); } - long img_dims[DIMS]; + bart_dim_t img_dims[DIMS]; md_select_dims(DIMS, ~COIL_FLAG, img_dims, max_dims); struct linop_s* coils = linop_fmac_dims_create(DIMS, cim_dims, img_dims, @@ -1734,7 +1734,7 @@ static const struct linop_s* form_chain(const struct bartorch_encoding* f, const */ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* f) { - const long* max_dims = f->max_dims; + const bart_dim_t* max_dims = f->max_dims; if ((BARTORCH_ENCODING_FFT != f->transform) && (0 != f->modulated)) error("bartorch: the modulated convention is a grid transform's; " @@ -1759,15 +1759,15 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* * of the image's, and the samples carry it too: the operator holds every * item of it rather than being applied once per item, because one bank * does not serve them all. */ - unsigned long outer_flags = (0 <= f->batch_dim) ? MD_BIT(f->batch_dim) : 0UL; + bart_flags_t outer_flags = (0 <= f->batch_dim) ? MD_BIT(f->batch_dim) : 0; - long map_dims[DIMS]; + bart_dim_t map_dims[DIMS]; md_select_dims(DIMS, FFT_FLAGS | COIL_FLAG | MAPS_FLAG | outer_flags, map_dims, max_dims); /* What the transform is asked for. Off a grid the samples are the * form's own, with the coefficients a basis contracts still on them; * on one they are the image's axes without the sets. */ - long out_dims[DIMS]; + bart_dim_t out_dims[DIMS]; if (BARTORCH_ENCODING_NUFFT == f->transform) { @@ -1776,14 +1776,14 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* } else { - md_select_dims(DIMS, (NULL != f->slice) ? ~0UL : ~MAPS_FLAG, out_dims, max_dims); + md_select_dims(DIMS, (NULL != f->slice) ? ~UINT64_C(0) : ~MAPS_FLAG, out_dims, max_dims); } /* Items with a trajectory of their own: the operator is built for one, * and each gets its own transform over its part of the trajectory. */ - long items = 1; - long item_max_dims[DIMS]; - long item_out_dims[DIMS]; + bart_dim_t items = 1; + bart_dim_t item_max_dims[DIMS]; + bart_dim_t item_out_dims[DIMS]; md_copy_dims(DIMS, item_max_dims, max_dims); md_copy_dims(DIMS, item_out_dims, out_dims); @@ -1813,7 +1813,7 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* /* A k-space factor laid out along the coils would have to be sliced * with them, which the loop cannot do. */ - bool sliced = sliceable((long)f->coil_batch, max_dims, map_dims, out_dims, 0UL) + bool sliced = sliceable((bart_dim_t)f->coil_batch, max_dims, map_dims, out_dims, 0) && ((NULL == f->weights) || (1 == f->wgh_dims[COIL_DIM])) && ((NULL == f->basis) || (1 == f->bas_dims[COIL_DIM])); @@ -1826,8 +1826,8 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* return form_chain(f, max_dims, map_dims, out_dims, &conf); } - struct sense_s* d = sense_slabs((long)f->coil_batch, 0 != f->fold_maps, - max_dims, map_dims, out_dims, 0UL, NULL != f->slice); + struct sense_s* d = sense_slabs((bart_dim_t)f->coil_batch, 0 != f->fold_maps, + max_dims, map_dims, out_dims, 0, NULL != f->slice); sense_hold(d, f->sens_dims, f->sens, f->kernels); @@ -1851,7 +1851,7 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* "done to a kernel; ask for the centred convention, or inflate the " "kernels first\n"); - if (!md_check_equal_dims(DIMS, map_dims, f->sens_dims, ~0UL)) + if (!md_check_equal_dims(DIMS, map_dims, f->sens_dims, ~UINT64_C(0))) error("bartorch: the modulation folded into the sensitivities is the grid's, so " "this convention needs a bank on the grid rather than one broadcast " "onto it\n"); @@ -1865,7 +1865,7 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* /* Off a grid the transform is asked for a slab of samples; on one it * works from the slab of coil images and says for itself what comes * back, because a basis contracts its coefficients away. */ - long slab_ksp_dims[DIMS]; + bart_dim_t slab_ksp_dims[DIMS]; md_copy_dims(DIMS, slab_ksp_dims, out_dims); slab_ksp_dims[COIL_DIM] = d->batch; @@ -1874,13 +1874,13 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* * and one position's shots where the shots were. Under a slab of one coil * and no sets that is the same memory as the layout around it, so the * relabelling copies nothing. */ - long slab_cim_dims[DIMS]; + bart_dim_t slab_cim_dims[DIMS]; md_copy_dims(DIMS, slab_cim_dims, d->cim_dims); if (d->stacked) { - long z = d->cim_dims[PHS2_DIM]; - long blocks = (NULL == f->stack_positions) ? z : f->stack_count; + bart_dim_t z = d->cim_dims[PHS2_DIM]; + bart_dim_t blocks = (NULL == f->stack_positions) ? z : f->stack_count; if ((1 != d->batch) || (1 != d->cim_dims[MAPS_DIM]) || (1 != slab_ksp_dims[MAPS_DIM]) || (blocks < 1) || (0 != slab_ksp_dims[PHS2_DIM] % blocks) @@ -1896,13 +1896,13 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* if (NULL != f->stack_positions) { d->stack_count = blocks; - d->stack_positions = xmalloc((size_t)blocks * sizeof(long)); + d->stack_positions = xmalloc((size_t)blocks * sizeof(bart_dim_t)); d->plane_elems = d->cim_dims[READ_DIM] * d->cim_dims[PHS1_DIM]; - for (long j = 0; j < blocks; j++) { + for (bart_dim_t j = 0; j < blocks; j++) { if ((f->stack_positions[j] < 0) || (f->stack_positions[j] >= z)) - error("bartorch: a stack's position %ld is off a z grid of %ld\n", + error("bartorch: a stack's position %" PRId64 " is off a z grid of %" PRId64 "\n", f->stack_positions[j], z); d->stack_positions[j] = f->stack_positions[j]; @@ -1930,13 +1930,13 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* /* An item's part of the trajectory and of the weights is contiguous: * the item axes are the slowest either array has. */ - long trj_dims[DIMS]; - long trj_strs[DIMS]; + bart_dim_t trj_dims[DIMS]; + bart_stride_t trj_strs[DIMS]; md_select_dims(DIMS, ~md_nontriv_dims(DIMS, f->item_dims), trj_dims, f->traj_dims); md_calc_strides(DIMS, trj_strs, f->traj_dims, CFL_SIZE); - long wgh_dims[DIMS]; - long wgh_strs[DIMS]; + bart_dim_t wgh_dims[DIMS]; + bart_stride_t wgh_strs[DIMS]; if (NULL != f->weights) { @@ -1946,12 +1946,12 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* /* So is an item's part of a term's segment weights, laid out after the * term the same way. */ - unsigned long item_flags = md_nontriv_dims(DIMS, f->item_dims); + bart_flags_t item_flags = md_nontriv_dims(DIMS, f->item_dims); - long seg_sample_dims[DIMS]; - long seg_image_dims[DIMS]; - long seg_sample_strs[DIMS]; - long seg_image_strs[DIMS]; + bart_dim_t seg_sample_dims[DIMS]; + bart_dim_t seg_image_dims[DIMS]; + bart_stride_t seg_sample_strs[DIMS]; + bart_stride_t seg_image_strs[DIMS]; if (0 != f->segments) { @@ -1964,10 +1964,10 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* d->item_slabs = xmalloc((size_t)items * sizeof(d->item_slabs[0])); d->item_image_step = md_calc_size(DIMS, d->img_dims); - long pos[DIMS]; + bart_dim_t pos[DIMS]; md_set_dims(DIMS, pos, 0); - for (long t = 0; t < items; t++) { + for (bart_dim_t t = 0; t < items; t++) { struct bartorch_encoding item = slab; @@ -1987,32 +1987,32 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* * them itself. */ if ((0 != f->segments) && !before_maps) { - long step_sample = md_calc_size(DIMS, f->segment_sample_dims); - long step_image = md_calc_size(DIMS, f->segment_image_dims); + bart_dim_t step_sample = md_calc_size(DIMS, f->segment_sample_dims); + bart_dim_t step_image = md_calc_size(DIMS, f->segment_image_dims); item.segment_sample_dims = seg_sample_dims; item.segment_image_dims = seg_image_dims; /* The terms stay contiguous, so an item's weights are copied * out term by term into arrays of their own. */ - long one_sample = md_calc_size(DIMS, seg_sample_dims); - long one_image = md_calc_size(DIMS, seg_image_dims); + bart_dim_t one_sample = md_calc_size(DIMS, seg_sample_dims); + bart_dim_t one_image = md_calc_size(DIMS, seg_image_dims); complex float* sample = md_alloc(1, MD_DIMS(f->segments * one_sample), CFL_SIZE); complex float* image = md_alloc(1, MD_DIMS(f->segments * one_image), CFL_SIZE); - for (long l = 0; l < f->segments; l++) { + for (bart_dim_t l = 0; l < f->segments; l++) { md_copy2(DIMS, seg_sample_dims, MD_STRIDES(DIMS, seg_sample_dims, CFL_SIZE), sample + l * one_sample, seg_sample_strs, (const complex float*)f->segment_sample + l * step_sample - + md_calc_offset(DIMS, seg_sample_strs, pos) / (long)CFL_SIZE, + + md_calc_offset(DIMS, seg_sample_strs, pos) / (bart_stride_t)CFL_SIZE, CFL_SIZE); md_copy2(DIMS, seg_image_dims, MD_STRIDES(DIMS, seg_image_dims, CFL_SIZE), image + l * one_image, seg_image_strs, (const complex float*)f->segment_image + l * step_image - + md_calc_offset(DIMS, seg_image_strs, pos) / (long)CFL_SIZE, + + md_calc_offset(DIMS, seg_image_strs, pos) / (bart_stride_t)CFL_SIZE, CFL_SIZE); } @@ -2027,7 +2027,7 @@ const struct linop_s* bartorch_encoding_operator(const struct bartorch_encoding* d->item_slabs[t] = transform; - md_next(DIMS, f->item_dims, ~0UL, pos); + md_next(DIMS, f->item_dims, ~UINT64_C(0), pos); } d->slab = d->item_slabs[0]; diff --git a/src/csrc/substitute/fft.cpp b/src/csrc/substitute/fft.cpp index e3902a4a..4ab08d2c 100644 --- a/src/csrc/substitute/fft.cpp +++ b/src/csrc/substitute/fft.cpp @@ -23,8 +23,10 @@ * pocketfft's. */ #include +#include #include #include +#include #include #include #include @@ -37,11 +39,26 @@ namespace { -/* DFTI, as much of it as this needs. MKL_LONG is `long` on the platforms - * that have MKL at all, and the two calls that take a variable argument are - * declared that way so the ABI is the one MKL was compiled with. */ +/* DFTI, as much of it as this needs. MKL_LONG is `long` in MKL's LP64 + * interface, which is the one every source of DFTI here exports: 64 bits + * on Linux and macOS, 32 on Windows. So a length, a stride or a distance + * DFTI is handed has to fit it, and a plan with one that does not is + * walked here or left to pocketfft, whose sizes are size_t and ptrdiff_t. + * The two calls that take a variable argument are declared that way so + * the ABI is the one MKL was compiled with. */ typedef long mkl_long; +bool fits_mkl_long(ptrdiff_t v) +{ + return (v >= (ptrdiff_t)std::numeric_limits::min()) + && (v <= (ptrdiff_t)std::numeric_limits::max()); +} + +bool fits_mkl_long(const fftwf_iodim64& d) +{ + return fits_mkl_long(d.n) && fits_mkl_long(d.is) && fits_mkl_long(d.os); +} + enum { DFTI_COMPLEX = 32, DFTI_NUMBER_OF_TRANSFORMS = 7, @@ -73,11 +90,11 @@ bool dfti_ready() && (NULL != dfti.forward) && (NULL != dfti.backward) && (NULL != dfti.release); } -std::atomic g_planned[2]; /* by MKL, by pocketfft */ +std::atomic g_planned[2]; /* by MKL, by pocketfft */ /* One loop dimension, in elements. */ struct loop { - long n, is, os; + ptrdiff_t n, is, os; }; struct plan_s { @@ -90,7 +107,8 @@ struct plan_s { bool forward; /* What DFTI was given, when it took it: the transformed axes and at - * most one loop axis. `outer` is what is left to walk. */ + * most one loop axis. `outer` is what is left to walk. A rank of + * zero is a plan DFTI is not given. */ int rank = 0; std::vector lengths; std::vector strides_in; /* displacement first, then one per length */ @@ -190,7 +208,7 @@ void dfti_walk(const plan_s& p, void* desc, size_t level, const std::complexrank = rank; + bool fits = true; + + for (int i = 0; i < rank; i++) + fits = fits && fits_mkl_long(dims[i]); + + p->rank = fits ? rank : 0; p->strides_in.push_back(0); p->strides_out.push_back(0); - for (int i = 0; i < rank; i++) { + for (int i = 0; i < p->rank; i++) { - p->lengths.push_back(dims[i].n); - p->strides_in.push_back(dims[i].is); - p->strides_out.push_back(dims[i].os); + p->lengths.push_back((mkl_long)dims[i].n); + p->strides_in.push_back((mkl_long)dims[i].is); + p->strides_out.push_back((mkl_long)dims[i].os); } + /* The loop axis DFTI takes has to fit it as well; one that does not + * is walked with the rest. */ int longest = -1; for (int i = 0; i < howmany_rank; i++) - if ((howmany_dims[i].n > 1) && ((longest < 0) || (howmany_dims[i].n > howmany_dims[longest].n))) + if ((howmany_dims[i].n > 1) && fits_mkl_long(howmany_dims[i]) + && ((longest < 0) || (howmany_dims[i].n > howmany_dims[longest].n))) longest = i; if (longest >= 0) diff --git a/src/csrc/substitute/finufft.c b/src/csrc/substitute/finufft.c index 1cf8dd4d..987480e0 100644 --- a/src/csrc/substitute/finufft.c +++ b/src/csrc/substitute/finufft.c @@ -187,7 +187,7 @@ struct bartorch_fi_plan { * them is gone. That is a property worth testing, and one RSS cannot be read * for: FINUFFT's own multithreaded execute retains a kilobyte per thread per * call, which any measurement of the process would drown this in. */ -static long fi_live_plans; +static int64_t fi_live_plans; /* `spread_only` asks FINUFFT for the spreading alone -- no transform, no * deapodisation -- which puts the kernel's own footprint on the grid. That is @@ -253,7 +253,7 @@ int bartorch_finufft_plan(int device, int type, int dim, const int64_t n_modes[3 /* A plan keeps the points by pointer rather than copying them, so the arrays * given here have to outlive it. */ -int bartorch_finufft_setpts(void* plan, long M, float* x, float* y, float* z) +int bartorch_finufft_setpts(void* plan, int64_t M, float* x, float* y, float* z) { const struct bartorch_fi_plan* p = plan; @@ -283,7 +283,7 @@ void bartorch_finufft_free(void* plan) fi_live_plans--; } -long bartorch_finufft_live_plans(void) +int64_t bartorch_finufft_live_plans(void) { return fi_live_plans; } diff --git a/src/csrc/substitute/nufft_finufft.c b/src/csrc/substitute/nufft_finufft.c index e0c4472e..d97089ba 100644 --- a/src/csrc/substitute/nufft_finufft.c +++ b/src/csrc/substitute/nufft_finufft.c @@ -58,61 +58,61 @@ struct bartorch_paired; #include "noncart/gpu_grid.h" /* csrc/kernels.cu: the set's phase and the coil's sensitivity in one pass. */ -extern void bartorch_cuda_phase_map_in(int N, const long dims[], const float shift[3], float scale, +extern void bartorch_cuda_phase_map_in(int N, const bart_dim_t dims[], const float shift[3], float scale, complex float* dst, const complex float* src, const complex float* map); -extern void bartorch_cuda_phase_map_out(int N, const long dims[], const float shift[3], float scale, +extern void bartorch_cuda_phase_map_out(int N, const bart_dim_t dims[], const float shift[3], float scale, complex float* dst, const complex float* src, const complex float* map); -extern void bartorch_cuda_gather(long V, const unsigned int* mask, const int* prefix, complex float* dst, const complex float* src); -extern int bartorch_cuda_contract_upper_real(long L, int R, complex float* bank, const float* mat); -extern int bartorch_cuda_contract_upper_real_bf16(long L, int R, complex float* bank, const void* mat); -extern void bartorch_cuda_scatter(long V, const unsigned int* mask, const int* prefix, complex float* dst, const complex float* src); +extern void bartorch_cuda_gather(bart_dim_t V, const unsigned int* mask, const int* prefix, complex float* dst, const complex float* src); +extern int bartorch_cuda_contract_upper_real(bart_dim_t L, int R, complex float* bank, const float* mat); +extern int bartorch_cuda_contract_upper_real_bf16(bart_dim_t L, int R, complex float* bank, const void* mat); +extern void bartorch_cuda_scatter(bart_dim_t V, const unsigned int* mask, const int* prefix, complex float* dst, const complex float* src); -extern struct bartorch_cb_fft* bartorch_cb_fft_create(const long dims[3]); +extern struct bartorch_cb_fft* bartorch_cb_fft_create(const bart_dim_t dims[3]); extern void bartorch_cb_fft_free(struct bartorch_cb_fft* p); -extern void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const long dims[], +extern void bartorch_cb_fft_forward(struct bartorch_cb_fft* p, int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, complex float* bank, complex float* volume, const complex float* src, const complex float* map); -extern void bartorch_cb_fft_inverse(struct bartorch_cb_fft* p, int N, const long dims[], +extern void bartorch_cb_fft_inverse(struct bartorch_cb_fft* p, int N, const bart_dim_t dims[], const float shift[3], float scale, const unsigned int* mask, const int* prefix, complex float* dst, complex float* volume, const complex float* bank, const complex float* map); #endif #ifdef BARTORCH_PAIRED -extern struct bartorch_paired* bartorch_paired_create(const long dims[3], int coeffs, int sets, const float (*shifts)[3]); +extern struct bartorch_paired* bartorch_paired_create(const bart_dim_t dims[3], int coeffs, int sets, const float (*shifts)[3]); extern void bartorch_paired_free(struct bartorch_paired* p); extern void bartorch_paired_in(const struct bartorch_paired* p, int k, const complex float* src, const complex float* map, complex float* scratch); extern void bartorch_paired_fused(const struct bartorch_paired* p, int k, complex float* scratch, - const void* psf0, const void* psf1, int bf16, const unsigned int* mask, const int* prefix, long L); + const void* psf0, const void* psf1, int bf16, const unsigned int* mask, const int* prefix, bart_dim_t L); extern void bartorch_paired_back(const struct bartorch_paired* p, int k, complex float* dst, const complex float* map, complex float* scratch); #endif #include "include/bartorch.h" -extern struct linop_s* bart_nufft_create2(int N, const long ksp_dims[N], const long cim_dims[N], const long traj_dims[N], const complex float* traj, const long wgh_dims[N], const complex float* weights, const long bas_dims[N], const complex float* basis, struct nufft_conf_s conf); -extern int bart_nufft_get_psf_dims(const struct linop_s* nufft, int N, long psf_dims[N]); -extern void bart_nufft_get_psf(const struct linop_s* nufft, int N, const long psf_dims[N], complex float* psf); -extern void bart_nufft_get_psf2(const struct linop_s* nufft, int N, const long psf_dims[N], const long psf_strs[N], complex float* psf); -extern void bart_nufft_update_psf(const struct linop_s* nufft, int ND, const long psf_dims[ND], const complex float* psf); -extern void bart_nufft_update_psf2(const struct linop_s* nufft, int ND, const long psf_dims[ND], const long psf_strs[ND], const complex float* psf); -extern void bart_nufft_update_traj(const struct linop_s* nufft, int N, const long trj_dims[N], const complex float* traj, const long wgh_dims[N], const complex float* weights, const long bas_dims[N], const complex float* basis); +extern struct linop_s* bart_nufft_create2(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], const bart_dim_t traj_dims[N], const complex float* traj, const bart_dim_t wgh_dims[N], const complex float* weights, const bart_dim_t bas_dims[N], const complex float* basis, struct nufft_conf_s conf); +extern int bart_nufft_get_psf_dims(const struct linop_s* nufft, int N, bart_dim_t psf_dims[N]); +extern void bart_nufft_get_psf(const struct linop_s* nufft, int N, const bart_dim_t psf_dims[N], complex float* psf); +extern void bart_nufft_get_psf2(const struct linop_s* nufft, int N, const bart_dim_t psf_dims[N], const bart_stride_t psf_strs[N], complex float* psf); +extern void bart_nufft_update_psf(const struct linop_s* nufft, int ND, const bart_dim_t psf_dims[ND], const complex float* psf); +extern void bart_nufft_update_psf2(const struct linop_s* nufft, int ND, const bart_dim_t psf_dims[ND], const bart_stride_t psf_strs[ND], const complex float* psf); +extern void bart_nufft_update_traj(const struct linop_s* nufft, int N, const bart_dim_t trj_dims[N], const complex float* traj, const bart_dim_t wgh_dims[N], const complex float* weights, const bart_dim_t bas_dims[N], const complex float* basis); extern const struct operator_s* bart_nufft_precond_create(const struct linop_s* nufft_op); /* Provided by psf.c, which computes the function this convolves with. */ -extern void bartorch_psf_shift(int NS, float shift[NS], int N, const long factors[N], int idx); -extern complex float* bartorch_psf_to_host(int N, const long psf_dims[N + 1], unsigned long flags, - const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, - const long wgh_dims[N + 1], const complex float* weights, +extern void bartorch_psf_shift(int NS, float shift[NS], int N, const bart_dim_t factors[N], int idx); +extern complex float* bartorch_psf_to_host(int N, const bart_dim_t psf_dims[N + 1], bart_flags_t flags, + const bart_dim_t trj_dims[N + 1], const complex float* traj, + const bart_dim_t bas_dims[N + 1], const complex float* basis, + const bart_dim_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag, - const long com_dims[N + 1], const long* idx, - const long com_psf_dims[N + 1], const long com_psf_dims3[N + 1], int real); + const bart_dim_t com_dims[N + 1], const bart_dim_t* idx, + const bart_dim_t com_psf_dims[N + 1], const bart_dim_t com_psf_dims3[N + 1], int real); /* Provided by finufft.c, which owns the FINUFFT entry points. */ extern int bartorch_finufft_plan(int device, int type, int dim, const int64_t n_modes[3], int ntrans, int isign, double eps, double upsampling, int spread_only, void** plan); -extern int bartorch_finufft_setpts(void* plan, long M, float* x, float* y, float* z); +extern int bartorch_finufft_setpts(void* plan, bart_dim_t M, float* x, float* y, float* z); extern int bartorch_finufft_exec(void* plan, complex float* c, complex float* f); extern void bartorch_finufft_free(void* plan); extern double bartorch_finufft_tolerance(void); @@ -127,7 +127,7 @@ extern double bartorch_finufft_upsampling(void); * pairs of sets convolved together by the pair kernels; and the functions * kept in bfloat16. */ enum { TP_PSF, TP_PAIR, TP_COMPRESSED, TP_CALLBACKS, TP_REAL, TP_PAIRED, TP_BF16, TP_COUNTERS }; -static long toeplitz_counters[TP_COUNTERS]; +static bart_dim_t toeplitz_counters[TP_COUNTERS]; /* Building a point spread function needs a transform of its own, and that * transform is nobody's normal: it is asked for one adjoint and freed. While @@ -135,7 +135,7 @@ static long toeplitz_counters[TP_COUNTERS]; * answered a normal either way. */ static _Thread_local int making_psf; -long bartorch_toeplitz_counter(int which) +int64_t bartorch_toeplitz_counter(int which) { return ((0 <= which) && (which < TP_COUNTERS)) ? toeplitz_counters[which] : -1; } @@ -173,7 +173,7 @@ struct fi_side { * transforms rather than inside one, so there it keeps them. */ int ntrans; - long executes; + bart_dim_t executes; }; struct nufft_fi_s { @@ -200,9 +200,9 @@ struct nufft_fi_s { double eps; double upsampling; - long samples; - long batch; - long image_elements; + bart_dim_t samples; + bart_dim_t batch; + bart_dim_t image_elements; float scale; /* out_dims is k-space as the caller sees it -- what BART calls out_dims, @@ -222,22 +222,22 @@ struct nufft_fi_s { */ int N; bool needs_tmp; - long* cim_dims; - long* out_dims; - long* out_strs; - long* grd_dims; - long* trf_strs; - long* wgh_dims; - long* wgh_strs; - long* bas_dims; - long* trj_dims; - long* ksp_dims; + bart_dim_t* cim_dims; + bart_dim_t* out_dims; + bart_stride_t* out_strs; + bart_dim_t* grd_dims; + bart_stride_t* trf_strs; + bart_dim_t* wgh_dims; + bart_stride_t* wgh_strs; + bart_dim_t* bas_dims; + bart_dim_t* trj_dims; + bart_dim_t* ksp_dims; /* What the normal is built from, for a trajectory that arrives after the * operator: a point spread function is over one, so there is none to * convolve with until there is a trajectory to make it from. */ struct nufft_conf_s conf; - long* bas_strs; + bart_stride_t* bas_strs; /* The function the normal convolves with, kept where the card is not. * @@ -259,7 +259,7 @@ struct nufft_fi_s { int slot_pending; /* a slot the card has not yet been held for, or -1 */ int coset; /* the set in it */ bool psf_registered; /* the host copy is page-locked */ - long psf_coset; /* elements in one set of the function */ + bart_dim_t psf_coset; /* elements in one set of the function */ size_t psf_size; /* a real function is stored as floats */ int cosets; int unit; /* sets in a slot: one, or the two of a pair */ @@ -298,15 +298,15 @@ static DEF_TYPEID(nufft_fi_s); * A trajectory can arrive after the operator: `nlinv` and the network models * build theirs against dimensions alone and fill it in with * `nufft_update_traj` once there is one. */ -static void install_traj(struct nufft_fi_s* d, const long traj_dims[], const complex float* traj) +static void install_traj(struct nufft_fi_s* d, const bart_dim_t traj_dims[], const complex float* traj) { int N = d->N; - long one_dims[N]; - long one_strs[N]; - long trj_strs[N]; + bart_dim_t one_dims[N]; + bart_stride_t one_strs[N]; + bart_stride_t trj_strs[N]; - md_select_dims(N, ~1UL, one_dims, traj_dims); + md_select_dims(N, ~UINT64_C(1), one_dims, traj_dims); md_calc_strides(N, one_strs, one_dims, CFL_SIZE); md_calc_strides(N, trj_strs, traj_dims, CFL_SIZE); @@ -329,7 +329,7 @@ static void install_traj(struct nufft_fi_s* d, const long traj_dims[], const com /* Memory on one side of the bus. A device only exists in a CUDA build, and * `device` is never set without one: `bartorch_on_device` says no, and * `bart_use_gpu` stays false. */ -static void* alloc_on(int device, int N, const long dims[N], size_t size) +static void* alloc_on(int device, int N, const bart_dim_t dims[N], size_t size) { #ifdef USE_CUDA if (device) @@ -381,7 +381,7 @@ static int side_build(struct nufft_fi_s* d, int which) return 12; } - long one[1] = { d->samples }; + bart_dim_t one[1] = { d->samples }; for (int i = 0; i < d->dim; i++) { @@ -425,7 +425,7 @@ static int side_retarget(struct nufft_fi_s* d, int which) if (NULL == s->forward_plan) return 0; - long one[1] = { d->samples }; + bart_dim_t one[1] = { d->samples }; for (int i = 0; i < d->dim; i++) md_copy(1, one, s->coord[i], d->radians[i], FL_SIZE); @@ -477,7 +477,7 @@ static complex float* transform_buffer(const struct nufft_fi_s* d, const void* r if (!d->needs_tmp) return NULL; - long dims[1] = { d->samples * d->batch }; + bart_dim_t dims[1] = { d->samples * d->batch }; return md_alloc_sameplace(1, dims, CFL_SIZE, ref); } @@ -502,7 +502,7 @@ static void nufft_fi_forward(const linop_data_t* _d, complex float* dst, const c int ret = 0; - for (long i = 0; (0 == ret) && (i < s->executes); i++) + for (bart_dim_t i = 0; (0 == ret) && (i < s->executes); i++) ret = bartorch_finufft_exec(s->forward_plan, out + i * s->ntrans * d->samples, (complex float*)src + i * s->ntrans * d->image_elements); @@ -573,7 +573,7 @@ static void nufft_fi_adjoint(const linop_data_t* _d, complex float* dst, const c int ret = 0; - for (long i = 0; (0 == ret) && (i < s->executes); i++) + for (bart_dim_t i = 0; (0 == ret) && (i < s->executes); i++) ret = bartorch_finufft_exec(s->adjoint_plan, (complex float*)src + i * s->ntrans * d->samples, dst + i * s->ntrans * d->image_elements); @@ -726,7 +726,7 @@ static void open_slots(struct nufft_fi_s* d, const void* ref) struct nufft_data* t = d->toeplitz_data; int ND = t->N + 1; - long psf_dims[ND]; + bart_dim_t psf_dims[ND]; md_copy_dims(ND, psf_dims, t->psf_dims); psf_dims[t->N] = d->unit; @@ -756,7 +756,7 @@ static void issue_coset(struct nufft_fi_s* d, int i, int slot) struct nufft_data* t = d->toeplitz_data; int ND = t->N + 1; - long psf_bytes = d->psf_coset * (long)d->psf_size * d->unit; + bart_dim_t psf_bytes = d->psf_coset * (bart_dim_t)d->psf_size * d->unit; const char* psf_src = (const char*)d->psf_host + (size_t)i * (size_t)psf_bytes; @@ -766,7 +766,7 @@ static void issue_coset(struct nufft_fi_s* d, int i, int slot) return; } - long psf_dims[ND]; + bart_dim_t psf_dims[ND]; md_copy_dims(ND, psf_dims, t->psf_dims); psf_dims[t->N] = d->unit; @@ -821,7 +821,7 @@ static void use_coset(struct nufft_fi_s* d, int s) struct nufft_data* t = d->toeplitz_data; int ND = t->N + 1; - long psf_dims[ND]; + bart_dim_t psf_dims[ND]; md_copy_dims(ND, psf_dims, t->psf_dims); psf_dims[t->N] = 1; @@ -839,7 +839,7 @@ static void coset_shift(const struct nufft_fi_s* d, int set, float shift[3]) { const struct nufft_data* t = d->toeplitz_data; - long factors[3]; + bart_dim_t factors[3]; for (int i = 0; i < 3; i++) factors[i] = ((t->img_dims[i] > 1) && MD_IS_SET(t->flags, i)) ? 2 : 1; @@ -852,7 +852,7 @@ static void coset_shift(const struct nufft_fi_s* d, int set, float shift[3]) * rather than read from a volume of it. It is BART's own kernel, over the * shift, centring and scale its precomputed phases carry, so what it applies * is what they would have. */ -static void apply_phase(const struct nufft_data* t, const long dims[], const float shift[3], +static void apply_phase(const struct nufft_data* t, const bart_dim_t dims[], const float shift[3], complex float* dst, const complex float* src, bool out) { float scale = 1.f / sqrtf((float)md_calc_size(3, t->img_dims)); @@ -902,7 +902,7 @@ static void apply_phase_map(const struct nufft_data* t, const float shift[3], enum { CONTRACT_CHUNK = 1 << 18 }; static void contract_bank(struct nufft_data* t, const void* psf, - const long bank_dims[], const long ciT_dims[], complex float* bank) + const bart_dim_t bank_dims[], const bart_dim_t ciT_dims[], complex float* bank) { int N = t->N; @@ -912,7 +912,7 @@ static void contract_bank(struct nufft_data* t, const void* psf, * buffer that has to be cleared and copied back. */ if (contraction_kernel && t->conf.real && t->conf.upper_triag && cuda_ondevice(bank)) { - long L = bank_dims[0]; + bart_dim_t L = bank_dims[0]; int R = (int)(md_calc_size(N, bank_dims) / L); if ((md_calc_size(N, ciT_dims) == md_calc_size(N, bank_dims)) @@ -921,17 +921,17 @@ static void contract_bank(struct nufft_data* t, const void* psf, } #endif - long locations = bank_dims[0]; - long chunk = MIN(locations, (long)CONTRACT_CHUNK); + bart_dim_t locations = bank_dims[0]; + bart_dim_t chunk = MIN(locations, (bart_dim_t)CONTRACT_CHUNK); - long bank_strs[N]; - long ciT_strs[N]; + bart_stride_t bank_strs[N]; + bart_stride_t ciT_strs[N]; md_calc_strides(N, bank_strs, bank_dims, CFL_SIZE); md_calc_strides(N, ciT_strs, ciT_dims, CFL_SIZE); - long in_dims[N]; - long out_dims[N]; + bart_dim_t in_dims[N]; + bart_dim_t out_dims[N]; md_copy_dims(N, in_dims, bank_dims); md_copy_dims(N, out_dims, ciT_dims); @@ -941,18 +941,18 @@ static void contract_bank(struct nufft_data* t, const void* psf, complex float* out = md_alloc_sameplace(N, out_dims, CFL_SIZE, bank); - for (long start = 0; start < locations; start += chunk) { + for (bart_dim_t start = 0; start < locations; start += chunk) { - long here = MIN(chunk, locations - start); + bart_dim_t here = MIN(chunk, locations - start); in_dims[0] = here; out_dims[0] = here; - long out_strs[N]; + bart_stride_t out_strs[N]; md_calc_strides(N, out_strs, out_dims, CFL_SIZE); - long max_dims[N]; - md_max_dims(N, ~0UL, max_dims, out_dims, in_dims); + bart_dim_t max_dims[N]; + md_max_dims(N, ~UINT64_C(0), max_dims, out_dims, in_dims); complex float* in = bank + start; const void* mat = (const char*)psf + (size_t)start * (size_t)t->psf_strs[0]; @@ -995,13 +995,13 @@ static void contract_bank(struct nufft_data* t, const void* psf, * it is a matrix at every frequency and the contraction needs somewhere to * land, so what comes back may not be what went in. */ static complex float* multiply_transfer(struct nufft_data* t, const void* psf, - const long cim_dims[], const long ciT_dims[], complex float* grid) + const bart_dim_t cim_dims[], const bart_dim_t ciT_dims[], complex float* grid) { int N = t->N; - if (md_check_equal_dims(N, cim_dims, ciT_dims, ~0UL)) { + if (md_check_equal_dims(N, cim_dims, ciT_dims, ~UINT64_C(0))) { - long cim_strs[N]; + bart_stride_t cim_strs[N]; md_calc_strides(N, cim_strs, cim_dims, CFL_SIZE); if (t->conf.real) @@ -1015,13 +1015,13 @@ static complex float* multiply_transfer(struct nufft_data* t, const void* psf, return grid; } - long max_dims[N]; - md_max_dims(N, ~0UL, max_dims, ciT_dims, cim_dims); + bart_dim_t max_dims[N]; + md_max_dims(N, ~UINT64_C(0), max_dims, ciT_dims, cim_dims); - long ciT_strs[N]; + bart_stride_t ciT_strs[N]; md_calc_strides(N, ciT_strs, ciT_dims, CFL_SIZE); - long cim_strs[N]; + bart_stride_t cim_strs[N]; md_calc_strides(N, cim_strs, cim_dims, CFL_SIZE); complex float* out = md_alloc_sameplace(N, ciT_dims, CFL_SIZE, grid); @@ -1154,7 +1154,7 @@ static struct bartorch_cb_fft* callbacks_for(struct nufft_fi_s* d) #ifdef USE_CUDA const struct nufft_data* t = d->toeplitz_data; - unsigned long flags = t->flags | t->conf.cfft; + bart_flags_t flags = t->flags | t->conf.cfft; bool covered = true; for (int i = 0; i < t->N; i++) @@ -1188,7 +1188,7 @@ static void slot_ready(struct nufft_fi_s* d) } /* Down to the places the samples reach, and back up with zeros elsewhere. */ -static void gather(const struct nufft_fi_s* d, long grid, complex float* dst, const complex float* src) +static void gather(const struct nufft_fi_s* d, bart_dim_t grid, complex float* dst, const complex float* src) { #ifdef USE_CUDA bartorch_cuda_gather(grid, d->kept_mask, d->kept_prefix, dst, src); @@ -1198,7 +1198,7 @@ static void gather(const struct nufft_fi_s* d, long grid, complex float* dst, co #endif } -static void scatter(const struct nufft_fi_s* d, long grid, complex float* dst, const complex float* src) +static void scatter(const struct nufft_fi_s* d, bart_dim_t grid, complex float* dst, const complex float* src) { #ifdef USE_CUDA bartorch_cuda_scatter(grid, d->kept_mask, d->kept_prefix, dst, src); @@ -1215,7 +1215,7 @@ static void coefficient_in(struct nufft_fi_s* d, struct bartorch_cb_fft* cb, con { struct nufft_data* t = d->toeplitz_data; int N = t->N; - long grid = md_calc_size(3, t->img_dims); + bart_dim_t grid = md_calc_size(3, t->img_dims); #ifdef USE_CUDA if (NULL != cb) { @@ -1242,7 +1242,7 @@ static void coefficient_out(struct nufft_fi_s* d, struct bartorch_cb_fft* cb, co { struct nufft_data* t = d->toeplitz_data; int N = t->N; - long grid = md_calc_size(3, t->img_dims); + bart_dim_t grid = md_calc_size(3, t->img_dims); #ifdef USE_CUDA if (NULL != cb) { @@ -1278,7 +1278,7 @@ static void coefficient_out(struct nufft_fi_s* d, struct bartorch_cb_fft* cb, co * inside the transforms, they run there. */ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex float* src, - const void* psf, const long map_strs[], const complex float* map, bool last) + const void* psf, const bart_stride_t map_strs[], const complex float* map, bool last) { struct nufft_data* t = d->toeplitz_data; int N = t->N; @@ -1293,9 +1293,9 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex /* The spectrum of one coil, gathered: the places the samples reach on * the first axis, the coefficients on the axis the function contracts. */ - long bank_dims[N]; - long ciT_bank_dims[N]; - long one_bank_dims[N]; + bart_dim_t bank_dims[N]; + bart_dim_t ciT_bank_dims[N]; + bart_dim_t one_bank_dims[N]; md_select_dims(N, ~MD_BIT(3), bank_dims, t->cim_dims); md_select_dims(N, ~MD_BIT(3), ciT_bank_dims, t->ciT_dims); @@ -1305,10 +1305,10 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex md_select_dims(N, MD_BIT(0), one_bank_dims, bank_dims); - long locations = bank_dims[0]; - long coils = t->cim_dims[3]; - long coeffs = md_calc_size(N, bank_dims) / locations; - long out_coeffs = md_calc_size(N, ciT_bank_dims) / locations; + bart_dim_t locations = bank_dims[0]; + bart_dim_t coils = t->cim_dims[3]; + bart_dim_t coeffs = md_calc_size(N, bank_dims) / locations; + bart_dim_t out_coeffs = md_calc_size(N, ciT_bank_dims) / locations; /* Where a coil's coefficient sits in what is read and written. Counted * here rather than read off `cim_strs`, which carries a zero for every @@ -1317,22 +1317,22 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex * With the sensitivity folded in, what is read and written is the image * rather than a coil image: it has no coil axis, every coil accumulates * into the same one, and the map is what tells them apart. */ - long vol = md_calc_size(3, t->cim_dims); + bart_dim_t vol = md_calc_size(3, t->cim_dims); - long coil_step = (NULL == map) ? vol : 0; - long rest_step = (NULL == map) ? vol * t->cim_dims[3] : vol; + bart_dim_t coil_step = (NULL == map) ? vol : 0; + bart_dim_t rest_step = (NULL == map) ? vol * t->cim_dims[3] : vol; - long map_coil_step = (NULL == map) ? 0 : map_strs[3] / (long)CFL_SIZE; + bart_dim_t map_coil_step = (NULL == map) ? 0 : map_strs[3] / (bart_stride_t)CFL_SIZE; complex float* volume = md_alloc_sameplace(N, t->img_dims, CFL_SIZE, dst); - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_coil_step; complex float* bank = md_alloc_sameplace(N, bank_dims, CFL_SIZE, dst); - for (long r = 0; r < coeffs; r++) + for (bart_dim_t r = 0; r < coeffs; r++) coefficient_in(d, cb, shift, bank + r * locations, volume, src + c * coil_step + r * rest_step, m); slot_ready(d); @@ -1342,14 +1342,14 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex bartorch_cuda_contract_upper_real_bf16(locations, (int)coeffs, bank, psf); else #endif - if (md_check_equal_dims(N, bank_dims, ciT_bank_dims, ~0UL)) + if (md_check_equal_dims(N, bank_dims, ciT_bank_dims, ~UINT64_C(0))) bank = multiply_transfer(t, psf, bank_dims, ciT_bank_dims, bank); else contract_bank(t, psf, bank_dims, ciT_bank_dims, bank); slot_read(d, last && (c == coils - 1)); - for (long r = 0; r < out_coeffs; r++) + for (bart_dim_t r = 0; r < out_coeffs; r++) coefficient_out(d, cb, shift, dst + c * coil_step + r * rest_step, volume, bank + r * locations, m); md_free(bank); @@ -1368,11 +1368,11 @@ static void packed_coset(struct nufft_fi_s* d, complex float* dst, const complex * turn. The kernels read a coil's coefficients a volume apart, which a folded * sensitivity or a single coil gives; anything else goes a set at a time. */ static void paired_unit(struct nufft_fi_s* d, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, bool last) + const bart_stride_t map_strs[], const complex float* map, bool last) { struct nufft_data* t = d->toeplitz_data; - long coils = t->cim_dims[3]; + bart_dim_t coils = t->cim_dims[3]; if ((NULL == map) && (1 != coils)) { @@ -1390,17 +1390,17 @@ static void paired_unit(struct nufft_fi_s* d, complex float* dst, const complex } #ifdef BARTORCH_PAIRED - long vol = md_calc_size(3, t->cim_dims); - long coeffs = md_calc_size(t->N, t->cim_dims) / md_calc_size(4, t->cim_dims); - long map_coil_step = (NULL == map) ? 0 : map_strs[3] / (long)CFL_SIZE; + bart_dim_t vol = md_calc_size(3, t->cim_dims); + bart_dim_t coeffs = md_calc_size(t->N, t->cim_dims) / md_calc_size(4, t->cim_dims); + bart_dim_t map_coil_step = (NULL == map) ? 0 : map_strs[3] / (bart_stride_t)CFL_SIZE; - long sdims[1] = { coeffs * vol }; + bart_dim_t sdims[1] = { coeffs * vol }; complex float* scratch = md_alloc_sameplace(1, sdims, CFL_SIZE, dst); const char* psf0 = d->psf_slot[d->slot]; const char* psf1 = psf0 + (size_t)d->psf_coset * d->psf_size; - for (long c = 0; c < coils; c++) { + for (bart_dim_t c = 0; c < coils; c++) { const complex float* m = (NULL == map) ? NULL : map + c * map_coil_step; @@ -1438,7 +1438,7 @@ static void paired_unit(struct nufft_fi_s* d, complex float* dst, const complex * triangle -- and the arrangements it does not cover go back to BART's own. */ static bool fused_coset(struct nufft_fi_s* d, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, bool last) + const bart_stride_t map_strs[], const complex float* map, bool last) { if (2 == d->unit) { @@ -1524,7 +1524,7 @@ static void coset_begin(struct nufft_fi_s* d, const void* ref) /* The set that is loaded, convolved with `src` and added to `dst`. */ static void coset_normal(struct nufft_fi_s* d, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, bool last) + const bart_stride_t map_strs[], const complex float* map, bool last) { if (!fused_coset(d, dst, src, map_strs, map, last)) error("bartorch: a streamed set with no function in its slot\n"); @@ -1572,7 +1572,7 @@ int bartorch_nufft_coset_folds(const struct linop_s* op) * each of those is half a gigabyte. Folded in here neither is made. */ void bartorch_nufft_coset_normal_sense(const struct linop_s* op, complex float* dst, const complex float* src, - const long map_strs[], const complex float* map, int last) + const int64_t map_strs[], const complex float* map, int last) { coset_normal(CAST_DOWN(nufft_fi_s, linop_get_data(op)), dst, src, map_strs, map, (0 != last)); } @@ -1726,9 +1726,9 @@ int bartorch_nufft_fallback_allowed(void) /* Which operator each nufft_create call returned, so a test can say that a * tool ran on FINUFFT rather than that FINUFFT was merely available. */ enum { CNT_FI, CNT_BART }; -static long counters[2]; +static bart_dim_t counters[2]; -long bartorch_nufft_counter(int which) +int64_t bartorch_nufft_counter(int which) { return ((0 == which) || (1 == which)) ? counters[which] : -1; } @@ -1787,8 +1787,8 @@ static int fi_measure_width(int device, int dim, double eps, double upsampling) enum { PROBE = 32 }; /* wider than any kernel FINUFFT will use */ int64_t n_modes[3] = { PROBE, PROBE, PROBE }; - long grid_dims[3] = { 1, 1, 1 }; - long one[1] = { 1 }; + bart_dim_t grid_dims[3] = { 1, 1, 1 }; + bart_dim_t one[1] = { 1 }; for (int i = 0; i < dim; i++) grid_dims[i] = PROBE; @@ -1833,8 +1833,8 @@ static int fi_measure_width(int device, int dim, double eps, double upsampling) md_free(grid); /* Along one axis through the middle, where the sample was put. */ - long stride = 1; - long centre = 0; + bart_stride_t stride = 1; + bart_dim_t centre = 0; for (int i = 1; i < dim; i++) { @@ -1909,7 +1909,7 @@ static double fi_tolerance_for(int device, int dim, double width, double upsampl * is also what keeps the doubled grid from ever being allocated, which is the * whole reason the decomposition is there. */ -static int spread_mask(struct nufft_data* data, const complex float* traj, long* max_idx) +static int spread_mask(struct nufft_data* data, const complex float* traj, bart_dim_t* max_idx) { int N = data->N; int ND = N + 1; @@ -1964,7 +1964,7 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* int dim = 0; int axis[3]; int64_t n_modes[3]; - unsigned long whole = 0UL; + bart_flags_t whole = 0; for (int i = 0; i < 3; i++) { @@ -1982,24 +1982,24 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* dim++; } - long factors[N]; + bart_dim_t factors[N]; for (int i = 0; i < N; i++) factors[i] = ((data->img_dims[i] > 1) && MD_IS_SET(data->flags, i)) ? 2 : 1; - long sets = md_calc_size(N, factors); + bart_dim_t sets = md_calc_size(N, factors); - long one_dims[ND]; - long one_strs[ND]; - long trj_strs[ND]; - long wgh_strs[ND]; + bart_dim_t one_dims[ND]; + bart_stride_t one_strs[ND]; + bart_stride_t trj_strs[ND]; + bart_stride_t wgh_strs[ND]; - md_select_dims(ND, ~1UL, one_dims, data->trj_dims); + md_select_dims(ND, ~UINT64_C(1), one_dims, data->trj_dims); md_calc_strides(ND, one_strs, one_dims, CFL_SIZE); md_calc_strides(ND, trj_strs, data->trj_dims, CFL_SIZE); md_calc_strides(ND, wgh_strs, data->wgh_dims, CFL_SIZE); - long samples = md_calc_size(ND, one_dims); + bart_dim_t samples = md_calc_size(ND, one_dims); /* One component of the trajectory per transformed axis, in grid samples, * before the shift of a set turns it into that set's coordinates. */ @@ -2023,7 +2023,7 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* else md_copy2(ND, one_dims, one_strs, samples_in, wgh_strs, pattern, CFL_SIZE); - long spread_dims[ND]; + bart_dim_t spread_dims[ND]; md_select_dims(ND, ~whole, spread_dims, data->com_dims); complex float* reach = alloc_on(device, ND, spread_dims, CFL_SIZE); @@ -2045,7 +2045,7 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* if (0 == dim) md_zfill(ND, spread_dims, mask, 1.); - for (long set = 0; (0 < dim) && (0 == ret) && (set < sets); set++) { + for (bart_dim_t set = 0; (0 < dim) && (0 == ret) && (set < sets); set++) { float shift[3]; bartorch_psf_shift(3, shift, N, factors, (int)set); @@ -2104,12 +2104,12 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* MD_STRIDES(ND, spread_dims, CFL_SIZE), mask, CFL_SIZE); md_free(mask); - long* idx = md_alloc(ND, data->com_dims, sizeof(long)); + bart_dim_t* idx = md_alloc(ND, data->com_dims, sizeof(bart_dim_t)); *max_idx = md_compress_mask_to_index(ND, data->com_dims, idx, mask_cpu); md_free(mask_cpu); multiplace_free(data->compress); - data->compress = multiplace_move_F(ND, data->com_dims, sizeof(long), idx); + data->compress = multiplace_move_F(ND, data->com_dims, sizeof(bart_dim_t), idx); debug_printf(DP_DEBUG1, "Compressing PSF to %.0f%%\n", 100. * *max_idx / md_calc_size(ND, data->com_dims)); @@ -2135,12 +2135,12 @@ static int spread_mask(struct nufft_data* data, const complex float* traj, long* * and cancels. A basis is small enough to ask about on the host, and the * angle is half the argument of the sum of its squares -- which is `e^{2it}` * times something real when there is one angle to find. */ -static bool basis_is_real(int N, const long bas_dims[N], const complex float* basis) +static bool basis_is_real(int N, const bart_dim_t bas_dims[N], const complex float* basis) { if (NULL == basis) return true; - long size = md_calc_size(N, bas_dims); + bart_dim_t size = md_calc_size(N, bas_dims); complex float* host = md_alloc(N, bas_dims, CFL_SIZE); md_copy(N, bas_dims, host, basis, CFL_SIZE); @@ -2148,7 +2148,7 @@ static bool basis_is_real(int N, const long bas_dims[N], const complex float* ba complex float squares = 0.; double energy = 0.; - for (long i = 0; i < size; i++) { + for (bart_dim_t i = 0; i < size; i++) { squares += host[i] * host[i]; energy += (double)crealf(host[i]) * crealf(host[i]) + (double)cimagf(host[i]) * cimagf(host[i]); @@ -2158,7 +2158,7 @@ static bool basis_is_real(int N, const long bas_dims[N], const complex float* ba double left = 0.; - for (long i = 0; i < size; i++) { + for (bart_dim_t i = 0; i < size; i++) { float part = cimagf(host[i] * turn); left += (double)part * part; @@ -2205,7 +2205,7 @@ static void install_psf(struct nufft_data* data, const complex float* traj, comp /* The places the samples reach. Worked out before the function is * built, because an entry is compressed as it is made. */ - long max_idx = 0; + bart_dim_t max_idx = 0; if (data->conf.compress_psf) { @@ -2232,13 +2232,13 @@ static void install_psf(struct nufft_data* data, const complex float* traj, comp * dimensional radial trajectory leaves the corners of the cube * outside its ball, which is a fraction 1 - pi/6 of it before * the spreading kernel's width is added back. */ - long grid = md_calc_size(ND, data->com_dims); - long entries = md_calc_size(N, data->psf_dims) / md_calc_size(3, data->psf_dims); + bart_dim_t grid = md_calc_size(ND, data->com_dims); + bart_dim_t entries = md_calc_size(N, data->psf_dims) / md_calc_size(3, data->psf_dims); size_t element = store_real ? FL_SIZE : CFL_SIZE; double dropped = 1. - (double)max_idx / (double)grid; - if (dropped * (double)entries * (double)element <= (double)sizeof(long)) { + if (dropped * (double)entries * (double)element <= (double)sizeof(bart_dim_t)) { debug_printf(DP_DEBUG1, "Not compressing: %.0f%% of the grid is reached\n", 100. * max_idx / grid); @@ -2254,9 +2254,9 @@ static void install_psf(struct nufft_data* data, const complex float* traj, comp #pragma omp atomic toeplitz_counters[TP_COMPRESSED]++; - long com_psf_dims[ND]; - long com_psf_dims3[ND]; - const long* idx = NULL; + bart_dim_t com_psf_dims[ND]; + bart_dim_t com_psf_dims3[ND]; + const bart_dim_t* idx = NULL; if (stream && (NULL != data->compress)) { @@ -2313,7 +2313,7 @@ static void install_psf(struct nufft_data* data, const complex float* traj, comp size_t size = store_real ? FL_SIZE : CFL_SIZE; - long com_psf_dims[ND]; + bart_dim_t com_psf_dims[ND]; md_compress_dims(ND, com_psf_dims, data->psf_dims, data->com_dims, max_idx); complex float* com_psf = md_alloc_sameplace(ND, com_psf_dims, size, traj); @@ -2388,21 +2388,21 @@ static void stream_psf(struct nufft_fi_s* d) int ND = t->N + 1; - long grid = md_calc_size(ND, t->com_dims); - const long* map = multiplace_read(t->compress, d->radians[0]); + bart_dim_t grid = md_calc_size(ND, t->com_dims); + const bart_dim_t* map = multiplace_read(t->compress, d->radians[0]); - long wdims[1] = { (grid + 31) / 32 }; + bart_dim_t wdims[1] = { (grid + 31) / 32 }; unsigned int* mask = xmalloc((size_t)wdims[0] * sizeof(unsigned int)); int* prefix = xmalloc((size_t)wdims[0] * sizeof(int)); - long n = 0; + bart_dim_t n = 0; - for (long w = 0; w < wdims[0]; w++) { + for (bart_dim_t w = 0; w < wdims[0]; w++) { unsigned int bits = 0; - for (long b = 0; (b < 32) && (w * 32 + b < grid); b++) + for (bart_dim_t b = 0; (b < 32) && (w * 32 + b < grid); b++) if (0 <= map[w * 32 + b]) bits |= 1u << b; @@ -2443,7 +2443,7 @@ static void stream_psf(struct nufft_fi_s* d) for (int i = 0; i < 8; i++) coset_shift(d, i, shifts[i]); - long coeffs = md_calc_size(t->N, t->cim_dims) / md_calc_size(4, t->cim_dims); + bart_dim_t coeffs = md_calc_size(t->N, t->cim_dims) / md_calc_size(4, t->cim_dims); d->paired = bartorch_paired_create(t->img_dims, (int)coeffs, 8, (const float (*)[3])shifts); } @@ -2462,11 +2462,11 @@ static void stream_psf(struct nufft_fi_s* d) * contraction kernel, so that kernel is a condition. */ if ((2 == d->unit) && bf16_enabled && contraction_kernel) { - long n = (long)cosets * d->psf_coset; + bart_dim_t n = (bart_dim_t)cosets * d->psf_coset; const float* in = (const float*)d->psf_host; uint16_t* out = (uint16_t*)d->psf_host; - for (long i = 0; i < n; i++) + for (bart_dim_t i = 0; i < n; i++) out[i] = to_bf16(in[i]); void* shrunk = realloc(d->psf_host, (size_t)n * sizeof(uint16_t)); @@ -2481,18 +2481,18 @@ static void stream_psf(struct nufft_fi_s* d) /* Page-locked, so a set's crossing runs behind the convolution rather * than holding up the host that issued it. */ d->psf_registered = (0 == bartorch_cuda_host_register(d->psf_host, - (long)cosets * d->psf_coset * (long)d->psf_size)); + (bart_dim_t)cosets * d->psf_coset * (bart_dim_t)d->psf_size)); /* One set is all BART is told it has, so its own loop runs once. */ t->lph_dims[t->N] = 1; - debug_printf(DP_DEBUG1, "Streaming the function: %d sets of %ld, %d to a slot\n", cosets, d->psf_coset, d->unit); + debug_printf(DP_DEBUG1, "Streaming the function: %d sets of %" PRId64 ", %d to a slot\n", cosets, d->psf_coset, d->unit); } -static const struct linop_s* toeplitz_for(int N, const long ksp_dims[N], const long cim_dims[N], - const long traj_dims[N], const complex float* traj, - const long wgh_dims[N], const complex float* weights, - const long bas_dims[N], const complex float* basis, struct nufft_conf_s conf, +static const struct linop_s* toeplitz_for(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], + const bart_dim_t traj_dims[N], const complex float* traj, + const bart_dim_t wgh_dims[N], const complex float* weights, + const bart_dim_t bas_dims[N], const complex float* basis, struct nufft_conf_s conf, complex float** to_host) { if (!conf.toeplitz) { @@ -2574,14 +2574,14 @@ static const struct linop_s* toeplitz_for(int N, const long ksp_dims[N], const l * its place before the function is built from it. */ bool along_samples = (NULL != basis) && ((1 != bas_dims[1]) || (1 != bas_dims[2])); - long stand_dims[N]; + bart_dim_t stand_dims[N]; complex float* stand = NULL; if (along_samples) { md_select_dims(N, ~(MD_BIT(1) | MD_BIT(2)), stand_dims, bas_dims); - long pos[N]; + bart_dim_t pos[N]; md_set_dims(N, pos, 0); complex float* whole = md_alloc(N, bas_dims, CFL_SIZE); @@ -2649,10 +2649,10 @@ static const struct linop_s* toeplitz_for(int N, const long ksp_dims[N], const l return op; } -static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_dims[N], - const long traj_dims[N], const complex float* traj, - const long wgh_dims[N], const complex float* weights, - const long bas_dims[N], const complex float* basis, struct nufft_conf_s conf) +static struct linop_s* try_create(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], + const bart_dim_t traj_dims[N], const complex float* traj, + const bart_dim_t wgh_dims[N], const complex float* weights, + const bart_dim_t bas_dims[N], const complex float* basis, struct nufft_conf_s conf) { /* Which sides the operator may be applied on. BART on a card applies * one to either: `pics` takes its first adjoint from the k-space it @@ -2676,7 +2676,7 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ if (1 != ksp_dims[0]) DECLINE(5); - if ((0UL != conf.flags) && (7UL != conf.flags)) + if ((0 != conf.flags) && (7 != conf.flags)) DECLINE(6); int dim = 0; @@ -2704,8 +2704,8 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ * A caller may pass either one in: `nufft` gives k-space with a single * coefficient and `pics` gives it with all of them, which is why BART * fills the coefficient axis in rather than reading it. */ - long out_dims[N]; - long grd_dims[N]; + bart_dim_t out_dims[N]; + bart_dim_t grd_dims[N]; md_copy_dims(N, out_dims, ksp_dims); md_copy_dims(N, grd_dims, ksp_dims); @@ -2747,8 +2747,8 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ * rest are separate transforms. A frame is a sample axis when the * trajectory varies across frames, which is what lets one plan over the * whole raveled trajectory serve every coefficient and every coil. */ - long samples = md_calc_size(N - 1, traj_dims + 1); - long batch = 1; + bart_dim_t samples = md_calc_size(N - 1, traj_dims + 1); + bart_dim_t batch = 1; for (int i = 1; i < N; i++) { @@ -2832,10 +2832,10 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ DECLINE(17); } - long image_elements = 1; + bart_dim_t image_elements = 1; for (int i = 0; i < dim; i++) - image_elements *= (long)n_modes[i]; + image_elements *= (bart_dim_t)n_modes[i]; PTR_ALLOC(struct nufft_fi_s, d); SET_TYPEID(nufft_fi_s, d); @@ -2888,17 +2888,17 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ d->N = N; d->host_weights = NULL; d->host_basis = NULL; - d->cim_dims = xmalloc((size_t)N * sizeof(long)); - d->out_dims = xmalloc((size_t)N * sizeof(long)); - d->out_strs = xmalloc((size_t)N * sizeof(long)); - d->grd_dims = xmalloc((size_t)N * sizeof(long)); - d->trf_strs = xmalloc((size_t)N * sizeof(long)); - d->wgh_dims = xmalloc((size_t)N * sizeof(long)); - d->wgh_strs = xmalloc((size_t)N * sizeof(long)); - d->bas_dims = xmalloc((size_t)N * sizeof(long)); - d->bas_strs = xmalloc((size_t)N * sizeof(long)); - d->trj_dims = xmalloc((size_t)N * sizeof(long)); - d->ksp_dims = xmalloc((size_t)N * sizeof(long)); + d->cim_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->out_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->out_strs = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->grd_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->trf_strs = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->wgh_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->wgh_strs = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->bas_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->bas_strs = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->trj_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); + d->ksp_dims = xmalloc((size_t)N * sizeof(bart_dim_t)); md_copy_dims(N, d->trj_dims, traj_dims); md_copy_dims(N, d->ksp_dims, ksp_dims); d->conf = conf; @@ -2916,26 +2916,26 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ * stepping over them, so the sample axes take the fastest strides and * the rest follow. BART's own order is already that unless a sample * axis sits above a batch axis. */ - long stride = 1; + bart_stride_t stride = 1; for (int i = 1; i < N; i++) if (1 < traj_dims[i]) { - d->trf_strs[i] = (1 == grd_dims[i]) ? 0 : stride * (long)CFL_SIZE; + d->trf_strs[i] = (1 == grd_dims[i]) ? 0 : stride * (bart_stride_t)CFL_SIZE; stride *= grd_dims[i]; } for (int i = 0; i < N; i++) if ((0 == i) || (1 >= traj_dims[i])) { - d->trf_strs[i] = (1 == grd_dims[i]) ? 0 : stride * (long)CFL_SIZE; + d->trf_strs[i] = (1 == grd_dims[i]) ? 0 : stride * (bart_stride_t)CFL_SIZE; stride *= grd_dims[i]; } - long grd_strs[N]; + bart_stride_t grd_strs[N]; md_calc_strides(N, grd_strs, grd_dims, CFL_SIZE); - d->needs_tmp = (NULL != basis) || (0 != memcmp(d->trf_strs, grd_strs, (size_t)N * sizeof(long))); + d->needs_tmp = (NULL != basis) || (0 != memcmp(d->trf_strs, grd_strs, (size_t)N * sizeof(bart_dim_t))); if (NULL != basis) { @@ -2991,10 +2991,10 @@ static struct linop_s* try_create(int N, const long ksp_dims[N], const long cim_ return op; } -struct linop_s* nufft_create2(int N, const long ksp_dims[N], const long cim_dims[N], - const long traj_dims[N], const complex float* traj, - const long wgh_dims[N], const complex float* weights, - const long bas_dims[N], const complex float* basis, struct nufft_conf_s conf) +struct linop_s* nufft_create2(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], + const bart_dim_t traj_dims[N], const complex float* traj, + const bart_dim_t wgh_dims[N], const complex float* weights, + const bart_dim_t bas_dims[N], const complex float* basis, struct nufft_conf_s conf) { struct linop_s* op = try_create(N, ksp_dims, cim_dims, traj_dims, traj, wgh_dims, weights, bas_dims, basis, conf); @@ -3014,11 +3014,11 @@ struct linop_s* nufft_create2(int N, const long ksp_dims[N], const long cim_dims return bart_nufft_create2(N, ksp_dims, cim_dims, traj_dims, traj, wgh_dims, weights, bas_dims, basis, barts_conf(conf)); } -struct linop_s* nufft_create(int N, const long ksp_dims[N], const long cim_dims[N], - const long traj_dims[N], const complex float* traj, +struct linop_s* nufft_create(int N, const bart_dim_t ksp_dims[N], const bart_dim_t cim_dims[N], + const bart_dim_t traj_dims[N], const complex float* traj, const complex float* weights, struct nufft_conf_s conf) { - long wgh_dims[N]; + bart_dim_t wgh_dims[N]; md_select_dims(N, ~MD_BIT(0), wgh_dims, traj_dims); return nufft_create2(N, ksp_dims, cim_dims, traj_dims, traj, wgh_dims, weights, NULL, NULL, conf); @@ -3033,7 +3033,7 @@ static void refuse(const char* what) "Turn the substitution off with bartorch.finufft.use_in_tools(False).\n", what); } -int nufft_get_psf_dims(const struct linop_s* nufft, int N, long psf_dims[N]) +int nufft_get_psf_dims(const struct linop_s* nufft, int N, bart_dim_t psf_dims[N]) { if (is_ours(nufft)) refuse("reading a point spread function"); @@ -3041,7 +3041,7 @@ int nufft_get_psf_dims(const struct linop_s* nufft, int N, long psf_dims[N]) return bart_nufft_get_psf_dims(nufft, N, psf_dims); } -void nufft_get_psf(const struct linop_s* nufft, int N, const long psf_dims[N], complex float* psf) +void nufft_get_psf(const struct linop_s* nufft, int N, const bart_dim_t psf_dims[N], complex float* psf) { if (is_ours(nufft)) refuse("reading a point spread function"); @@ -3049,7 +3049,7 @@ void nufft_get_psf(const struct linop_s* nufft, int N, const long psf_dims[N], c bart_nufft_get_psf(nufft, N, psf_dims, psf); } -void nufft_get_psf2(const struct linop_s* nufft, int N, const long psf_dims[N], const long psf_strs[N], complex float* psf) +void nufft_get_psf2(const struct linop_s* nufft, int N, const bart_dim_t psf_dims[N], const bart_stride_t psf_strs[N], complex float* psf) { if (is_ours(nufft)) refuse("reading a point spread function"); @@ -3057,7 +3057,7 @@ void nufft_get_psf2(const struct linop_s* nufft, int N, const long psf_dims[N], bart_nufft_get_psf2(nufft, N, psf_dims, psf_strs, psf); } -void nufft_update_psf(const struct linop_s* nufft, int ND, const long psf_dims[ND], const complex float* psf) +void nufft_update_psf(const struct linop_s* nufft, int ND, const bart_dim_t psf_dims[ND], const complex float* psf) { if (is_ours(nufft)) refuse("supplying a point spread function"); @@ -3065,7 +3065,7 @@ void nufft_update_psf(const struct linop_s* nufft, int ND, const long psf_dims[N bart_nufft_update_psf(nufft, ND, psf_dims, psf); } -void nufft_update_psf2(const struct linop_s* nufft, int ND, const long psf_dims[ND], const long psf_strs[ND], const complex float* psf) +void nufft_update_psf2(const struct linop_s* nufft, int ND, const bart_dim_t psf_dims[ND], const bart_stride_t psf_strs[ND], const complex float* psf) { if (is_ours(nufft)) refuse("supplying a point spread function"); @@ -3077,7 +3077,7 @@ void nufft_update_psf2(const struct linop_s* nufft, int ND, const long psf_dims[ * alone and hand the trajectory over here, once per frame of a run. The * plans belong to it, so both sides go and are made again on the next * transform, over whatever else arrived with it. */ -void nufft_update_traj(const struct linop_s* nufft, int N, const long trj_dims[N], const complex float* traj, const long wgh_dims[N], const complex float* weights, const long bas_dims[N], const complex float* basis) +void nufft_update_traj(const struct linop_s* nufft, int N, const bart_dim_t trj_dims[N], const complex float* traj, const bart_dim_t wgh_dims[N], const complex float* weights, const bart_dim_t bas_dims[N], const complex float* basis) { if (!is_ours(nufft)) { @@ -3091,13 +3091,13 @@ void nufft_update_traj(const struct linop_s* nufft, int N, const long trj_dims[N error("bartorch: a trajectory of %d axes for an operator of %d\n", N, d->N); if (d->samples != md_calc_size(N - 1, trj_dims + 1)) - error("bartorch: a trajectory of %ld samples for an operator of %ld\n", + error("bartorch: a trajectory of %" PRId64 " samples for an operator of %" PRId64 "\n", md_calc_size(N - 1, trj_dims + 1), d->samples); - if ((NULL != weights) && !md_check_equal_dims(N, wgh_dims, d->wgh_dims, ~0UL)) + if ((NULL != weights) && !md_check_equal_dims(N, wgh_dims, d->wgh_dims, ~UINT64_C(0))) error("bartorch: weights of a shape the operator was not built for\n"); - if ((NULL != basis) && !md_check_equal_dims(N, bas_dims, d->bas_dims, ~0UL)) + if ((NULL != basis) && !md_check_equal_dims(N, bas_dims, d->bas_dims, ~UINT64_C(0))) error("bartorch: a basis of a shape the operator was not built for\n"); pthread_mutex_lock(&d->lock); diff --git a/src/csrc/substitute/psf.c b/src/csrc/substitute/psf.c index 498471f9..c65f006e 100644 --- a/src/csrc/substitute/psf.c +++ b/src/csrc/substitute/psf.c @@ -45,7 +45,7 @@ #include "noncart/nufft.h" /* The weights the normal carries are the transform's, squared. */ -static complex float* square_weights(int N, const long wgh_dims[N], const complex float* weights) +static complex float* square_weights(int N, const bart_dim_t wgh_dims[N], const complex float* weights) { if (NULL == weights) return NULL; @@ -58,8 +58,8 @@ static complex float* square_weights(int N, const long wgh_dims[N], const comple /* A subspace basis enters the normal as its own Gram matrix, laid out along * the coefficient axis the transform contracts. */ -static complex float* square_basis(bool upper_triag, int N, long sqr_bas_dims[N], - const long bas_dims[N], const complex float* basis, const long ksp_dims[N]) +static complex float* square_basis(bool upper_triag, int N, bart_dim_t sqr_bas_dims[N], + const bart_dim_t bas_dims[N], const complex float* basis, const bart_dim_t ksp_dims[N]) { if (NULL == basis) { @@ -69,10 +69,10 @@ static complex float* square_basis(bool upper_triag, int N, long sqr_bas_dims[N] assert(1 == bas_dims[7]); - long bas_dimsT[N]; + bart_dim_t bas_dimsT[N]; md_transpose_dims(N, 6, 7, bas_dimsT, bas_dims); - md_max_dims(N, ~0UL, sqr_bas_dims, bas_dims, bas_dimsT); + md_max_dims(N, ~UINT64_C(0), sqr_bas_dims, bas_dims, bas_dimsT); sqr_bas_dims[5] = ksp_dims[5]; complex float* sqr = md_alloc_sameplace(N, sqr_bas_dims, CFL_SIZE, basis); @@ -97,7 +97,7 @@ static complex float* square_basis(bool upper_triag, int N, long sqr_bas_dims[N] if (upper_triag) { - long tri_dims[N]; + bart_dim_t tri_dims[N]; complex float* tri = hermite_to_uppertriag(6, 6, 6, N, tri_dims, sqr_bas_dims, sqr); md_free(sqr); @@ -133,23 +133,23 @@ static struct nufft_conf_s psf_conf(bool periodic, bool lowmem, bool vptr) } /* The adjoint transform of ones, which is what a point spread function is. */ -static complex float* psf_int(int N, const long img_dims[N], const long trj_dims[N], const complex float* traj, - const long bas_dims[N], const complex float* basis, - const long wgh_dims[N], const complex float* weights, +static complex float* psf_int(int N, const bart_dim_t img_dims[N], const bart_dim_t trj_dims[N], const complex float* traj, + const bart_dim_t bas_dims[N], const complex float* basis, + const bart_dim_t wgh_dims[N], const complex float* weights, bool periodic, bool lowmem, bool upper_triag) { - long ksp_dims[N]; + bart_dim_t ksp_dims[N]; md_select_dims(N, ~MD_BIT(0), ksp_dims, trj_dims); if (NULL != weights) - md_max_dims(N, ~0UL, ksp_dims, ksp_dims, wgh_dims); + md_max_dims(N, ~UINT64_C(0), ksp_dims, ksp_dims, wgh_dims); - long sqr_bas_dims[N]; + bart_dim_t sqr_bas_dims[N]; complex float* sqr_basis = square_basis(upper_triag, N, sqr_bas_dims, bas_dims, basis, ksp_dims); complex float* sqr_weights = square_weights(N, wgh_dims, weights); - long img_dims2[N]; + bart_dim_t img_dims2[N]; md_copy_dims(N, img_dims2, img_dims); if (upper_triag) { @@ -185,9 +185,9 @@ static complex float* psf_int(int N, const long img_dims[N], const long trj_dims return psf; } -complex float* compute_psf(int N, const long img_dims[N], const long trj_dims[N], const complex float* traj, - const long bas_dims[N], const complex float* basis, - const long wgh_dims[N], const complex float* weights, +complex float* compute_psf(int N, const bart_dim_t img_dims[N], const bart_dim_t trj_dims[N], const complex float* traj, + const bart_dim_t bas_dims[N], const complex float* basis, + const bart_dim_t wgh_dims[N], const complex float* weights, bool periodic, bool lowmem) { return psf_int(N, img_dims, trj_dims, traj, bas_dims, basis, wgh_dims, weights, @@ -196,16 +196,16 @@ complex float* compute_psf(int N, const long img_dims[N], const long trj_dims[N] /* On the grid twice over, which is where a convolution the size of the image * has room to be one. */ -complex float* compute_psf2(int N, const long psf_dims[N + 1], unsigned long flags, const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, const long wgh_dims[N + 1], const complex float* weights, +complex float* compute_psf2(int N, const bart_dim_t psf_dims[N + 1], bart_flags_t flags, const bart_dim_t trj_dims[N + 1], const complex float* traj, + const bart_dim_t bas_dims[N + 1], const complex float* basis, const bart_dim_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag) { int ND = N + 1; - long img_dims[ND]; + bart_dim_t img_dims[ND]; md_select_dims(ND, ~MD_BIT(N + 0), img_dims, psf_dims); - long img2_dims[ND]; + bart_dim_t img2_dims[ND]; md_copy_dims(ND, img2_dims, img_dims); for (int i = 0; i < N; i++) @@ -224,7 +224,7 @@ complex float* compute_psf2(int N, const long psf_dims[N + 1], unsigned long fla complex float* psf = md_alloc_sameplace(ND, psf_dims, CFL_SIZE, traj); - long factors[N]; + bart_dim_t factors[N]; for (int i = 0; i < N; i++) factors[i] = ((img_dims[i] > 1) && (MD_IS_SET(flags, i))) ? 2 : 1; @@ -236,7 +236,7 @@ complex float* compute_psf2(int N, const long psf_dims[N + 1], unsigned long fla return psf; } -static void psf_factors(int N, unsigned long flags, long factors[N], const long dims[N]) +static void psf_factors(int N, bart_flags_t flags, bart_dim_t factors[N], const bart_dim_t dims[N]) { flags = flags & md_nontriv_dims(N, dims); @@ -247,7 +247,7 @@ static void psf_factors(int N, unsigned long flags, long factors[N], const long /* The shift of one set of frequencies, as `nufft.c` computes it. Shared * because the mask a compressed function keeps is gridded at the same shifts * the function itself was decomposed at. */ -void bartorch_psf_shift(int NS, float shift[NS], int N, const long factors[N], int idx) +void bartorch_psf_shift(int NS, float shift[NS], int N, const int64_t factors[N], int idx) { assert(NS <= N); @@ -272,34 +272,34 @@ void bartorch_psf_shift(int NS, float shift[NS], int N, const long factors[N], i * samples reach, and the shape it takes once only those are kept. */ struct psf_packing { - const long* com_dims; - const long* idx; - const long* com_psf_dims; /* the whole compressed function */ - const long* com_psf_dims3; /* one set of frequencies of it */ + const bart_dim_t* com_dims; + const bart_dim_t* idx; + const bart_dim_t* com_psf_dims; /* the whole compressed function */ + const bart_dim_t* com_psf_dims3; /* one set of frequencies of it */ }; static complex float* psf_decomposed(bool to_host, bool real_out, const struct psf_packing* pack, - int N, const long psf_dims[N + 1], unsigned long flags, const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, const long wgh_dims[N + 1], const complex float* weights, + int N, const bart_dim_t psf_dims[N + 1], bart_flags_t flags, const bart_dim_t trj_dims[N + 1], const complex float* traj, + const bart_dim_t bas_dims[N + 1], const complex float* basis, const bart_dim_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag) { assert(to_host || !real_out); int ND = N + 1; - long ksp_dims[ND]; + bart_dim_t ksp_dims[ND]; md_select_dims(ND, ~MD_BIT(0), ksp_dims, trj_dims); ksp_dims[N] = psf_dims[N]; if (NULL != weights) - md_max_dims(ND, ~0UL, ksp_dims, ksp_dims, wgh_dims); + md_max_dims(ND, ~UINT64_C(0), ksp_dims, ksp_dims, wgh_dims); - long sqr_bas_dims[ND]; + bart_dim_t sqr_bas_dims[ND]; complex float* sqr_basis = square_basis(upper_triag, ND, sqr_bas_dims, bas_dims, basis, ksp_dims); complex float* sqr_weights = square_weights(ND, wgh_dims, weights); - long psf_dims2[ND]; + bart_dim_t psf_dims2[ND]; md_copy_dims(ND, psf_dims2, psf_dims); if (upper_triag) { @@ -314,11 +314,11 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p struct nufft_conf_s conf = psf_conf(periodic, lowmem, is_vptr(traj)); - long trj_dims2[ND]; + bart_dim_t trj_dims2[ND]; md_copy_dims(ND, trj_dims2, trj_dims); trj_dims2[N] = psf_dims2[N]; - long factors[ND]; + bart_dim_t factors[ND]; psf_factors(ND, flags, factors, psf_dims); complex float tp[trj_dims2[N]][trj_dims2[0]]; @@ -332,9 +332,9 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p tp[k][j] = (1 != psf_dims2[j] ? 0.5 * psf_dims2[j] : 0.) + shift[j]; } - long ksp_dims2[ND]; - long psf_dims3[ND]; - long trj_dims3[ND]; + bart_dim_t ksp_dims2[ND]; + bart_dim_t psf_dims3[ND]; + bart_dim_t trj_dims3[ND]; md_select_dims(ND, ~MD_BIT(N), ksp_dims2, ksp_dims); md_select_dims(ND, ~MD_BIT(N), psf_dims3, psf_dims2); @@ -348,14 +348,14 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p * which is far cheaper than allocating it page-locked. A real function * is made real here, an entry at a time, so what crosses is half of what * it would be. */ - const long* whole_dims = (NULL != pack) ? pack->com_psf_dims : psf_dims; + const bart_dim_t* whole_dims = (NULL != pack) ? pack->com_psf_dims : psf_dims; size_t out_size = real_out ? FL_SIZE : CFL_SIZE; complex float* psf = to_host - ? bartorch_host_alloc(md_calc_size(ND, whole_dims) * (long)out_size, 0) + ? bartorch_host_alloc(md_calc_size(ND, whole_dims) * (bart_dim_t)out_size, 0) : md_alloc_sameplace(ND, whole_dims, CFL_SIZE, traj); - long psf_coset = md_calc_size(ND, (NULL != pack) ? pack->com_psf_dims3 : psf_dims3); + bart_dim_t psf_coset = md_calc_size(ND, (NULL != pack) ? pack->com_psf_dims3 : psf_dims3); /* One coefficient of the function at a time. * @@ -364,20 +364,20 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p * together: at 256^3 with four coefficients that is ten images where * one would do. Each entry is its own gridding of the samples weighted * by its own pair of basis coefficients. */ - long one_dims[ND]; - long one_bas_dims[ND]; + bart_dim_t one_dims[ND]; + bart_dim_t one_bas_dims[ND]; md_copy_dims(ND, one_dims, psf_dims3); md_copy_dims(ND, one_bas_dims, sqr_bas_dims); - long pairs = psf_dims3[COEFF_DIM]; + bart_dim_t pairs = psf_dims3[COEFF_DIM]; one_dims[COEFF_DIM] = 1; one_bas_dims[COEFF_DIM] = 1; - long pair_stride = md_calc_size(ND, one_dims); - long basis_stride = (NULL == sqr_basis) ? 0 : md_calc_size(ND, one_bas_dims); - long out_pair_stride = (NULL != pack) ? md_calc_size(ND, pack->com_psf_dims3) / pairs : pair_stride; + bart_stride_t pair_stride = md_calc_size(ND, one_dims); + bart_stride_t basis_stride = (NULL == sqr_basis) ? 0 : md_calc_size(ND, one_bas_dims); + bart_stride_t out_pair_stride = (NULL != pack) ? md_calc_size(ND, pack->com_psf_dims3) / pairs : pair_stride; /* One transform for the whole function, over the trajectory as it is, * with neither weights nor basis: both are per-sample factors and go @@ -389,8 +389,8 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p struct linop_s* op = nufft_create2(ND, ksp_dims2, one_dims, trj_dims3, traj, NULL, NULL, NULL, NULL, conf); - long kstrs[ND]; - long tstrs[ND]; + bart_stride_t kstrs[ND]; + bart_stride_t tstrs[ND]; md_calc_strides(ND, kstrs, ksp_dims2, CFL_SIZE); md_calc_strides(ND, tstrs, trj_dims3, CFL_SIZE); @@ -406,7 +406,7 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p complex float* ramp_h = md_alloc(ND, one_dims, CFL_SIZE); complex float* ramp = on_device ? md_alloc_sameplace(ND, one_dims, CFL_SIZE, traj) : ramp_h; - long one_com[ND]; + bart_dim_t one_com[ND]; if (NULL != pack) { @@ -414,7 +414,7 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p one_com[COEFF_DIM] = 1; } - const long* made_dims = (NULL != pack) ? one_com : one_dims; + const bart_dim_t* made_dims = (NULL != pack) ? one_com : one_dims; complex float* packed = (NULL != pack) ? md_alloc_sameplace(ND, one_com, CFL_SIZE, traj) : NULL; float* packed_real = real_out ? md_alloc_sameplace(ND, made_dims, FL_SIZE, traj) : NULL; @@ -466,7 +466,7 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p * one set of frequencies: one image, gridded, transformed, and * -- where a compressed function was asked for -- reduced to the * places the samples reach before the next one is made. */ - for (long q = 0; q < pairs; q++) { + for (bart_dim_t q = 0; q < pairs; q++) { if (NULL != sqr_basis) md_zmulc2(ND, ksp_dims2, kstrs, kern_q, kstrs, kern, @@ -522,8 +522,8 @@ static complex float* psf_decomposed(bool to_host, bool real_out, const struct p return psf; } -complex float* compute_psf2_decomposed(int N, const long psf_dims[N + 1], unsigned long flags, const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, const long wgh_dims[N + 1], const complex float* weights, +complex float* compute_psf2_decomposed(int N, const bart_dim_t psf_dims[N + 1], bart_flags_t flags, const bart_dim_t trj_dims[N + 1], const complex float* traj, + const bart_dim_t bas_dims[N + 1], const complex float* basis, const bart_dim_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag) { return psf_decomposed(false, false, NULL, N, psf_dims, flags, trj_dims, traj, bas_dims, basis, @@ -532,11 +532,11 @@ complex float* compute_psf2_decomposed(int N, const long psf_dims[N + 1], unsign /* The same function, left where the card is not: neither it nor any set of * frequencies but the one being made is ever resident. */ -complex float* bartorch_psf_to_host(int N, const long psf_dims[N + 1], unsigned long flags, const long trj_dims[N + 1], const complex float* traj, - const long bas_dims[N + 1], const complex float* basis, const long wgh_dims[N + 1], const complex float* weights, +complex float* bartorch_psf_to_host(int N, const int64_t psf_dims[N + 1], uint64_t flags, const int64_t trj_dims[N + 1], const complex float* traj, + const int64_t bas_dims[N + 1], const complex float* basis, const int64_t wgh_dims[N + 1], const complex float* weights, bool periodic, bool lowmem, bool upper_triag, - const long com_dims[N + 1], const long* idx, - const long com_psf_dims[N + 1], const long com_psf_dims3[N + 1], int real) + const int64_t com_dims[N + 1], const int64_t* idx, + const int64_t com_psf_dims[N + 1], const int64_t com_psf_dims3[N + 1], int real) { struct psf_packing pack = { com_dims, idx, com_psf_dims, com_psf_dims3 }; diff --git a/tests/test_abi.py b/tests/test_abi.py index 6a799b15..e7cf534a 100644 --- a/tests/test_abi.py +++ b/tests/test_abi.py @@ -8,7 +8,9 @@ """ import ast +import ctypes import importlib +import re import sys from pathlib import Path @@ -104,3 +106,18 @@ def test_the_generator_refuses_a_type_it_has_not_been_taught(): def test_bartorch_imports_without_naming_ctypes_at_the_top_level(): """The package's own surface is shapes and tensors.""" importlib.import_module("bartorch") + + +def test_the_header_names_no_type_whose_width_is_the_platforms(): + # long is 32 bits on Windows and 64 elsewhere, so a shape or a stride + # carried in one would be truncated there and not here. + code = gen_abi.strip_directives(gen_abi.strip_comments(gen_abi.HEADER.read_text())) + assert not re.search(r"\blong\b", code) + + +def test_build_info_reports_the_platforms_data_model(): + import bartorch + + info = dict(item.split("=", 1) for item in bartorch.build_info().split(",")) + assert int(info["long"]) == ctypes.sizeof(ctypes.c_long) + assert info["long"] == ("4" if sys.platform == "win32" else "8") diff --git a/tests/test_nlop_step.py b/tests/test_nlop_step.py index 3a632472..377e397b 100644 --- a/tests/test_nlop_step.py +++ b/tests/test_nlop_step.py @@ -488,7 +488,11 @@ def test_a_model_of_items_steps_each_as_it_would_step_alone(off_grid): kspace = torch.stack([single.forward(*pair) for pair in zip(*_phantoms(items, n, coils))]) def run(F, y): - block = nlop.IRGNMBlock(cg_maxiter=30) + # A tolerance stops each item's conjugate gradients on its own residual. + # At zero, BART's batched solve stops an item only once its residual is + # exactly zero and runs until every item's is, so whether an item waits + # on the others is a question of underflow. + block = nlop.IRGNMBlock(cg_maxiter=30, cg_tol=1e-4) state = block.start(y, F) for _ in range(4): state = block(state, F)