Relocate absolute pointers in module constants

.github/workflows/ci.yml

@@ -362,4 +362,14 @@ jobs:
 
       - name: Test external module with dmod_loader
         run: |
-          ${GITHUB_WORKSPACE}/build/examples/system/dmod_loader ${GITHUB_WORKSPACE}/build/dmf/external_test_make.dmf
+          ${GITHUB_WORKSPACE}/build/examples/system/dmod_loader ${GITHUB_WORKSPACE}/build/dmf/external_test_make.dmf
+  arm-pointer-relocations:
+    name: ARM absolute pointer relocation metadata
+    runs-on: ubuntu-latest
+    container:
+      image: chocotechnologies/dmod:1.0.4
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          command -v arm-none-eabi-gcc
+          python3 tests/integration/test_arm_pointer_relocations.py

docs/absolute-pointer-relocations.md

@@ -0,0 +1,29 @@
+# Absolute pointer relocations on ARM and x86-64
+
+PIC alone does not relocate pointers embedded in static aggregates. GCC can
+also create such aggregates while optimizing a local structure initializer.
+Callbacks and third-party constant dispatch tables can therefore contain link
+offsets even when every instruction is position independent.
+
+After `dmod_setup_external_module()`, call `dmod_enable_pointer_relocations()`
+before defining modules. The SDK preserves ELF relocation metadata and
+records R_ARM_ABS32/R_ARM_TARGET1 or R_X86_64_64 pointer locations in the DMF. It leaves the
+existing header, API, GOT and license relocations to their existing handlers.
+Non-pointer integers are not guessed or scanned for possible addresses.
+Unsupported absolute instruction relocations fail the build.
+
+The implementation supports ARM ELF32 and x86-64 ELF64 and needs Python 3 at build time. Linker relaxation is disabled so the static link cannot turn GOT references into absolute instruction immediates. Use host tools and a
+runtime built from an SDK supporting this feature. An opted-in module has a
+256-byte header: the unchanged 240-byte legacy header followed by a 16-byte
+extension (magic 0x524c4f43, version 1, table byte offset, record count). The
+legacy HeaderSize and footer header-section size describe the extended header.
+The table is appended after the original binary image and contains sorted
+32-bit byte offsets. Each identifies a pointer-sized module-relative value.
+The loader validates the entire table before applying it after the existing
+section relocations. Thumb function pointer bits are preserved. Crossplatform
+inspection validates the directory without writing native-size pointers.
+
+Older loaders reject the extended HeaderSize. Legacy modules retain their
+original format and still load with the updated runtime. This is opt-in and
+currently limited to ARM ELF32 and x86-64 ELF64; other architectures need their own reviewed
+relocation mapping before enabling it.

inc/dmod_types.h

@@ -175,6 +175,18 @@ typedef struct
     bool                ManualLoad;  
 } Dmod_ModuleHeader_t;
 
+/* Optional extension, immediately following the legacy module header. Older
+ * loaders reject the larger HeaderSize instead of executing unrelocated code.
+ * Records are sorted uint32_t offsets of pointer-sized module-relative values. */
+#define DMOD_POINTER_RELOCATIONS_MAGIC 0x524C4F43u
+#define DMOD_POINTER_RELOCATIONS_VERSION 1u
+typedef struct {
+    uint32_t Magic;
+    uint32_t Version;
+    uint32_t Offset;
+    uint32_t Count;
+} Dmod_PointerRelocations_t;
+
 /**
  * @brief DMFC Header
  * 

inc/private/dmod_ldr.h

@@ -16,9 +16,11 @@ extern bool Dmod_Ldr_LoadFooter( Dmod_Context_t* Context );
 extern bool Dmod_Ldr_LoadOutput( Dmod_Context_t* Context );
 extern bool Dmod_Ldr_LoadInput( Dmod_Context_t* Context );
 extern bool Dmod_Ldr_LoadGot( Dmod_Context_t* Context );
+extern bool Dmod_Ldr_LoadPointerRelocations(Dmod_Context_t* Context);
 extern bool Dmod_Ldr_LoadBss( Dmod_Context_t* Context );
 
 #ifdef __cplusplus
 }
 #endif
+
 #endif // DMOD_LDR_H

scripts/CMakeLists.txt

@@ -1,3 +1,8 @@
+# Enable SDK-generated relocations for absolute pointers in module constants.
+function(dmod_enable_pointer_relocations)
+    set(DMOD_RELOCATE_ABSOLUTE_POINTERS ON PARENT_SCOPE)
+endfunction()
+
 if(NOT DEFINED DMOD_SCRIPTS_DIR)
     set(DMOD_SCRIPTS_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Path to the scripts directory")
 endif()
@@ -768,6 +773,17 @@ function(dmod_create_module moduleName version moduleType)
     target_link_options(${moduleName} PRIVATE -L ${DMOD_SCRIPTS_DIR} -pie -nostartfiles -nostdlib -Xlinker --discard-all -Wl,--gc-sections,--undefined=ModuleHeader )
     target_link_options(${moduleName} PRIVATE -T ${DMOD_SCRIPTS_DIR}/module.ld  -static)
 
+    # Static initializers can contain absolute pointers outside the GOT,
+    # including compiler-generated aggregate constants and third-party tables.
+    # Opt in until all deployed loaders understand the extended header.
+    if(DMOD_RELOCATE_ABSOLUTE_POINTERS)
+        find_package(Python3 REQUIRED COMPONENTS Interpreter)
+        target_compile_definitions(${moduleName} PRIVATE DMOD_RELOCATE_ABSOLUTE_POINTERS=1)
+        target_link_options(${moduleName} PRIVATE -Wl,--emit-relocs,--no-relax)
+        set_property(TARGET ${moduleName} APPEND PROPERTY LINK_DEPENDS
+            ${DMOD_SCRIPTS_DIR}/relocate_pointers.py)
+    endif()
+
     set(ELF_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${moduleName}")
     set(DMF_OUTPUT "${DMOD_DMF_DIR}/${moduleName}.dmf")
     set(DMFC_OUTPUT "${DMOD_DMFC_DIR}/${moduleName}.dmfc")
@@ -781,6 +797,13 @@ function(dmod_create_module moduleName version moduleType)
         COMMENT "Converting ELF to binary"
     )
 
+    if(DMOD_RELOCATE_ABSOLUTE_POINTERS)
+        add_custom_command(TARGET ${moduleName} POST_BUILD
+            COMMAND ${Python3_EXECUTABLE} ${DMOD_SCRIPTS_DIR}/relocate_pointers.py
+                ${ELF_OUTPUT} ${DMF_OUTPUT}
+            COMMENT "Recording absolute pointer relocations" VERBATIM)
+    endif()
+
     if(TODMFC)
         add_custom_command(
             TARGET ${moduleName} POST_BUILD

scripts/dmod_header.c.in

@@ -61,4 +61,10 @@ volatile const Dmod_ModuleHeader_t ModuleHeader DMOD_SECTION(".header") DMOD_USE
     .ManualLoad         = @DMOD_MANUAL_LOAD@,
 };
 
-volatile const Dmod_ModuleHeader_t* DMOD_Header DMOD_GLOBAL_POINTER = &ModuleHeader;
+volatile const Dmod_ModuleHeader_t* DMOD_Header DMOD_GLOBAL_POINTER = &ModuleHeader;
+#if defined(DMOD_RELOCATE_ABSOLUTE_POINTERS) && DMOD_RELOCATE_ABSOLUTE_POINTERS
+volatile const Dmod_PointerRelocations_t Dmod_PointerRelocations
+    DMOD_SECTION(".header.relocations") DMOD_USED = {
+        DMOD_POINTER_RELOCATIONS_MAGIC, DMOD_POINTER_RELOCATIONS_VERSION, 0, 0
+    };
+#endif

scripts/install-sdk.cmake

@@ -31,7 +31,7 @@ install(DIRECTORY "${DMOD_DIR}/lib/third-party/FastLZ"
     PATTERN "LICENSE.MIT")
 install(DIRECTORY "${DMOD_DIR}/scripts/"
     DESTINATION "${DMOD_SDK_INSTALL_DIR}/scripts" COMPONENT development
-    FILES_MATCHING PATTERN "CMakeLists.txt" PATTERN "*.in" PATTERN "*.ld"
+    FILES_MATCHING PATTERN "CMakeLists.txt" PATTERN "*.in" PATTERN "*.ld" PATTERN "*.py"
     PATTERN "memory_analysis.cmake")
 install(DIRECTORY "${DMOD_DIR}/cmake/runtime"
     DESTINATION "${DMOD_SDK_INSTALL_DIR}/cmake" COMPONENT development)

scripts/module.ld

@@ -17,6 +17,7 @@ SECTIONS
     {
         __header_start = .;
         *(.header)
+        KEEP(*(.header.relocations))
         __header_end = .;
     } : header
 

scripts/relocate_pointers.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Add explicit absolute pointer fixups to an ARM ELF32 or x86-64 DMF (stdlib only).
+
+--emit-relocs preserves linker-resolved references. PC-relative references need
+no fixup. Existing header/API/GOT/license relocations remain owned by the loader.
+Unknown absolute instruction relocations are rejected instead of silently
+producing a non-position-independent module.
+"""
+import struct
+import sys
+from pathlib import Path
+
+
+def convert(elf, binary):
+    if len(elf) < 64 or elf[:4] != b'\x7fELF' or elf[5] != 1 or elf[4] not in (1,2):
+        raise ValueError('Expected little-endian ELF32/ELF64')
+    machine = struct.unpack_from('<H', elf, 18)[0]
+    wide = elf[4] == 2
+    if (machine,wide) not in ((40,False),(62,True)):
+        raise ValueError('Expected ARM ELF32 or x86-64 ELF64')
+    width = 8 if wide else 4
+    shoff = struct.unpack_from('<Q' if wide else '<I',elf,40 if wide else 32)[0]
+    entsize,count,names_index = struct.unpack_from('<HHH',elf,58 if wide else 46)
+    section_format = '<IIQQQQIIQQ' if wide else '<10I'
+    if entsize != struct.calcsize(section_format) or not count or names_index >= count:
+        raise ValueError('Invalid section directory')
+    sections = [struct.unpack_from(section_format,elf,shoff+i*entsize) for i in range(count)]
+    names = sections[names_index]
+    strings = elf[names[4]:names[4] + names[5]]
+    def name(section):
+        start = section[0]
+        return strings[start:strings.index(b'\0', start)].decode('ascii')
+    by_name = {name(s): s for s in sections}
+    header = by_name['.header']
+    if header[3] != 0 or header[5] != 256:
+        raise ValueError('Expected legacy 240-byte header plus 16-byte relocation extension')
+    if struct.unpack_from('<I', binary)[0] != 0x444d4f44 or struct.unpack_from('<I', binary, 12)[0] != width:
+        raise ValueError('Invalid DMF pointer size')
+    if struct.unpack_from('<2I', binary, 240) != (0x524c4f43, 1):
+        raise ValueError('Missing relocation extension')
+    excluded = [(s[3], s[3] + s[5]) for n, s in by_name.items()
+                if n in ('.header', '.inputs', '.outputs', '.got', '.footer')]
+    # License.Text has its own historical loader relocation.
+    for section in sections:
+        if section[1] != 2:
+            continue
+        strings_section = sections[section[6]]
+        symbol_names = elf[strings_section[4]:strings_section[4] + strings_section[5]]
+        for pos in range(section[4], section[4] + section[5], 24 if wide else 16):
+            if wide:
+                n,info,other,index,value,size = struct.unpack_from('<IBBHQQ',elf,pos)
+            else:
+                n,value,size,info,other,index = struct.unpack_from('<IIIBBH',elf,pos)
+            if symbol_names[n:symbol_names.index(b'\0', n)] == b'License' and size >= width:
+                excluded.append((value + size - width, value + size))
+    offsets = set()
+    for section in sections:
+        if section[1] != (4 if wide else 9): # SHT_RELA / SHT_REL
+            continue
+        target = sections[section[7]]
+        if not target[2] & 2:
+            continue
+        symbols = sections[section[6]]
+        record_size = 24 if wide else 8
+        if section[9] != record_size:
+            raise ValueError('Invalid relocation record size')
+        for pos in range(section[4],section[4]+section[5],record_size):
+            offset,info = struct.unpack_from('<QQ' if wide else '<II',elf,pos)
+            kind,symbol = (info&0xffffffff,info>>32) if wide else (info&255,info>>8)
+            if any(start <= offset < end for start,end in excluded):
+                continue
+            symbol_index = struct.unpack_from('<H',elf,symbols[4]+symbol*(24 if wide else 16)+(6 if wide else 14))[0]
+            if symbol_index in (0,0xfff1):
+                continue
+            absolute = (1,) if wide else (2,38)
+            relative = (0,2,4,9,24,25,26,27,28,29,41,42) if wide else (0,3,10,24,25,26,28,29,30,42,96,51,102,103)
+            if kind in absolute:
+                if offset+width > len(binary):
+                    raise ValueError('Pointer relocation outside image')
+                value = struct.unpack_from('<Q' if wide else '<I',binary,offset)[0]
+                if value >= len(binary):
+                    raise ValueError('Absolute pointer outside module')
+                offsets.add(offset)
+            elif kind not in relative:
+                raise ValueError(f'Unsupported relocation {kind} at {offset:#x}')
+    result = bytearray(binary)
+    while len(result) % 4:
+        result.append(0)
+    table = len(result)
+    for offset in sorted(offsets):
+        result.extend(struct.pack('<I', offset))
+    struct.pack_into('<I', result, 4, header[5])
+    struct.pack_into('<II', result, 248, table, len(offsets))
+    return result, len(offsets)
+
+
+def main():
+    elf, output = map(Path, sys.argv[1:])
+    result, count = convert(elf.read_bytes(), output.read_bytes())
+    output.write_bytes(result)
+    print(f'{count} absolute pointer relocations recorded')
+
+
+if __name__ == '__main__':
+    main()

src/system/private/dmod_ldr.c

@@ -40,7 +40,8 @@ bool Dmod_Ldr_LoadHeader( Dmod_Context_t* Context )
     }
 
     // check header size
-    if( header->HeaderSize != sizeof( Dmod_ModuleHeader_t ) )
+    if( header->HeaderSize != sizeof( Dmod_ModuleHeader_t )
+     && header->HeaderSize != sizeof(Dmod_ModuleHeader_t) + sizeof(Dmod_PointerRelocations_t) )
     {
         DMOD_LOG_ERROR("Cannot load header - invalid header size: %d != %d\n", header->HeaderSize, sizeof( Dmod_ModuleHeader_t ) );
         return false;
@@ -151,7 +152,7 @@ bool Dmod_Ldr_LoadFooter( Dmod_Context_t* Context )
 
     Dmod_ModuleFooter_t* footer = Context->Header->Footer.Ptr;
 
-    if( footer->Header.SectionStart != 0 || footer->Header.SectionSize != sizeof( Dmod_ModuleHeader_t ) )
+    if( footer->Header.SectionStart != 0 || footer->Header.SectionSize != Context->Header->HeaderSize )
     {
         DMOD_LOG_ERROR("Cannot load footer - invalid header section\n");
         return false;
@@ -593,6 +594,50 @@ bool Dmod_Ldr_LoadBss( Dmod_Context_t* Context )
     return true;
 }
 
+/* Validate the whole table before applying any fixup. Do not trust counts,
+ * offsets or pointer alignment from a downloaded module. */
+bool Dmod_Ldr_LoadPointerRelocations(Dmod_Context_t* Context)
+{
+    if(Context == NULL || Context->Header == NULL) return false;
+    if(Context->Header->HeaderSize == sizeof(Dmod_ModuleHeader_t)) return true;
+    if(Context->Header->HeaderSize != sizeof(Dmod_ModuleHeader_t) + sizeof(Dmod_PointerRelocations_t)
+       || Context->Size < Context->Header->HeaderSize) return false;
+    Dmod_PointerRelocations_t extension;
+    memcpy(&extension, (uint8_t*)Context->Data + sizeof(Dmod_ModuleHeader_t), sizeof(extension));
+    if(extension.Magic != DMOD_POINTER_RELOCATIONS_MAGIC
+       || extension.Version != DMOD_POINTER_RELOCATIONS_VERSION
+       || extension.Offset < Context->Header->HeaderSize
+       || extension.Offset > Context->Size
+       || extension.Count > (Context->Size - extension.Offset) / sizeof(uint32_t)) return false;
+    /* Both native loading and crossplatform inspection validate every record.
+     * Only native loading applies the already validated pointer fixups. */
+    size_t width = Context->Header->PointerSize;
+    if(width != 4 && width != 8) return false;
+    if(!Dmod_SystemCrossplatformMode && width != sizeof(uintptr_t)) return false;
+    uint32_t previous = 0;
+    for(size_t i = 0; i < extension.Count; ++i) {
+        uint32_t offset;
+        memcpy(&offset, (uint8_t*)Context->Data + extension.Offset + i * sizeof(offset), sizeof(offset));
+        if(offset < Context->Header->HeaderSize || offset > extension.Offset
+           || width > extension.Offset - offset || (i && (offset <= previous || offset - previous < width))) return false;
+        uint64_t value = 0;
+        memcpy(&value, (uint8_t*)Context->Data + offset, width);
+        if(value >= extension.Offset) return false;
+        if(!Dmod_SystemCrossplatformMode && value > UINTPTR_MAX - (uintptr_t)Context->Data) return false;
+        previous = offset;
+    }
+    if(!Dmod_SystemCrossplatformMode) {
+        for(size_t i = 0; i < extension.Count; ++i) {
+            uint32_t offset; uintptr_t value;
+            memcpy(&offset, (uint8_t*)Context->Data + extension.Offset + i * sizeof(offset), sizeof(offset));
+            memcpy(&value, (uint8_t*)Context->Data + offset, width);
+            value += (uintptr_t)Context->Data;
+            memcpy((uint8_t*)Context->Data + offset, &value, width);
+        }
+    }
+    return true;
+}
+
 /**
  * @brief Load module
  * 
@@ -612,7 +657,8 @@ bool Dmod_Ldr_Load( Dmod_Context_t* Context )
      || !Dmod_Ldr_LoadOutput( Context )
      || !Dmod_Ldr_LoadInput( Context )
      || !Dmod_Ldr_LoadGot( Context )
-     || !Dmod_Ldr_LoadBss( Context ) )
+     || !Dmod_Ldr_LoadBss( Context )
+     || !Dmod_Ldr_LoadPointerRelocations( Context ) )
     {
         return false;
     }