Feature/preallocate large header

pom.xml

@@ -216,6 +216,11 @@
     </build>
 
     <dependencies>
+        <dependency>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+            <version>1.3.6</version>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>

src/main/java/com/riscure/trs/ReadOnlyTraceSet.java

@@ -22,7 +22,7 @@ public class ReadOnlyTraceSet extends TraceSet {
     private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d).";
     private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'";
     // This is excessive for the header, but it's only the initial maximum
-    private static final long MAX_METADATA_SIZE = 100_000_000L;
+    private static final long INITIAL_MEMORY_SIZE = 100_000_000L;
 
     private final int metaDataSize;
     private final FileInputStream readStream;
@@ -44,7 +44,7 @@ public class ReadOnlyTraceSet extends TraceSet {
 
         //the file might be bigger than the buffer, in which case we partially buffer it in memory
         this.fileSize = channel.size();
-        long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE);
+        long initialBufferSize = Math.min(fileSize, INITIAL_MEMORY_SIZE);
 
         this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize);
         this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer);
@@ -129,6 +129,19 @@ private long calculateTraceSize() {
     public void close() throws IOException, TRSFormatException {
         super.close();
         closeReader();
+        awaitFileUnmapping();
+    }
+
+    private static void awaitFileUnmapping() throws IOException {
+        // Unfortunately, the current solution requires a garbage collect to have been performed before the issue is resolved.
+        // Other fixes required either a Java 8 Cleaner.clean() call not accessible from Java 21, or a Java 20 Arena.close(),
+        // which is not been finalized in Java 21.
+        System.gc();
+        try {
+            Thread.sleep(100);
+        } catch (InterruptedException e) {
+            throw new IOException(e);
+        }
     }
 
     @Override

src/main/java/com/riscure/trs/TRSMetaData.java

@@ -29,6 +29,19 @@ private void init() {
         }
     }
 
+    /**
+     * @return a modifiable copy of this metadata object
+     */
+    public TRSMetaData modifiable() {
+        TRSMetaData copy = new TRSMetaData();
+        for (TRSTag tag : TRSTag.values()) {
+            copy.put(tag, get(tag));
+        }
+        copy.put(TRSTag.TRACE_SET_PARAMETERS, getTraceSetParameters().copy());
+        copy.put(TRSTag.TRACE_PARAMETER_DEFINITIONS, getTraceParameterDefinitions().copy());
+        return copy;
+    }
+
     /**
      * Add the data associated with the supplied tag to this metadata.
      * This will overwrite any existing value

src/main/java/com/riscure/trs/TRSMetaDataUtils.java

@@ -7,15 +7,76 @@
 
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 
+import static com.riscure.trs.TraceSet.DEFAULT_METADATA_SIZE;
+
 public class TRSMetaDataUtils {
     private static final String IGNORED_UNKNOWN_TAG = "ignored unknown metadata tag '%02X' while reading a TRS file\n";
     private static final String TAG_LENGTH_INVALID = "The length field following tag '%s' has value '%X', which is not between 0 and 0xffff";
     private static final String UNSUPPORTED_TAG_TYPE = "Unsupported tag type for tag '%s': %s";
     private static final String REWINDING_STREAM = "The output stream is not at the start of the file. Rewinding stream.";
 
+    /**
+     * Writes the provided TRS metadata to the stream.
+     *
+     * @param raf                 the file output opened in random access mode
+     * @param metaData            the metadata to write
+     * @throws IOException        if any write error occurs
+     * @throws TRSFormatException if the metadata contains unsupported tags
+     */
+    public static void writeTRSMetaData(RandomAccessFile raf, TRSMetaData metaData) throws IOException, TRSFormatException {
+        // We always write at the start of the file
+        raf.seek(0);
+        for (TRSTag tag : TRSTag.values()) {
+            if (tag.equals(TRSTag.TRACE_BLOCK) || tag.equals(TRSTag.PADDING)) continue; //PADDING and TRACE BLOCK should be the last writes
+            if (!tag.isRequired() && metaData.hasDefaultValue(tag)) continue; //ignore if default and not required
+            raf.write(tag.getValue());
+            if (tag.getType() == String.class) {
+                String s = metaData.getString(tag);
+                byte[] stringBytes = s.getBytes(StandardCharsets.UTF_8);
+                writeLength(raf, stringBytes.length);
+                raf.write(stringBytes);
+            } else if (tag.getType() == Float.class) {
+                float f = metaData.getFloat(tag);
+                writeLength(raf, tag.getLength());
+                writeInt(raf, Float.floatToIntBits(f), tag.getLength());
+            } else if (tag.getType() == Boolean.class) {
+                int value = metaData.getBoolean(tag) ? 1 : 0;
+                writeLength(raf, tag.getLength());
+                writeInt(raf, value, tag.getLength());
+            } else if (tag.getType() == Integer.class) {
+                writeLength(raf, tag.getLength());
+                writeInt(raf, metaData.getInt(tag), tag.getLength());
+            } else if (tag.getType() == TraceSetParameterMap.class) {
+                byte[] serialized = metaData.getTraceSetParameters().serialize();
+                writeLength(raf, serialized.length);
+                raf.write(serialized);
+            } else if (tag.getType() == TraceParameterDefinitionMap.class) {
+                byte[] serialized = metaData.getTraceParameterDefinitions().serialize();
+                writeLength(raf, serialized.length);
+                raf.write(serialized);
+            } else {
+                throw new TRSFormatException(String.format(UNSUPPORTED_TAG_TYPE, tag.getName(), tag.getType()));
+            }
+        }
+        // Grow the metadata up to 1M, creating an empty buffer in the trace set
+        // This allows us to grow the header without having to rewrite the whole file
+        if (raf.getChannel().position() < DEFAULT_METADATA_SIZE) {
+            raf.write(TRSTag.PADDING.getValue());
+            int expectedLength = (int) (DEFAULT_METADATA_SIZE - raf.getChannel().position());
+            // The length of the padding will be the maximum size minus the current position minus the number of bytes used for the length tag minus the length of the trace block tag minus the length of the trace block length tag
+            int paddingLength = expectedLength - computeLengthBytes(expectedLength) - 2;
+            writeLength(raf, paddingLength);
+            byte[] bytes = new byte[paddingLength];
+            raf.write(bytes);
+        }
+        raf.write(TRSTag.TRACE_BLOCK.getValue());
+        raf.write(TRSTag.TRACE_BLOCK.getLength());
+    }
+
     /**
      * Writes the provided TRS metadata to the stream.
      *
@@ -30,7 +91,7 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData)
             fos.getChannel().position(0);
         }
         for (TRSTag tag : TRSTag.values()) {
-            if (tag.equals(TRSTag.TRACE_BLOCK)) continue;                     //TRACE BLOCK should be the last write
+            if (tag.equals(TRSTag.TRACE_BLOCK) || tag.equals(TRSTag.PADDING)) continue; //PADDING and TRACE BLOCK should be the last writes
             if (!tag.isRequired() && metaData.hasDefaultValue(tag)) continue; //ignore if default and not required
             fos.write(tag.getValue());
             if (tag.getType() == String.class) {
@@ -61,16 +122,47 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData)
                 throw new TRSFormatException(String.format(UNSUPPORTED_TAG_TYPE, tag.getName(), tag.getType()));
             }
         }
+        // Grow the metadata up to 1M, creating an empty buffer in the trace set
+        // This allows us to grow the header without having to rewrite the whole file
+        if (fos.getChannel().position() < DEFAULT_METADATA_SIZE) {
+            fos.write(TRSTag.PADDING.getValue());
+            int expectedLength = (int) (DEFAULT_METADATA_SIZE - fos.getChannel().position());
+            // The length of the padding will be the maximum size minus the current position minus the number of bytes used for the length tag minus the length of the trace block tag minus the length of the trace block length tag
+            int paddingLength = expectedLength - computeLengthBytes(expectedLength) - 2;
+            writeLength(fos, paddingLength);
+            byte[] bytes = new byte[paddingLength];
+            fos.write(bytes);
+        }
         fos.write(TRSTag.TRACE_BLOCK.getValue());
         fos.write(TRSTag.TRACE_BLOCK.getLength());
     }
 
+    private static int computeLengthBytes(int length) {
+        int lengthBytes = 0;
+        if (length > 0x7F) {
+            int lenlen = 1 + (int) (Math.log(length) / Math.log(256));
+            lengthBytes++;
+            for (int i = 0; i < lenlen; i++) {
+                lengthBytes++;
+            }
+        } else {
+            lengthBytes++;
+        }
+        return lengthBytes;
+    }
+
     private static void writeInt(FileOutputStream fos, int value, int length) throws IOException {
         for (int i = 0; i < length; i++) {
             fos.write((byte) (value >> (i * 8)));
         }
     }
 
+    private static void writeInt(RandomAccessFile raf, int value, int length) throws IOException {
+        for (int i = 0; i < length; i++) {
+            raf.write((byte) (value >> (i * 8)));
+        }
+    }
+
     private static void writeLength(FileOutputStream fos, long length) throws IOException {
         if (length > 0x7F) {
             int lenlen = 1 + (int) (Math.log(length) / Math.log(256));
@@ -83,6 +175,18 @@ private static void writeLength(FileOutputStream fos, long length) throws IOExce
         }
     }
 
+    private static void writeLength(RandomAccessFile raf, long length) throws IOException {
+        if (length > 0x7F) {
+            int lenlen = 1 + (int) (Math.log(length) / Math.log(256));
+            raf.write((byte) (0x80 + lenlen));
+            for (int i = 0; i < lenlen; i++) {
+                raf.write((byte) (length >> (i * 8)));
+            }
+        } else {
+            raf.write((byte) length);
+        }
+    }
+
     /**
      * Reads the meta data of a TRS file. The {@code ByteBuffer} is assumed to be positioned at the start of the file; A
      * {@code TRSFormatException} will probably be thrown otherwise, since it cannot be parsed.
@@ -124,7 +228,7 @@ public static String readName(LittleEndianInputStream dis) throws IOException {
 
     private static void readAndStoreData(ByteBuffer buffer, byte tag, int length, TRSMetaData trsMD)
             throws TRSFormatException {
-        boolean hasValidLength = (0 <= length & length <= 0xffff);
+        boolean hasValidLength = (0 <= length & length <= 0xffffff);
         TRSTag trsTag;
         try {
             trsTag = TRSTag.fromValue(tag);

src/main/java/com/riscure/trs/TraceSet.java

@@ -1,13 +1,19 @@
 package com.riscure.trs;
 
+import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap;
+import com.riscure.trs.parameter.traceset.TraceSetParameterMap;
+
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.nio.file.Path;
 import java.util.List;
 
-import static com.riscure.trs.enums.TRSTag.TRS_VERSION;
+import static com.riscure.trs.enums.TRSTag.*;
 
 public abstract class TraceSet implements AutoCloseable {
     protected static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed.";
+    // We want to pre-allocate 256k for the header, so we can grow it if needed without re-writing the whole file
+    public static final long DEFAULT_METADATA_SIZE = 256_000L;
 
     //Shared variables
     private final Path path;
@@ -135,7 +141,36 @@ public static TraceSet create(String file) throws IOException {
      * @throws IOException if the file creation failed
      */
     public static TraceSet create(String file, TRSMetaData metaData) throws IOException {
-        metaData.put(TRS_VERSION, 2, false);
+        metaData.put(TRS_VERSION, 3, false);
         return new WritableTraceSet(file, metaData);
     }
+
+    /**
+     * Overwrite the metadata associated with this trace set
+     * If this traceset is in read mode, this is only possible under certain conditions:
+     * 1) The opened trace set is a V3 set
+     * 2) There is empty remaining space (i.e. padding) in the pre-allocated metadata
+     *
+     * TODO: We should probably limit the changes to specific tags. e.g. the number of traces should not be modified,
+     * TODO: but the TSPM is fine. The definition map may be updated, but the size must remain the same
+     */
+    public static void updateParameterMaps(String file, TraceSetParameterMap tspm, TraceParameterDefinitionMap tpdm) throws IOException, TRSFormatException {
+        TRSMetaData metaData;
+        try (TraceSet ts = open(file)) {
+            metaData = ts.getMetaData();
+        }
+
+        if (metaData.getInt(TRS_VERSION) < 3) throw new IOException(String.format("This trace set is version %d. Only version 3 and upwards support updating metadata.", metaData.getInt(TRS_VERSION)));
+        // TODO check this
+        //if (metaDataSize > DEFAULT_METADATA_SIZE) throw new IOException("The meta data has already grown beyond the padding size. This trace set does not support updating the meta data.");
+        if (metaData.getTraceParameterDefinitions().totalSize() != tpdm.totalSize()) throw new IOException("The provided parameter definitions are of a different size than the current ones. While it's possible to change the definitions, the size must match.");
+
+        metaData.put(TRACE_SET_PARAMETERS, tspm);
+        metaData.put(TRACE_PARAMETER_DEFINITIONS, tpdm);
+
+        // Open the file in append mode so we can overwrite the header only
+        try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
+            TRSMetaDataUtils.writeTRSMetaData(raf, metaData);
+        }
+    }
 }

src/main/java/com/riscure/trs/WritableTraceSet.java

@@ -35,7 +35,7 @@ public class WritableTraceSet extends TraceSet {
 
     WritableTraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException {
         super(Paths.get(outputFileName));
-        this.metaData = metaData;
+        this.metaData = metaData.modifiable();
         this.writeStream = new FileOutputStream(outputFileName);
     }
 
@@ -55,6 +55,7 @@ public void add(Trace trace) throws IOException, TRSFormatException {
             metaData.put(TITLE_SPACE, titleLength, false);
             metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false);
             metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters()));
+            metaData.put(NUMBER_OF_TRACES, 0);
             TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData);
             firstTrace = false;
         }

src/main/java/com/riscure/trs/enums/TRSTag.java

@@ -54,7 +54,8 @@ public enum TRSTag {
     XY_SCAN_HEIGHT                   (0x74, "HE", false,    Integer.class,      4,      0,                 "Number of steps in the \"y\" direction during XY scan"),
     XY_MEASUREMENTS_PER_SPOT         (0x75, "ME", false,    Integer.class,      4,      0,                 "Number of consecutive measurements done per spot during XY scan"),
     TRACE_SET_PARAMETERS             (0x76, "GP", false,    TraceSetParameterMap.class, 0, UnmodifiableTraceSetParameterMap.of(new TraceSetParameterMap()),                 "The set of custom global trace set parameters"),
-    TRACE_PARAMETER_DEFINITIONS      (0x77, "LP", false,    TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()),                 "The set of custom local trace parameters");
+    TRACE_PARAMETER_DEFINITIONS      (0x77, "LP", false,    TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()),                 "The set of custom local trace parameters"),
+    PADDING                          (0xFF, "FF", false,    Integer.class,      0,      0,                 "Empty value to allow growing the metadata");
 
     private static final String UNKNOWN_TAG = "Unknown tag: 0x%X";
 

src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java

@@ -1,14 +1,19 @@
 package com.riscure.trs.parameter.trace.definition;
 
+import com.riscure.trs.TRSFormatException;
 import com.riscure.trs.TRSMetaDataUtils;
 import com.riscure.trs.io.LittleEndianInputStream;
 import com.riscure.trs.io.LittleEndianOutputStream;
 import com.riscure.trs.parameter.TraceParameter;
 import com.riscure.trs.parameter.trace.TraceParameterMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -18,6 +23,8 @@
  * This explicitly implements LinkedHashMap to ensure that the data is retrieved in the same order as it was added
  */
 public class TraceParameterDefinitionMap extends LinkedHashMap<String, TraceParameterDefinition<TraceParameter>> {
+    private static final Log LOG = LogFactory.getLog(TraceParameterDefinitionMap.class);
+    private static final String NAME_TOO_LONG = "Name of length %d exceeds maximum length of %d bytes%nName will be truncated to the maximum length%n";
 
     public TraceParameterDefinitionMap() {
         super();
@@ -43,14 +50,20 @@ public int totalSize() {
      * @return this map converted to a byte array, serialized according to the TRS V2 standard definition
      * @throws RuntimeException if the map failed to serialize correctly
      */
-    public byte[] serialize() {
+    public byte[] serialize() throws IOException, TRSFormatException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         try (LittleEndianOutputStream dos = new LittleEndianOutputStream(baos)) {
             //Write NE
             dos.writeShort(size());
             for (Map.Entry<String, TraceParameterDefinition<TraceParameter>> entry : entrySet()) {
                 byte[] nameBytes = entry.getKey().getBytes(StandardCharsets.UTF_8);
                 //Write NL
+                if (nameBytes.length > Short.MAX_VALUE) {
+                    LOG.warn(String.format(NAME_TOO_LONG, nameBytes.length, Short.MAX_VALUE));
+                    nameBytes = new byte[Short.MAX_VALUE];
+                    CharBuffer name = CharBuffer.wrap(entry.getKey());
+                    StandardCharsets.UTF_8.newEncoder().encode(name, ByteBuffer.wrap(nameBytes), true);
+                }
                 dos.writeShort(nameBytes.length);
                 //Write N
                 dos.write(nameBytes);
@@ -59,8 +72,6 @@ public byte[] serialize() {
             }
             dos.flush();
             return baos.toByteArray();
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
         }
     }
 
@@ -89,10 +100,10 @@ public static TraceParameterDefinitionMap deserialize(byte[] bytes) {
     }
 
     /**
-     * Create a set of definitions based on the parameters present in a trace.
+     * Create a map of definitions based on the parameters present in a trace.
      *
      * @param parameters the parameters of the trace
-     * @return a set of definitions based on the parameters present in a trace
+     * @return a map of definitions based on the parameters present in a trace
      */
     public static TraceParameterDefinitionMap createFrom(TraceParameterMap parameters) {
         TraceParameterDefinitionMap definitions = new TraceParameterDefinitionMap();