diff --git a/pom.xml b/pom.xml index c3112cc..c10265c 100644 --- a/pom.xml +++ b/pom.xml @@ -216,6 +216,11 @@ + + commons-logging + commons-logging + 1.3.6 + org.junit.jupiter junit-jupiter-api diff --git a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java index b22b0c0..c99716f 100644 --- a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java +++ b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java @@ -22,7 +22,7 @@ public class ReadOnlyTraceSet extends TraceSet { private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d)."; private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; // This is excessive for the header, but it's only the initial maximum - private static final long MAX_METADATA_SIZE = 100_000_000L; + private static final long INITIAL_MEMORY_SIZE = 100_000_000L; private final int metaDataSize; private final FileInputStream readStream; @@ -44,7 +44,7 @@ public class ReadOnlyTraceSet extends TraceSet { //the file might be bigger than the buffer, in which case we partially buffer it in memory this.fileSize = channel.size(); - long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE); + long initialBufferSize = Math.min(fileSize, INITIAL_MEMORY_SIZE); this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); @@ -129,6 +129,19 @@ private long calculateTraceSize() { public void close() throws IOException, TRSFormatException { super.close(); closeReader(); + awaitFileUnmapping(); + } + + private static void awaitFileUnmapping() throws IOException { + // Unfortunately, the current solution requires a garbage collect to have been performed before the issue is resolved. + // Other fixes required either a Java 8 Cleaner.clean() call not accessible from Java 21, or a Java 20 Arena.close(), + // which is not been finalized in Java 21. + System.gc(); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new IOException(e); + } } @Override diff --git a/src/main/java/com/riscure/trs/TRSMetaData.java b/src/main/java/com/riscure/trs/TRSMetaData.java index e2fb127..fa9829b 100644 --- a/src/main/java/com/riscure/trs/TRSMetaData.java +++ b/src/main/java/com/riscure/trs/TRSMetaData.java @@ -29,6 +29,19 @@ private void init() { } } + /** + * @return a modifiable copy of this metadata object + */ + public TRSMetaData modifiable() { + TRSMetaData copy = new TRSMetaData(); + for (TRSTag tag : TRSTag.values()) { + copy.put(tag, get(tag)); + } + copy.put(TRSTag.TRACE_SET_PARAMETERS, getTraceSetParameters().copy()); + copy.put(TRSTag.TRACE_PARAMETER_DEFINITIONS, getTraceParameterDefinitions().copy()); + return copy; + } + /** * Add the data associated with the supplied tag to this metadata. * This will overwrite any existing value diff --git a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java index dcc0ebf..7dd2565 100644 --- a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java +++ b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java @@ -7,15 +7,76 @@ import java.io.FileOutputStream; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import static com.riscure.trs.TraceSet.DEFAULT_METADATA_SIZE; + public class TRSMetaDataUtils { private static final String IGNORED_UNKNOWN_TAG = "ignored unknown metadata tag '%02X' while reading a TRS file\n"; private static final String TAG_LENGTH_INVALID = "The length field following tag '%s' has value '%X', which is not between 0 and 0xffff"; private static final String UNSUPPORTED_TAG_TYPE = "Unsupported tag type for tag '%s': %s"; private static final String REWINDING_STREAM = "The output stream is not at the start of the file. Rewinding stream."; + /** + * Writes the provided TRS metadata to the stream. + * + * @param raf the file output opened in random access mode + * @param metaData the metadata to write + * @throws IOException if any write error occurs + * @throws TRSFormatException if the metadata contains unsupported tags + */ + public static void writeTRSMetaData(RandomAccessFile raf, TRSMetaData metaData) throws IOException, TRSFormatException { + // We always write at the start of the file + raf.seek(0); + for (TRSTag tag : TRSTag.values()) { + if (tag.equals(TRSTag.TRACE_BLOCK) || tag.equals(TRSTag.PADDING)) continue; //PADDING and TRACE BLOCK should be the last writes + if (!tag.isRequired() && metaData.hasDefaultValue(tag)) continue; //ignore if default and not required + raf.write(tag.getValue()); + if (tag.getType() == String.class) { + String s = metaData.getString(tag); + byte[] stringBytes = s.getBytes(StandardCharsets.UTF_8); + writeLength(raf, stringBytes.length); + raf.write(stringBytes); + } else if (tag.getType() == Float.class) { + float f = metaData.getFloat(tag); + writeLength(raf, tag.getLength()); + writeInt(raf, Float.floatToIntBits(f), tag.getLength()); + } else if (tag.getType() == Boolean.class) { + int value = metaData.getBoolean(tag) ? 1 : 0; + writeLength(raf, tag.getLength()); + writeInt(raf, value, tag.getLength()); + } else if (tag.getType() == Integer.class) { + writeLength(raf, tag.getLength()); + writeInt(raf, metaData.getInt(tag), tag.getLength()); + } else if (tag.getType() == TraceSetParameterMap.class) { + byte[] serialized = metaData.getTraceSetParameters().serialize(); + writeLength(raf, serialized.length); + raf.write(serialized); + } else if (tag.getType() == TraceParameterDefinitionMap.class) { + byte[] serialized = metaData.getTraceParameterDefinitions().serialize(); + writeLength(raf, serialized.length); + raf.write(serialized); + } else { + throw new TRSFormatException(String.format(UNSUPPORTED_TAG_TYPE, tag.getName(), tag.getType())); + } + } + // Grow the metadata up to 1M, creating an empty buffer in the trace set + // This allows us to grow the header without having to rewrite the whole file + if (raf.getChannel().position() < DEFAULT_METADATA_SIZE) { + raf.write(TRSTag.PADDING.getValue()); + int expectedLength = (int) (DEFAULT_METADATA_SIZE - raf.getChannel().position()); + // The length of the padding will be the maximum size minus the current position minus the number of bytes used for the length tag minus the length of the trace block tag minus the length of the trace block length tag + int paddingLength = expectedLength - computeLengthBytes(expectedLength) - 2; + writeLength(raf, paddingLength); + byte[] bytes = new byte[paddingLength]; + raf.write(bytes); + } + raf.write(TRSTag.TRACE_BLOCK.getValue()); + raf.write(TRSTag.TRACE_BLOCK.getLength()); + } + /** * Writes the provided TRS metadata to the stream. * @@ -30,7 +91,7 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData) fos.getChannel().position(0); } for (TRSTag tag : TRSTag.values()) { - if (tag.equals(TRSTag.TRACE_BLOCK)) continue; //TRACE BLOCK should be the last write + if (tag.equals(TRSTag.TRACE_BLOCK) || tag.equals(TRSTag.PADDING)) continue; //PADDING and TRACE BLOCK should be the last writes if (!tag.isRequired() && metaData.hasDefaultValue(tag)) continue; //ignore if default and not required fos.write(tag.getValue()); if (tag.getType() == String.class) { @@ -61,16 +122,47 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData) throw new TRSFormatException(String.format(UNSUPPORTED_TAG_TYPE, tag.getName(), tag.getType())); } } + // Grow the metadata up to 1M, creating an empty buffer in the trace set + // This allows us to grow the header without having to rewrite the whole file + if (fos.getChannel().position() < DEFAULT_METADATA_SIZE) { + fos.write(TRSTag.PADDING.getValue()); + int expectedLength = (int) (DEFAULT_METADATA_SIZE - fos.getChannel().position()); + // The length of the padding will be the maximum size minus the current position minus the number of bytes used for the length tag minus the length of the trace block tag minus the length of the trace block length tag + int paddingLength = expectedLength - computeLengthBytes(expectedLength) - 2; + writeLength(fos, paddingLength); + byte[] bytes = new byte[paddingLength]; + fos.write(bytes); + } fos.write(TRSTag.TRACE_BLOCK.getValue()); fos.write(TRSTag.TRACE_BLOCK.getLength()); } + private static int computeLengthBytes(int length) { + int lengthBytes = 0; + if (length > 0x7F) { + int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); + lengthBytes++; + for (int i = 0; i < lenlen; i++) { + lengthBytes++; + } + } else { + lengthBytes++; + } + return lengthBytes; + } + private static void writeInt(FileOutputStream fos, int value, int length) throws IOException { for (int i = 0; i < length; i++) { fos.write((byte) (value >> (i * 8))); } } + private static void writeInt(RandomAccessFile raf, int value, int length) throws IOException { + for (int i = 0; i < length; i++) { + raf.write((byte) (value >> (i * 8))); + } + } + private static void writeLength(FileOutputStream fos, long length) throws IOException { if (length > 0x7F) { int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); @@ -83,6 +175,18 @@ private static void writeLength(FileOutputStream fos, long length) throws IOExce } } + private static void writeLength(RandomAccessFile raf, long length) throws IOException { + if (length > 0x7F) { + int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); + raf.write((byte) (0x80 + lenlen)); + for (int i = 0; i < lenlen; i++) { + raf.write((byte) (length >> (i * 8))); + } + } else { + raf.write((byte) length); + } + } + /** * Reads the meta data of a TRS file. The {@code ByteBuffer} is assumed to be positioned at the start of the file; A * {@code TRSFormatException} will probably be thrown otherwise, since it cannot be parsed. @@ -124,7 +228,7 @@ public static String readName(LittleEndianInputStream dis) throws IOException { private static void readAndStoreData(ByteBuffer buffer, byte tag, int length, TRSMetaData trsMD) throws TRSFormatException { - boolean hasValidLength = (0 <= length & length <= 0xffff); + boolean hasValidLength = (0 <= length & length <= 0xffffff); TRSTag trsTag; try { trsTag = TRSTag.fromValue(tag); diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index 751187c..9fcc6c7 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -1,13 +1,19 @@ package com.riscure.trs; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; +import com.riscure.trs.parameter.traceset.TraceSetParameterMap; + import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.file.Path; import java.util.List; -import static com.riscure.trs.enums.TRSTag.TRS_VERSION; +import static com.riscure.trs.enums.TRSTag.*; public abstract class TraceSet implements AutoCloseable { protected static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; + // We want to pre-allocate 256k for the header, so we can grow it if needed without re-writing the whole file + public static final long DEFAULT_METADATA_SIZE = 256_000L; //Shared variables private final Path path; @@ -135,7 +141,36 @@ public static TraceSet create(String file) throws IOException { * @throws IOException if the file creation failed */ public static TraceSet create(String file, TRSMetaData metaData) throws IOException { - metaData.put(TRS_VERSION, 2, false); + metaData.put(TRS_VERSION, 3, false); return new WritableTraceSet(file, metaData); } + + /** + * Overwrite the metadata associated with this trace set + * If this traceset is in read mode, this is only possible under certain conditions: + * 1) The opened trace set is a V3 set + * 2) There is empty remaining space (i.e. padding) in the pre-allocated metadata + * + * TODO: We should probably limit the changes to specific tags. e.g. the number of traces should not be modified, + * TODO: but the TSPM is fine. The definition map may be updated, but the size must remain the same + */ + public static void updateParameterMaps(String file, TraceSetParameterMap tspm, TraceParameterDefinitionMap tpdm) throws IOException, TRSFormatException { + TRSMetaData metaData; + try (TraceSet ts = open(file)) { + metaData = ts.getMetaData(); + } + + if (metaData.getInt(TRS_VERSION) < 3) throw new IOException(String.format("This trace set is version %d. Only version 3 and upwards support updating metadata.", metaData.getInt(TRS_VERSION))); + // TODO check this + //if (metaDataSize > DEFAULT_METADATA_SIZE) throw new IOException("The meta data has already grown beyond the padding size. This trace set does not support updating the meta data."); + if (metaData.getTraceParameterDefinitions().totalSize() != tpdm.totalSize()) throw new IOException("The provided parameter definitions are of a different size than the current ones. While it's possible to change the definitions, the size must match."); + + metaData.put(TRACE_SET_PARAMETERS, tspm); + metaData.put(TRACE_PARAMETER_DEFINITIONS, tpdm); + + // Open the file in append mode so we can overwrite the header only + try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) { + TRSMetaDataUtils.writeTRSMetaData(raf, metaData); + } + } } diff --git a/src/main/java/com/riscure/trs/WritableTraceSet.java b/src/main/java/com/riscure/trs/WritableTraceSet.java index aef3306..050436c 100644 --- a/src/main/java/com/riscure/trs/WritableTraceSet.java +++ b/src/main/java/com/riscure/trs/WritableTraceSet.java @@ -35,7 +35,7 @@ public class WritableTraceSet extends TraceSet { WritableTraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { super(Paths.get(outputFileName)); - this.metaData = metaData; + this.metaData = metaData.modifiable(); this.writeStream = new FileOutputStream(outputFileName); } @@ -55,6 +55,7 @@ public void add(Trace trace) throws IOException, TRSFormatException { metaData.put(TITLE_SPACE, titleLength, false); metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false); metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters())); + metaData.put(NUMBER_OF_TRACES, 0); TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); firstTrace = false; } diff --git a/src/main/java/com/riscure/trs/enums/TRSTag.java b/src/main/java/com/riscure/trs/enums/TRSTag.java index 32038dd..e99ea33 100644 --- a/src/main/java/com/riscure/trs/enums/TRSTag.java +++ b/src/main/java/com/riscure/trs/enums/TRSTag.java @@ -54,7 +54,8 @@ public enum TRSTag { XY_SCAN_HEIGHT (0x74, "HE", false, Integer.class, 4, 0, "Number of steps in the \"y\" direction during XY scan"), XY_MEASUREMENTS_PER_SPOT (0x75, "ME", false, Integer.class, 4, 0, "Number of consecutive measurements done per spot during XY scan"), TRACE_SET_PARAMETERS (0x76, "GP", false, TraceSetParameterMap.class, 0, UnmodifiableTraceSetParameterMap.of(new TraceSetParameterMap()), "The set of custom global trace set parameters"), - TRACE_PARAMETER_DEFINITIONS (0x77, "LP", false, TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()), "The set of custom local trace parameters"); + TRACE_PARAMETER_DEFINITIONS (0x77, "LP", false, TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()), "The set of custom local trace parameters"), + PADDING (0xFF, "FF", false, Integer.class, 0, 0, "Empty value to allow growing the metadata"); private static final String UNKNOWN_TAG = "Unknown tag: 0x%X"; diff --git a/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java b/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java index dedd43d..2525df0 100644 --- a/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java +++ b/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java @@ -1,14 +1,19 @@ package com.riscure.trs.parameter.trace.definition; +import com.riscure.trs.TRSFormatException; import com.riscure.trs.TRSMetaDataUtils; import com.riscure.trs.io.LittleEndianInputStream; import com.riscure.trs.io.LittleEndianOutputStream; import com.riscure.trs.parameter.TraceParameter; import com.riscure.trs.parameter.trace.TraceParameterMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import java.util.Map; @@ -18,6 +23,8 @@ * This explicitly implements LinkedHashMap to ensure that the data is retrieved in the same order as it was added */ public class TraceParameterDefinitionMap extends LinkedHashMap> { + private static final Log LOG = LogFactory.getLog(TraceParameterDefinitionMap.class); + private static final String NAME_TOO_LONG = "Name of length %d exceeds maximum length of %d bytes%nName will be truncated to the maximum length%n"; public TraceParameterDefinitionMap() { super(); @@ -43,7 +50,7 @@ public int totalSize() { * @return this map converted to a byte array, serialized according to the TRS V2 standard definition * @throws RuntimeException if the map failed to serialize correctly */ - public byte[] serialize() { + public byte[] serialize() throws IOException, TRSFormatException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (LittleEndianOutputStream dos = new LittleEndianOutputStream(baos)) { //Write NE @@ -51,6 +58,12 @@ public byte[] serialize() { for (Map.Entry> entry : entrySet()) { byte[] nameBytes = entry.getKey().getBytes(StandardCharsets.UTF_8); //Write NL + if (nameBytes.length > Short.MAX_VALUE) { + LOG.warn(String.format(NAME_TOO_LONG, nameBytes.length, Short.MAX_VALUE)); + nameBytes = new byte[Short.MAX_VALUE]; + CharBuffer name = CharBuffer.wrap(entry.getKey()); + StandardCharsets.UTF_8.newEncoder().encode(name, ByteBuffer.wrap(nameBytes), true); + } dos.writeShort(nameBytes.length); //Write N dos.write(nameBytes); @@ -59,8 +72,6 @@ public byte[] serialize() { } dos.flush(); return baos.toByteArray(); - } catch (IOException ex) { - throw new RuntimeException(ex); } } @@ -89,10 +100,10 @@ public static TraceParameterDefinitionMap deserialize(byte[] bytes) { } /** - * Create a set of definitions based on the parameters present in a trace. + * Create a map of definitions based on the parameters present in a trace. * * @param parameters the parameters of the trace - * @return a set of definitions based on the parameters present in a trace + * @return a map of definitions based on the parameters present in a trace */ public static TraceParameterDefinitionMap createFrom(TraceParameterMap parameters) { TraceParameterDefinitionMap definitions = new TraceParameterDefinitionMap(); diff --git a/src/test/java/TestTraceSet.java b/src/test/java/TestTraceSet.java index ba6e132..781a232 100644 --- a/src/test/java/TestTraceSet.java +++ b/src/test/java/TestTraceSet.java @@ -11,7 +11,6 @@ import com.riscure.trs.parameter.trace.TraceParameterMap; import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; -import com.riscure.trs.parameter.traceset.TraceSetParameter; import com.riscure.trs.parameter.traceset.TraceSetParameterMap; import com.riscure.trs.types.*; import org.junit.jupiter.api.AfterAll; @@ -26,10 +25,11 @@ import java.nio.file.Path; import java.util.*; +import static com.riscure.trs.TraceSet.DEFAULT_METADATA_SIZE; import static org.junit.jupiter.api.Assertions.*; -public class TestTraceSet { +class TestTraceSet { private static Path tempDir; private static final String BYTES_TRS = "bytes.trs"; private static final String SHORTS_TRS = "shorts.trs"; @@ -48,25 +48,25 @@ public class TestTraceSet { public static void createTempDir() throws IOException, TRSFormatException { tempDir = Files.createTempDirectory("TestTraceSet"); - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(BYTE_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + SHORTS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + SHORTS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(SHORT_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + INTS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + INTS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(INT_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + FLOATS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + FLOATS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(FLOAT_SAMPLES)); } @@ -74,29 +74,24 @@ public static void createTempDir() throws IOException, TRSFormatException { } @AfterAll - public static void cleanup() throws InterruptedException { - //We need to allow a little time for java to release all handles - System.gc(); - Thread.sleep(100); + public static void cleanup() { for (File file : Objects.requireNonNull(tempDir.toFile().listFiles())) { try { Files.delete(file.toPath()); } catch (IOException e) { - System.err.printf("Failed to delete temporary file '%s'%n", file.toPath().toAbsolutePath().toString()); - e.printStackTrace(); + System.err.printf("Failed to delete temporary file '%s'%n", file.toPath().toAbsolutePath()); } } try { Files.delete(tempDir); } catch (IOException e) { - System.err.printf("Failed to delete temporary folder '%s'%n", tempDir.toFile().toPath().toAbsolutePath().toString()); - e.printStackTrace(); + System.err.printf("Failed to delete temporary folder '%s'%n", tempDir.toFile().toPath().toAbsolutePath()); } } @Test void testOpenBytes() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.BYTE, encoding); @@ -111,7 +106,7 @@ void testOpenBytes() throws IOException, TRSFormatException { @Test void testOpenShorts() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + SHORTS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + SHORTS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.SHORT, encoding); @@ -126,7 +121,7 @@ void testOpenShorts() throws IOException, TRSFormatException { @Test void testOpenInts() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + INTS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + INTS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.INT, encoding); @@ -141,7 +136,7 @@ void testOpenInts() throws IOException, TRSFormatException { @Test void testOpenFloats() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + FLOATS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + FLOATS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.FLOAT, encoding); @@ -157,13 +152,11 @@ void testOpenFloats() throws IOException, TRSFormatException { @Test void testUTF8Title() throws IOException, TRSFormatException { String title = "씨브 크레그스만"; - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet ts = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet ts = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name)) { ts.add(Trace.create(title, new float[0], new TraceParameterMap())); - } catch (TRSFormatException e) { - throw e; } - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertEquals(title, readable.get(0).getTitle()); } } @@ -171,9 +164,6 @@ void testUTF8Title() throws IOException, TRSFormatException { /** * This tests adding several different types of information to the trace set header. The three parameters are chosen * to match the three major cases: Strings, primitives, and arbitrary (serializable) objects. - * - * @throws IOException - * @throws TRSFormatException */ @Test void testWriteTraceSetParameters() throws IOException, TRSFormatException { @@ -198,39 +188,38 @@ void testWriteTraceSetParameters() throws IOException, TRSFormatException { //parameters.put("XYZ offset", XYZ_TEST_VALUE); metaData.put(TRSTag.TRACE_SET_PARAMETERS, parameters); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData).close(); + String name = UUID.randomUUID() + TRS; + TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData).close(); //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceSetParameterMap readTraceSetParameterMap = readable.getMetaData().getTraceSetParameters(); parameters.forEach((s, traceSetParameter) -> assertEquals(traceSetParameter, readTraceSetParameterMap.get(s))); } } /** - * This tests adding a parameter with a name of 100000 characters - * - * @throws IOException - * @throws TRSFormatException + * This tests adding a parameter with a name of 100000 characters. + * Expectation: The name will be truncated to the maximum allowed length when writing, + * when reading back and comparing with the original metadata, the values will differ */ @Test void testWriteTraceParametersInvalidName() throws IOException, TRSFormatException { TRSMetaData metaData = TRSMetaData.create(); String parameterName = String.format("%100000s", "XYZ"); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put(parameterName, 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - assertThrows(TRSFormatException.class, () -> { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { - TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); - parameterDefinitions.forEach((key, parameter) -> assertEquals(parameterName, key)); + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { + TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); + for (var def : parameterDefinitions.keySet()) { + assertNotEquals(parameterName.length(), def.length()); } - }); + } } /** @@ -238,9 +227,6 @@ void testWriteTraceParametersInvalidName() throws IOException, TRSFormatExceptio * - if no length is specified, the first string is leading * - if a string is longer than the length specified, it should be truncated * - when truncated, a string should still be valid UTF-8 (truncated at character level, not byte level) - * - * @throws IOException - * @throws TRSFormatException */ @Test void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormatException { @@ -252,8 +238,8 @@ void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormat strings.add("ab"); strings.add("abcdefgh汉字"); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { for (int k = 0; k < 25; k++) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTEARRAY", new byte[]{(byte) k, (byte) k, (byte) k}); @@ -269,17 +255,14 @@ void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormat /** * This tests whether all getters are working as expected - * - * @throws IOException - * @throws TRSFormatException */ @Test void testReadTraceParametersTyped() throws IOException, TRSFormatException { TRSMetaData metaData = TRSMetaData.create(); List testParameters = new ArrayList<>(); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { for (int k = 0; k < 25; k++) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTE", (byte) k); @@ -307,7 +290,7 @@ void testReadTraceParametersTyped() throws IOException, TRSFormatException { } private void readBackGeneric(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -322,7 +305,7 @@ private void readBackGeneric(List testParameters, String name } private void readBackTyped(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -391,7 +374,7 @@ private void readBackTyped(List testParameters, String name) } private void readBackTypedKeys(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -428,8 +411,8 @@ private void readBackTypedKeys(List testParameters, String na throw new RuntimeException("Unexpected type: " + parameter.getType()); } if (parameter.getLength() > 1 && typedKey.getCls().isArray()) { - assertArrayEquals(Arrays.asList(correctValue.getOrElseThrow(typedKey)).toArray(), - Arrays.asList(trace.getParameters().getOrElseThrow(typedKey)).toArray()); + assertArrayEquals(Collections.singletonList(correctValue.getOrElseThrow(typedKey)).toArray(), + Collections.singletonList(trace.getParameters().getOrElseThrow(typedKey)).toArray()); } else { assertEquals(correctValue.get(typedKey), trace.getParameters().get(typedKey)); } @@ -440,42 +423,37 @@ private void readBackTypedKeys(List testParameters, String na /** * This tests getting a value of the wrong type correctly throws an exception - * - * @throws IOException - * @throws TRSFormatException */ @Test void testExceptionWrongType() throws IOException, TRSFormatException { TRSMetaData metaData = TRSMetaData.create(); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTE", (byte) 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertThrows(ClassCastException.class, () -> readable.get(0).getParameters().getDouble("BYTE")); } } /** * This - * @throws IOException - * @throws TRSFormatException */ @Test void testContainsNonArray() throws IOException, TRSFormatException { ByteTypeKey byteKey = new ByteTypeKey("BYTE"); - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put(byteKey, (byte) 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertTrue(readable.get(0).getParameters().get(byteKey).isPresent()); } } @@ -503,7 +481,7 @@ void testInvalidParameterLength() { */ @Test void testModificationAfterReadback() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { assertThrows(UnsupportedOperationException.class, () -> readable.getMetaData().getTraceSetParameters().put("SHOULD_FAIL", 0)); assertThrows(UnsupportedOperationException.class, () -> readable.getMetaData().getTraceParameterDefinitions().put("SHOULD_FAIL", new TraceParameterDefinition(ParameterType.BYTE, (short)1, (short)1))); for (int k = 0; k < NUMBER_OF_TRACES; k++) { @@ -675,13 +653,50 @@ void testFileReleasing() throws IOException, TRSFormatException, InterruptedExce try (TraceSet traceSet = TraceSet.open(filePath)) { traceSet.getMetaData().getTraceSetParameters(); } - // Unfortunately, the current solution requires a garbage collect to have been performed before the issue is resolved. - // Other fixes required either a Java 8 Cleaner.clean() call not accessible from Java 21, or a Java 20 Arena.close(), - // which is not been finalized in Java 21. - System.gc(); - Thread.sleep(1000); // Assert that the opened file has been closed again, by deleting it. File file = new File(filePath); - assert(file.delete()); + assertTrue(file.delete()); + } + + /** + * This test checks whether version 3 correctly allocates 1MB of header space by default + */ + @Test + void testDefaultHeaderSize() throws IOException, TRSFormatException { + Path filePath = tempDir.resolve("large_header.trs"); + TRSMetaData metaData = new TRSMetaData(); + metaData.put(TRSTag.TRS_VERSION, 3); + try (TraceSet ts = TraceSet.create(filePath.toString(), metaData)) { + ts.add(new Trace(new float[]{0})); + } + assertTrue(filePath.toFile().length() > DEFAULT_METADATA_SIZE); + } + + /** + * This test checks whether we can successfully add information to the header of a traceset file without + * increasing its size + */ + @Test + void testOverwritingMetadata() throws IOException, TRSFormatException { + String filename = tempDir.toAbsolutePath() + File.separator + BYTES_TRS; + long originalFileSize = new File(filename).length(); + + TraceSetParameterMap tspm; + TraceParameterDefinitionMap tpdm; + try (TraceSet readable = TraceSet.open(filename)) { + assertFalse(readable.getMetaData().getTraceSetParameters().containsKey("test")); + + tspm = readable.getMetaData().getTraceSetParameters().copy(); + tpdm = readable.getMetaData().getTraceParameterDefinitions().copy(); + + tspm.put("test", "This value should exist afterwards"); + } + + TraceSet.updateParameterMaps(filename, tspm, tpdm); + + try (TraceSet readable = TraceSet.open(filename)) { + assertTrue(readable.getMetaData().getTraceSetParameters().containsKey("test")); + } + assertEquals(originalFileSize, new File(filename).length()); } }