From 65ed41617a70262fde34062f1a413abb8cccaa29 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 19 Sep 2025 10:35:44 +0200 Subject: [PATCH] #79: Split off Read and Write mode from TraceSet --- .../com/riscure/trs/ReadOnlyTraceSet.java | 199 +++++++++ src/main/java/com/riscure/trs/TraceSet.java | 378 +----------------- .../com/riscure/trs/WritableTraceSet.java | 219 ++++++++++ 3 files changed, 433 insertions(+), 363 deletions(-) create mode 100644 src/main/java/com/riscure/trs/ReadOnlyTraceSet.java create mode 100644 src/main/java/com/riscure/trs/WritableTraceSet.java diff --git a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java new file mode 100644 index 0000000..b22b0c0 --- /dev/null +++ b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java @@ -0,0 +1,199 @@ +package com.riscure.trs; + +import com.riscure.trs.enums.Encoding; +import com.riscure.trs.parameter.trace.TraceParameterMap; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; + +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Paths; + +import static com.riscure.trs.enums.TRSTag.*; +import static com.riscure.trs.enums.TRSTag.TRS_VERSION; + +public class ReadOnlyTraceSet extends TraceSet { + private static final String TRACE_SET_IN_READ_MODE = "TraceSet is in read mode. Please open the TraceSet in write mode."; + private static final String ERROR_READING_FILE = "Error reading TRS file: file size (%d) != meta data (%d) + trace size (%d) * nr of traces (%d)"; + private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d)."; + private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; + // This is excessive for the header, but it's only the initial maximum + private static final long MAX_METADATA_SIZE = 100_000_000L; + + private final int metaDataSize; + private final FileInputStream readStream; + private final LargePreMappedFile mappedFile; + private final float[] preallocatedSampleArray; + private final TRSMetaData metaData; + private final long fileSize; //the total number of bytes in the underlying file + + private ByteBuffer metaDataBuffer; + private byte[] preallocatedByteArray; + private short[] preallocatedShortArray; + private int[] preallocatedIntArray; + + ReadOnlyTraceSet(String inputFileName) throws IOException, TRSFormatException { + super(Paths.get(inputFileName)); + + this.readStream = new FileInputStream(inputFileName); + FileChannel channel = readStream.getChannel(); + + //the file might be bigger than the buffer, in which case we partially buffer it in memory + this.fileSize = channel.size(); + long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE); + + this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); + this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); + this.metaDataSize = metaDataBuffer.position(); + this.metaDataBuffer.limit(metaDataSize); + + long traceSize = calculateTraceSize(); + this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); + + int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); + this.preallocatedSampleArray = new float[numberOfSamples]; + } + + /** + * Get a trace from the set at the specified index + * @param index the index of the Trace to read from the file + * @return the Trace at the requested trace index + * @throws IOException if a read error occurs + * @throws IllegalArgumentException if this TraceSet is not ready be read from + */ + @Override + public Trace get(int index) throws IOException { + if (!isOpen()) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); + + long traceSize = calculateTraceSize(); + long nrOfTraces = this.metaData.getInt(NUMBER_OF_TRACES); + if (index >= nrOfTraces) { + String msg = String.format(TRACE_INDEX_OUT_OF_BOUNDS, index, nrOfTraces); + throw new IllegalArgumentException(msg); + } + + long calculatedFileSize = metaDataSize + traceSize * nrOfTraces; + if (fileSize != calculatedFileSize) { + String msg = String.format(ERROR_READING_FILE, fileSize, metaDataSize, traceSize, nrOfTraces); + throw new IllegalStateException(msg); + } + + ByteBuffer buffer = mappedFile.getBuffer(index); + + String traceTitle = this.readTraceTitle(buffer); + if (traceTitle.trim().isEmpty()) { + traceTitle = String.format("%s %d", metaData.getString(GLOBAL_TITLE), index); + } + + try { + TraceParameterMap traceParameterMap; + if (metaData.getInt(TRS_VERSION) > 1) { + TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); + int size = traceParameterDefinitionMap.totalSize(); + byte[] data = new byte[size]; + buffer.get(data); + traceParameterMap = TraceParameterMap.deserialize(data, traceParameterDefinitionMap); + } else { + //legacy mode + byte[] data = readData(buffer); + traceParameterMap = new TraceParameterMap(); + if (data.length > 0) { + traceParameterMap.put("LEGACY_DATA", data); + } + } + + float[] samples = readSamples(buffer); + // Since we are using an internal sample array in this class, Trace.create() should duplicate it internally + return Trace.create(traceTitle, samples, traceParameterMap); + } catch (TRSFormatException ex) { + throw new IOException(ex); + } + } + + @Override + public void add(Trace trace) throws IOException, TRSFormatException { + throw new IllegalArgumentException(TRACE_SET_IN_READ_MODE); + } + + private long calculateTraceSize() { + int sampleSize = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)).getSize(); + long sampleSpace = metaData.getInt(NUMBER_OF_SAMPLES) * (long) sampleSize; + return sampleSpace + metaData.getInt(DATA_LENGTH) + metaData.getInt(TITLE_SPACE); + } + + @Override + public void close() throws IOException, TRSFormatException { + super.close(); + closeReader(); + } + + @Override + public TRSMetaData getMetaData() { + return metaData; + } + + private void closeReader() throws IOException { + metaDataBuffer = null; + mappedFile.close(); + readStream.close(); + } + + protected String readTraceTitle(ByteBuffer buffer) { + byte[] titleArray = new byte[metaData.getInt(TITLE_SPACE)]; + buffer.get(titleArray); + return new String(titleArray); + } + + protected byte[] readData(ByteBuffer buffer) { + int inputSize = metaData.getInt(DATA_LENGTH); + byte[] comDataArray = new byte[inputSize]; + buffer.get(comDataArray); + return comDataArray; + } + + /* + * We can reuse the buffers when not dealing with float samples. They are instantiated once just in time if needed. + */ + protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { + switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { + case BYTE: + this.preallocatedByteArray = this.preallocatedByteArray == null ? new byte[preallocatedSampleArray.length] : this.preallocatedByteArray; + buffer.get(preallocatedByteArray); + // Manual copy of byte[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedByteArray[k]; + } + break; + case SHORT: + this.preallocatedShortArray = this.preallocatedShortArray == null ? new short[preallocatedSampleArray.length] : this.preallocatedShortArray; + ShortBuffer shortView = buffer.asShortBuffer(); + shortView.get(preallocatedShortArray); + // Manual copy of short[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedShortArray[k]; + } + break; + case FLOAT: + FloatBuffer floatView = buffer.asFloatBuffer(); + floatView.get(preallocatedSampleArray); + break; + case INT: + this.preallocatedIntArray = this.preallocatedIntArray == null ? new int[preallocatedSampleArray.length] : this.preallocatedIntArray; + IntBuffer intView = buffer.asIntBuffer(); + intView.get(preallocatedIntArray); + // Manual copy of int[] into float[] + for (int k = 0; k < preallocatedIntArray.length; k++) { + preallocatedSampleArray[k] = (float) preallocatedIntArray[k]; + } + break; + default: + throw new TRSFormatException(String.format(UNKNOWN_SAMPLE_CODING, metaData.getInt(SAMPLE_CODING))); + } + + return preallocatedSampleArray; + } +} diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index 2de90ef..751187c 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -1,98 +1,21 @@ package com.riscure.trs; -import com.riscure.trs.enums.Encoding; -import com.riscure.trs.enums.ParameterType; -import com.riscure.trs.parameter.TraceParameter; -import com.riscure.trs.parameter.primitive.StringParameter; -import com.riscure.trs.parameter.trace.TraceParameterMap; -import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; -import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; - -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.nio.*; -import java.nio.channels.FileChannel; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; import java.util.List; -import java.util.Map; - -import static com.riscure.trs.enums.TRSTag.*; - -public class TraceSet implements AutoCloseable { - private static final String ERROR_READING_FILE = "Error reading TRS file: file size (%d) != meta data (%d) + trace size (%d) * nr of traces (%d)"; - private static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; - private static final String TRACE_SET_IN_WRITE_MODE = "TraceSet is in write mode. Please open the TraceSet in read mode."; - private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d)."; - private static final String TRACE_SET_IN_READ_MODE = "TraceSet is in read mode. Please open the TraceSet in write mode."; - private static final String TRACE_LENGTH_DIFFERS = "All traces in a set need to be the same length, but current trace length (%d) differs from the previous trace(s) (%d)"; - private static final String TRACE_DATA_LENGTH_DIFFERS = "All traces in a set need to have the same data length, but current trace data length (%d) differs from the previous trace(s) (%d)"; - private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; - private static final String PARAMETER_NOT_DEFINED = "Parameter %s is saved in the trace, but was not found in the header definition"; - // This is excessive for the header, but it's only the initial maximum - private static final long MAX_METADATA_SIZE = 100_000_000L; - - //Reading variables - private int metaDataSize; - private FileInputStream readStream; - - private ByteBuffer metaDataBuffer; - private LargePreMappedFile mappedFile; - private float[] preallocatedSampleArray; - private byte[] preallocatedByteArray; - private short[] preallocatedShortArray; - private int[] preallocatedIntArray; - private long fileSize; //the total number of bytes in the underlying file +import static com.riscure.trs.enums.TRSTag.TRS_VERSION; - //Writing variables - private FileOutputStream writeStream; - - private boolean firstTrace = true; +public abstract class TraceSet implements AutoCloseable { + protected static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; //Shared variables - private final TRSMetaData metaData; - private final boolean writing; //whether the trace is opened in write mode private final Path path; - private final CharsetDecoder utf8Decoder = StandardCharsets.UTF_8.newDecoder(); - private boolean open; - private TraceSet(String inputFileName) throws IOException, TRSFormatException { - this.writing = false; - this.open = true; - this.path = Paths.get(inputFileName); - this.readStream = new FileInputStream(inputFileName); - FileChannel channel = readStream.getChannel(); - - //the file might be bigger than the buffer, in which case we partially buffer it in memory - this.fileSize = channel.size(); - long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE); - - this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); - this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); - this.metaDataSize = metaDataBuffer.position(); - this.metaDataBuffer.limit(metaDataSize); - - long traceSize = calculateTraceSize(); - this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); - - int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); - this.preallocatedSampleArray = new float[numberOfSamples]; - } - - private TraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { + protected TraceSet(Path path) { + this.path = path; this.open = true; - this.writing = true; - this.metaData = metaData; - this.path = Paths.get(outputFileName); - this.writeStream = new FileOutputStream(outputFileName); } /** @@ -102,10 +25,11 @@ public Path getPath() { return path; } - private long calculateTraceSize() { - int sampleSize = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)).getSize(); - long sampleSpace = metaData.getInt(NUMBER_OF_SAMPLES) * (long) sampleSize; - return sampleSpace + metaData.getInt(DATA_LENGTH) + metaData.getInt(TITLE_SPACE); + /** + * @return whether this trace set is currently open + */ + public boolean isOpen() { + return open; } /** @@ -115,54 +39,7 @@ private long calculateTraceSize() { * @throws IOException if a read error occurs * @throws IllegalArgumentException if this TraceSet is not ready be read from */ - public Trace get(int index) throws IOException { - if (!open) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); - if (writing) throw new IllegalArgumentException(TRACE_SET_IN_WRITE_MODE); - - long traceSize = calculateTraceSize(); - long nrOfTraces = this.metaData.getInt(NUMBER_OF_TRACES); - if (index >= nrOfTraces) { - String msg = String.format(TRACE_INDEX_OUT_OF_BOUNDS, index, nrOfTraces); - throw new IllegalArgumentException(msg); - } - - long calculatedFileSize = metaDataSize + traceSize * nrOfTraces; - if (fileSize != calculatedFileSize) { - String msg = String.format(ERROR_READING_FILE, fileSize, metaDataSize, traceSize, nrOfTraces); - throw new IllegalStateException(msg); - } - - ByteBuffer buffer = mappedFile.getBuffer(index); - - String traceTitle = this.readTraceTitle(buffer); - if (traceTitle.trim().isEmpty()) { - traceTitle = String.format("%s %d", metaData.getString(GLOBAL_TITLE), index); - } - - try { - TraceParameterMap traceParameterMap; - if (metaData.getInt(TRS_VERSION) > 1) { - TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); - int size = traceParameterDefinitionMap.totalSize(); - byte[] data = new byte[size]; - buffer.get(data); - traceParameterMap = TraceParameterMap.deserialize(data, traceParameterDefinitionMap); - } else { - //legacy mode - byte[] data = readData(buffer); - traceParameterMap = new TraceParameterMap(); - if (data.length > 0) { - traceParameterMap.put("LEGACY_DATA", data); - } - } - - float[] samples = readSamples(buffer); - // Since we are using an internal sample array in this class, Trace.create() should duplicate it internally - return Trace.create(traceTitle, samples, traceParameterMap); - } catch (TRSFormatException ex) { - throw new IOException(ex); - } - } + public abstract Trace get(int index) throws IOException; /** * Add a trace to a writable TraceSet @@ -170,243 +47,18 @@ public Trace get(int index) throws IOException { * @throws IOException if any write error occurs * @throws TRSFormatException if the formatting of the trace is invalid */ - public void add(Trace trace) throws IOException, TRSFormatException { - if (!open) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); - if (!writing) throw new IllegalArgumentException(TRACE_SET_IN_READ_MODE); - if (firstTrace) { - int dataLength = trace.getData() == null ? 0 : trace.getData().length; - int titleLength = trace.getTitle() == null ? 0 : trace.getTitle().getBytes(StandardCharsets.UTF_8).length; - metaData.put(NUMBER_OF_SAMPLES, trace.getNumberOfSamples(), false); - metaData.put(DATA_LENGTH, dataLength, false); - metaData.put(TITLE_SPACE, titleLength, false); - metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false); - metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters())); - TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); - firstTrace = false; - } - truncateStrings(trace, metaData); - checkValid(trace); - - trace.setTraceSet(this); - writeTrace(trace); - - int numberOfTraces = metaData.getInt(NUMBER_OF_TRACES); - metaData.put(NUMBER_OF_TRACES, numberOfTraces + 1); - } - - /** - * This method makes sure that the trace title and any added string parameters adhere to the preset maximum length - * @param trace the trace to update - * @param metaData the metadata specifying the maximum string lengths - */ - private void truncateStrings(Trace trace, TRSMetaData metaData) { - int titleSpace = metaData.getInt(TITLE_SPACE); - trace.setTitle(fitUtf8StringToByteLength(trace.getTitle(), titleSpace)); - TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); - for (Map.Entry> definition : traceParameterDefinitionMap.entrySet()) { - TraceParameterDefinition value = definition.getValue(); - String key = definition.getKey(); - if (value.getType() == ParameterType.STRING) { - short stringLength = value.getLength(); - String stringValue = ((StringParameter) trace.getParameters().get(key)).getValue(); - if (stringLength != stringValue.getBytes(StandardCharsets.UTF_8).length) { - trace.getParameters().put(key, fitUtf8StringToByteLength(stringValue, stringLength)); - } - } - } - } - - /** - * Fits a string to the number of characters that fit in X bytes avoiding multi byte characters being cut in - * half at the cut off point. Also handles surrogate pairs where 2 characters in the string is actually one literal - * character. If the string is too long, it is truncated. If it's too short, it's padded with NUL characters. - * @param s the string to fit - * @param maxBytes the number of bytes required - */ - private String fitUtf8StringToByteLength(String s, int maxBytes) { - if (s == null) { - return null; - } - byte[] sba = s.getBytes(StandardCharsets.UTF_8); - if (sba.length <= maxBytes) { - return new String(Arrays.copyOf(sba, maxBytes)); - } - // Ensure truncation by having byte buffer = maxBytes - ByteBuffer bb = ByteBuffer.wrap(sba, 0, maxBytes); - CharBuffer cb = CharBuffer.allocate(maxBytes); - // Ignore an incomplete character - utf8Decoder.reset(); - utf8Decoder.onMalformedInput(CodingErrorAction.IGNORE); - utf8Decoder.decode(bb, cb, true); - utf8Decoder.flush(cb); - return new String(cb.array(), 0, cb.position()); - } - - private void writeTrace(Trace trace) throws TRSFormatException, IOException { - String title = trace.getTitle() == null ? "" : trace.getTitle(); - writeStream.write(title.getBytes(StandardCharsets.UTF_8)); - byte[] data = trace.getData() == null ? new byte[0] : trace.getData(); - writeStream.write(data); - Encoding encoding = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)); - writeStream.write(toByteArray(trace.getSample(), encoding)); - } - - private byte[] toByteArray(float[] samples, Encoding encoding) throws TRSFormatException { - byte[] result; - switch (encoding) { - case ILLEGAL: - throw new TRSFormatException("Illegal sample encoding"); - case BYTE: - result = new byte[samples.length]; - for (int k = 0; k < samples.length; k++) { - if (samples[k] != (byte)samples[k]) throw new IllegalArgumentException("Byte sample encoding too small"); - result[k] = (byte) samples[k]; - } - break; - case SHORT: - result = new byte[samples.length * 2]; - for (int k = 0; k < samples.length; k++) { - if (samples[k] != (short)samples[k]) throw new IllegalArgumentException("Short sample encoding too small"); - short value = (short) samples[k]; - result[2*k] = (byte) value; - result[2*k + 1] = (byte) (value >> 8); - } - break; - case INT: - result = new byte[samples.length * 4]; - for (int k = 0; k < samples.length; k++) { - int value = (int) samples[k]; - result[4*k] = (byte) value; - result[4*k + 1] = (byte) (value >> 8); - result[4*k + 2] = (byte) (value >> 16); - result[4*k + 3] = (byte) (value >> 24); - } - break; - case FLOAT: - result = new byte[samples.length * 4]; - for (int k = 0; k < samples.length; k++) { - int value = Float.floatToIntBits(samples[k]); - result[4*k] = (byte) value; - result[4*k + 1] = (byte) (value >> 8); - result[4*k + 2] = (byte) (value >> 16); - result[4*k + 3] = (byte) (value >> 24); - } - break; - default: - throw new TRSFormatException(String.format("Sample encoding not supported: %s", encoding.name())); - } - return result; - } + public abstract void add(Trace trace) throws IOException, TRSFormatException; @Override public void close() throws IOException, TRSFormatException { open = false; - if (writing) closeWriter(); - else closeReader(); - } - - private void checkValid(Trace trace) { - int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); - if (metaData.getInt(NUMBER_OF_SAMPLES) != trace.getNumberOfSamples()) { - throw new IllegalArgumentException(String.format(TRACE_LENGTH_DIFFERS, - trace.getNumberOfSamples(), - numberOfSamples)); - } - - int dataLength = metaData.getInt(DATA_LENGTH); - int traceDataLength = trace.getData() == null ? 0 : trace.getData().length; - if (metaData.getInt(DATA_LENGTH) != traceDataLength) { - throw new IllegalArgumentException(String.format(TRACE_DATA_LENGTH_DIFFERS, - traceDataLength, - dataLength)); - } - - for (Map.Entry entry : trace.getParameters().entrySet()) { - if (!metaData.getTraceParameterDefinitions().containsKey(entry.getKey())) { - throw new IllegalArgumentException(String.format(PARAMETER_NOT_DEFINED, entry.getKey())); - } - } - } - - private void closeReader() throws IOException { - metaDataBuffer = null; - mappedFile.close(); - readStream.close(); - } - - private void closeWriter() throws IOException, TRSFormatException { - try { - //reset writer to start of file and overwrite header - writeStream.getChannel().position(0); - TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); - writeStream.flush(); - } finally { - writeStream.close(); - } } /** * Get the metadata associated with this trace set * @return the metadata associated with this trace set */ - public TRSMetaData getMetaData() { - return metaData; - } - - protected String readTraceTitle(ByteBuffer buffer) { - byte[] titleArray = new byte[metaData.getInt(TITLE_SPACE)]; - buffer.get(titleArray); - return new String(titleArray); - } - - protected byte[] readData(ByteBuffer buffer) { - int inputSize = metaData.getInt(DATA_LENGTH); - byte[] comDataArray = new byte[inputSize]; - buffer.get(comDataArray); - return comDataArray; - } - - /* - * We can reuse the buffers when not dealing with float samples. They are instantiated once just in time if needed. - */ - protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { - switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { - case BYTE: - this.preallocatedByteArray = this.preallocatedByteArray == null ? new byte[preallocatedSampleArray.length] : this.preallocatedByteArray; - buffer.get(preallocatedByteArray); - // Manual copy of byte[] into float[] - for (int k = 0; k < preallocatedSampleArray.length; k++) { - preallocatedSampleArray[k] = preallocatedByteArray[k]; - } - break; - case SHORT: - this.preallocatedShortArray = this.preallocatedShortArray == null ? new short[preallocatedSampleArray.length] : this.preallocatedShortArray; - ShortBuffer shortView = buffer.asShortBuffer(); - shortView.get(preallocatedShortArray); - // Manual copy of short[] into float[] - for (int k = 0; k < preallocatedSampleArray.length; k++) { - preallocatedSampleArray[k] = preallocatedShortArray[k]; - } - break; - case FLOAT: - FloatBuffer floatView = buffer.asFloatBuffer(); - floatView.get(preallocatedSampleArray); - break; - case INT: - this.preallocatedIntArray = this.preallocatedIntArray == null ? new int[preallocatedSampleArray.length] : this.preallocatedIntArray; - IntBuffer intView = buffer.asIntBuffer(); - intView.get(preallocatedIntArray); - // Manual copy of int[] into float[] - for (int k = 0; k < preallocatedIntArray.length; k++) { - preallocatedSampleArray[k] = (float) preallocatedIntArray[k]; - } - break; - default: - throw new TRSFormatException(String.format(UNKNOWN_SAMPLE_CODING, metaData.getInt(SAMPLE_CODING))); - } - - return preallocatedSampleArray; - } + public abstract TRSMetaData getMetaData(); /** * Factory method. This creates a new open TraceSet for reading. @@ -418,7 +70,7 @@ protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { * @throws TRSFormatException when any incorrect formatting of the TRS file is encountered */ public static TraceSet open(String file) throws IOException, TRSFormatException { - return new TraceSet(file); + return new ReadOnlyTraceSet(file); } /** @@ -484,6 +136,6 @@ public static TraceSet create(String file) throws IOException { */ public static TraceSet create(String file, TRSMetaData metaData) throws IOException { metaData.put(TRS_VERSION, 2, false); - return new TraceSet(file, metaData); + return new WritableTraceSet(file, metaData); } } diff --git a/src/main/java/com/riscure/trs/WritableTraceSet.java b/src/main/java/com/riscure/trs/WritableTraceSet.java new file mode 100644 index 0000000..aef3306 --- /dev/null +++ b/src/main/java/com/riscure/trs/WritableTraceSet.java @@ -0,0 +1,219 @@ +package com.riscure.trs; + +import com.riscure.trs.enums.Encoding; +import com.riscure.trs.enums.ParameterType; +import com.riscure.trs.parameter.TraceParameter; +import com.riscure.trs.parameter.primitive.StringParameter; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Map; + +import static com.riscure.trs.enums.TRSTag.*; + +public class WritableTraceSet extends TraceSet { + private static final String TRACE_SET_IN_WRITE_MODE = "TraceSet is in write mode. Please open the TraceSet in read mode."; + private static final String TRACE_LENGTH_DIFFERS = "All traces in a set need to be the same length, but current trace length (%d) differs from the previous trace(s) (%d)"; + private static final String TRACE_DATA_LENGTH_DIFFERS = "All traces in a set need to have the same data length, but current trace data length (%d) differs from the previous trace(s) (%d)"; + private static final String PARAMETER_NOT_DEFINED = "Parameter %s is saved in the trace, but was not found in the header definition"; + + private final CharsetDecoder utf8Decoder = StandardCharsets.UTF_8.newDecoder(); + private final TRSMetaData metaData; + private final FileOutputStream writeStream; + + private boolean firstTrace = true; + + WritableTraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { + super(Paths.get(outputFileName)); + this.metaData = metaData; + this.writeStream = new FileOutputStream(outputFileName); + } + + @Override + public Trace get(int index) throws IOException { + throw new IllegalArgumentException(TRACE_SET_IN_WRITE_MODE); + } + + @Override + public void add(Trace trace) throws IOException, TRSFormatException { + if (!isOpen()) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); + if (firstTrace) { + int dataLength = trace.getData() == null ? 0 : trace.getData().length; + int titleLength = trace.getTitle() == null ? 0 : trace.getTitle().getBytes(StandardCharsets.UTF_8).length; + metaData.put(NUMBER_OF_SAMPLES, trace.getNumberOfSamples(), false); + metaData.put(DATA_LENGTH, dataLength, false); + metaData.put(TITLE_SPACE, titleLength, false); + metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false); + metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters())); + TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); + firstTrace = false; + } + truncateStrings(trace, metaData); + checkValid(trace); + + trace.setTraceSet(this); + writeTrace(trace); + + int numberOfTraces = metaData.getInt(NUMBER_OF_TRACES); + metaData.put(NUMBER_OF_TRACES, numberOfTraces + 1); + } + + @Override + public TRSMetaData getMetaData() { + return metaData; + } + + @Override + public void close() throws IOException, TRSFormatException { + super.close(); + closeWriter(); + } + + private void closeWriter() throws IOException, TRSFormatException { + try { + //reset writer to start of file and overwrite header + writeStream.getChannel().position(0); + TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); + writeStream.flush(); + } finally { + writeStream.close(); + } + } + + /** + * This method makes sure that the trace title and any added string parameters adhere to the preset maximum length + * @param trace the trace to update + * @param metaData the metadata specifying the maximum string lengths + */ + private void truncateStrings(Trace trace, TRSMetaData metaData) { + int titleSpace = metaData.getInt(TITLE_SPACE); + trace.setTitle(fitUtf8StringToByteLength(trace.getTitle(), titleSpace)); + TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); + for (Map.Entry> definition : traceParameterDefinitionMap.entrySet()) { + TraceParameterDefinition value = definition.getValue(); + String key = definition.getKey(); + if (value.getType() == ParameterType.STRING) { + short stringLength = value.getLength(); + String stringValue = ((StringParameter) trace.getParameters().get(key)).getValue(); + if (stringLength != stringValue.getBytes(StandardCharsets.UTF_8).length) { + trace.getParameters().put(key, fitUtf8StringToByteLength(stringValue, stringLength)); + } + } + } + } + + /** + * Fits a string to the number of characters that fit in X bytes avoiding multi byte characters being cut in + * half at the cut off point. Also handles surrogate pairs where 2 characters in the string is actually one literal + * character. If the string is too long, it is truncated. If it's too short, it's padded with NUL characters. + * @param s the string to fit + * @param maxBytes the number of bytes required + */ + private String fitUtf8StringToByteLength(String s, int maxBytes) { + if (s == null) { + return null; + } + byte[] sba = s.getBytes(StandardCharsets.UTF_8); + if (sba.length <= maxBytes) { + return new String(Arrays.copyOf(sba, maxBytes)); + } + // Ensure truncation by having byte buffer = maxBytes + ByteBuffer bb = ByteBuffer.wrap(sba, 0, maxBytes); + CharBuffer cb = CharBuffer.allocate(maxBytes); + // Ignore an incomplete character + utf8Decoder.reset(); + utf8Decoder.onMalformedInput(CodingErrorAction.IGNORE); + utf8Decoder.decode(bb, cb, true); + utf8Decoder.flush(cb); + return new String(cb.array(), 0, cb.position()); + } + + private void writeTrace(Trace trace) throws TRSFormatException, IOException { + String title = trace.getTitle() == null ? "" : trace.getTitle(); + writeStream.write(title.getBytes(StandardCharsets.UTF_8)); + byte[] data = trace.getData() == null ? new byte[0] : trace.getData(); + writeStream.write(data); + Encoding encoding = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)); + writeStream.write(toByteArray(trace.getSample(), encoding)); + } + + private byte[] toByteArray(float[] samples, Encoding encoding) throws TRSFormatException { + byte[] result; + switch (encoding) { + case ILLEGAL: + throw new TRSFormatException("Illegal sample encoding"); + case BYTE: + result = new byte[samples.length]; + for (int k = 0; k < samples.length; k++) { + if (samples[k] != (byte)samples[k]) throw new IllegalArgumentException("Byte sample encoding too small"); + result[k] = (byte) samples[k]; + } + break; + case SHORT: + result = new byte[samples.length * 2]; + for (int k = 0; k < samples.length; k++) { + if (samples[k] != (short)samples[k]) throw new IllegalArgumentException("Short sample encoding too small"); + short value = (short) samples[k]; + result[2*k] = (byte) value; + result[2*k + 1] = (byte) (value >> 8); + } + break; + case INT: + result = new byte[samples.length * 4]; + for (int k = 0; k < samples.length; k++) { + int value = (int) samples[k]; + result[4*k] = (byte) value; + result[4*k + 1] = (byte) (value >> 8); + result[4*k + 2] = (byte) (value >> 16); + result[4*k + 3] = (byte) (value >> 24); + } + break; + case FLOAT: + result = new byte[samples.length * 4]; + for (int k = 0; k < samples.length; k++) { + int value = Float.floatToIntBits(samples[k]); + result[4*k] = (byte) value; + result[4*k + 1] = (byte) (value >> 8); + result[4*k + 2] = (byte) (value >> 16); + result[4*k + 3] = (byte) (value >> 24); + } + break; + default: + throw new TRSFormatException(String.format("Sample encoding not supported: %s", encoding.name())); + } + return result; + } + + private void checkValid(Trace trace) { + int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); + if (metaData.getInt(NUMBER_OF_SAMPLES) != trace.getNumberOfSamples()) { + throw new IllegalArgumentException(String.format(TRACE_LENGTH_DIFFERS, + trace.getNumberOfSamples(), + numberOfSamples)); + } + + int dataLength = metaData.getInt(DATA_LENGTH); + int traceDataLength = trace.getData() == null ? 0 : trace.getData().length; + if (metaData.getInt(DATA_LENGTH) != traceDataLength) { + throw new IllegalArgumentException(String.format(TRACE_DATA_LENGTH_DIFFERS, + traceDataLength, + dataLength)); + } + + for (Map.Entry entry : trace.getParameters().entrySet()) { + if (!metaData.getTraceParameterDefinitions().containsKey(entry.getKey())) { + throw new IllegalArgumentException(String.format(PARAMETER_NOT_DEFINED, entry.getKey())); + } + } + } +}