diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index acf3b6452..3a878a6bf 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -16,10 +16,10 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Set up JDK 11 + - name: Set up JDK 21 uses: actions/setup-java@v5 with: - java-version: '11' + java-version: '21' distribution: 'zulu' cache: maven diff --git a/Dockerfile b/Dockerfile index b921534cc..23da3f3b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ # Install library dependencies before actually building source. # This caches libraries into an image layer that can be reused when only source code has changed. -FROM azul/zulu-openjdk-debian:11 as build_environment +FROM azul/zulu-openjdk-debian:21 as build_environment LABEL maintainer="Forrest Collman , Eric Trautman " RUN apt-get update && apt-get install -y maven @@ -85,7 +85,7 @@ RUN mkdir -p /root/render-lib && \ # Once web service application is built, set up jetty server and deploy application to it. # NOTE: jetty version should be kept in sync with values in render/render-ws/pom.xml and render/render-ws/src/main/scripts/install.sh -FROM jetty:10.0.13-jre11 as render-ws +FROM jetty:10.0.26-jre21 as render-ws # add packages not included in base image: # curl and coreutils are always needed for gnu readlink, tzdata is needed to set timezone diff --git a/docs/pom.xml b/docs/pom.xml index e1a099058..4bad085ba 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -6,7 +6,7 @@ modules-root org.janelia.render - 4.3.0-SNAPSHOT + 5.0.0-SNAPSHOT Render Documents diff --git a/docs/src/site/markdown/how-to/how-to-local-spark-intellij.md b/docs/src/site/markdown/how-to/how-to-local-spark-intellij.md index 1fb918a1c..3fd2594de 100644 --- a/docs/src/site/markdown/how-to/how-to-local-spark-intellij.md +++ b/docs/src/site/markdown/how-to/how-to-local-spark-intellij.md @@ -1,7 +1,7 @@ # How To: Run Spark Locally in IntelliJ It can be useful to run and test render spark clients within an IDE -[using a local master URL](https://spark.apache.org/docs/3.4.1/submitting-applications.html#master-urls). Unfortunately +[using a local master URL](https://spark.apache.org/docs/4.0.4/submitting-applications.html#master-urls). Unfortunately because we need to exclude spark libraries from the render-ws-spark-client fat jar, you'll need to add a few extra settings to your IntelliJ run configuration to get spark clients to run within the IDE. @@ -10,7 +10,7 @@ few extra settings to your IntelliJ run configuration to get spark clients to ru The hadoop client runtime library is required to run spark locally. If you don't already have it in a local directory, an easy way to install it is to: - Comment out the library's exclusion block in the -[render-ws-spark-client pom.xml](../../../../../render-ws-spark-client/pom.xml#L213-L216): +[render-ws-spark-client pom.xml](../../../../../render-ws-spark-client/pom.xml#L267-L270): ``` 4.0.1 - - 2.1.3 - - - 3.1.9 - - 2.14.3 + + 2.18.6 1.6.2 @@ -173,9 +168,9 @@ gpl_v2 Howard Hughes Medical Institute - - 11 - [11,] + + 21 + [21,] false @@ -189,7 +184,20 @@ maven-compiler-plugin - 11 + 21 + + none diff --git a/render-app/pom.xml b/render-app/pom.xml index f1a5aca3b..73bb340b3 100644 --- a/render-app/pom.xml +++ b/render-app/pom.xml @@ -5,7 +5,7 @@ org.janelia.render modules-root - 4.3.0-SNAPSHOT + 5.0.0-SNAPSHOT Render Application @@ -164,11 +164,11 @@ n5-hdf5 - + com.fasterxml.jackson.core jackson-databind - ${jackson-version} @@ -182,7 +182,6 @@ sc.fiji Descriptor_based_registration - 2.1.3 diff --git a/render-app/src/main/java/org/janelia/alignment/match/CanvasPeakMatcher.java b/render-app/src/main/java/org/janelia/alignment/match/CanvasPeakMatcher.java index 0300c2129..ebba4c1d8 100644 --- a/render-app/src/main/java/org/janelia/alignment/match/CanvasPeakMatcher.java +++ b/render-app/src/main/java/org/janelia/alignment/match/CanvasPeakMatcher.java @@ -7,8 +7,8 @@ import mpicbg.imglib.type.numeric.real.FloatType; import mpicbg.models.Model; import mpicbg.models.PointMatch; -import mpicbg.pointdescriptor.matcher.Matcher; -import mpicbg.pointdescriptor.matcher.SubsetMatcher; +import net.preibisch.mvrecon.process.pointcloud.pointdescriptor.matcher.Matcher; +import net.preibisch.mvrecon.process.pointcloud.pointdescriptor.matcher.SubsetMatcher; import mpicbg.util.Timer; import org.janelia.alignment.match.parameters.GeometricDescriptorParameters; diff --git a/render-app/src/main/java/org/janelia/alignment/match/parameters/GeometricDescriptorParameters.java b/render-app/src/main/java/org/janelia/alignment/match/parameters/GeometricDescriptorParameters.java index 23bd38dff..50e9ae365 100644 --- a/render-app/src/main/java/org/janelia/alignment/match/parameters/GeometricDescriptorParameters.java +++ b/render-app/src/main/java/org/janelia/alignment/match/parameters/GeometricDescriptorParameters.java @@ -5,7 +5,7 @@ import java.io.Serializable; import java.util.Objects; -import mpicbg.spim.segmentation.InteractiveDoG; +import net.preibisch.mvrecon.fiji.plugin.interestpointdetection.interactive.HelperFunctions; import plugin.DescriptorParameters; @@ -149,7 +149,7 @@ public DescriptorParameters toDescriptorParameters() { dp.redundancy = redundancy; dp.significance = significance; dp.sigma1 = sigma; - dp.sigma2 = InteractiveDoG.computeSigma2(sigma.floatValue(), InteractiveDoG.standardSensitivity ); + dp.sigma2 = HelperFunctions.computeSigma2(sigma, 4); // 4 = former InteractiveDoG.standardSensitivity dp.threshold = threshold; dp.localization = localization.code; dp.lookForMinima = lookForMinima; diff --git a/render-app/src/main/java/process/GeometricDescriptorMatcher.java b/render-app/src/main/java/process/GeometricDescriptorMatcher.java index bade81e44..d4031145a 100644 --- a/render-app/src/main/java/process/GeometricDescriptorMatcher.java +++ b/render-app/src/main/java/process/GeometricDescriptorMatcher.java @@ -7,7 +7,7 @@ import mpicbg.imglib.type.numeric.real.FloatType; import mpicbg.models.Model; import mpicbg.models.PointMatch; -import mpicbg.pointdescriptor.matcher.Matcher; +import net.preibisch.mvrecon.process.pointcloud.pointdescriptor.matcher.Matcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/render-app/src/test/java/org/janelia/alignment/match/GeometricDescriptorSIFTMatcherTest.java b/render-app/src/test/java/org/janelia/alignment/match/GeometricDescriptorSIFTMatcherTest.java index 89a67dea7..18539b1d4 100644 --- a/render-app/src/test/java/org/janelia/alignment/match/GeometricDescriptorSIFTMatcherTest.java +++ b/render-app/src/test/java/org/janelia/alignment/match/GeometricDescriptorSIFTMatcherTest.java @@ -15,7 +15,7 @@ import mpicbg.models.AffineModel2D; import mpicbg.models.Point; import mpicbg.models.PointMatch; -import mpicbg.spim.io.IOFunctions; +import net.preibisch.legacy.io.IOFunctions; import mpicbg.trakem2.transform.TransformMeshMappingWithMasks.ImageProcessorWithMasks; import org.janelia.alignment.RenderParameters; diff --git a/render-app/src/test/java/org/janelia/alignment/spec/TileSpecTest.java b/render-app/src/test/java/org/janelia/alignment/spec/TileSpecTest.java index 2228c0868..98729427e 100644 --- a/render-app/src/test/java/org/janelia/alignment/spec/TileSpecTest.java +++ b/render-app/src/test/java/org/janelia/alignment/spec/TileSpecTest.java @@ -34,19 +34,19 @@ public void testJsonProcessing() { final Map.Entry firstMipMap = tileSpec.getFirstMipmapEntry(); Assert.assertNotNull("first mipmap entry is null", firstMipMap); Assert.assertEquals("mipmap sorting failed, unexpected first entry returned", - new Integer(0), firstMipMap.getKey()); + Integer.valueOf(0), firstMipMap.getKey()); - final ChannelSpec channelSpec = tileSpec.getAllChannels().get(0); + final ChannelSpec channelSpec = tileSpec.getAllChannels().getFirst(); Map.Entry floorMipMap = channelSpec.getFloorMipmapEntry(3); Assert.assertNotNull("floor 3 mipmap entry is null", floorMipMap); Assert.assertEquals("invalid key for floor 3 mipmap entry", - new Integer(3), floorMipMap.getKey()); + Integer.valueOf(3), floorMipMap.getKey()); floorMipMap = channelSpec.getFloorMipmapEntry(4); Assert.assertNotNull("floor 4 mipmap entry is null", floorMipMap); Assert.assertEquals("invalid key for floor 3 mipmap entry", - new Integer(3), floorMipMap.getKey()); + Integer.valueOf(3), floorMipMap.getKey()); final FilterSpec filterSpec = channelSpec.getFilterSpec(); Assert.assertNotNull("filterSpec is null", filterSpec); diff --git a/render-ws-java-client/Dockerfile b/render-ws-java-client/Dockerfile index cfccdbb18..d8799bb87 100644 --- a/render-ws-java-client/Dockerfile +++ b/render-ws-java-client/Dockerfile @@ -14,7 +14,7 @@ # so I switched to use the same zulu image used for the final image # rather than try to debug alpine issues. -FROM azul/zulu-openjdk-debian:11 as builder +FROM azul/zulu-openjdk-debian:21 as builder RUN apt-get update && apt-get install -y git maven @@ -34,7 +34,7 @@ RUN git clone --branch $GIT_TAG https://github.com/saalfeldlab/render.git . \ # --------------------------------------------------------------------- # Stage 2: Create slimmed down final image that only contains client jar and scripts. -FROM azul/zulu-openjdk-debian:11 +FROM azul/zulu-openjdk-debian:21 # ARGs need to be re-declared for each stage in multi-stage builds - who knew? ARG GIT_COMMIT diff --git a/render-ws-java-client/pom.xml b/render-ws-java-client/pom.xml index 6a304d1cd..c25392a68 100644 --- a/render-ws-java-client/pom.xml +++ b/render-ws-java-client/pom.xml @@ -5,7 +5,7 @@ modules-root org.janelia.render - 4.3.0-SNAPSHOT + 5.0.0-SNAPSHOT Render Web Service Java Client @@ -111,7 +111,7 @@ - sc.fiji:SPIM_Registration + net.preibisch:multiview-reconstruction META-INF/json/** @@ -270,11 +270,11 @@ ${n5-version} - + com.fasterxml.jackson.core jackson-databind - ${jackson-version} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/FixMaskClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/FixMaskClient.java index 589bb0837..36d63dd8c 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/FixMaskClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/FixMaskClient.java @@ -98,7 +98,7 @@ public void runClient(final String[] args) client.setUpDerivedStack(); for (final String z : parameters.zValues) { - client.fixStackDataForZ(new Double(z)); + client.fixStackDataForZ(Double.valueOf(z)); } if (parameters.completeTargetStack) { @@ -128,7 +128,7 @@ private FixMaskClient(final Parameters parameters) .filter(path -> path.toString().endsWith(".png")) .forEach(path -> fixedMaskNamesToUrls.put(path.getFileName().toString(), "file:" + path)); - if (this.fixedMaskNamesToUrls.size() == 0) { + if (this.fixedMaskNamesToUrls.isEmpty()) { throw new IOException(fixedMasksDirectoryPath + " does not contain any fixed .png mask files"); } } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MontageOutlierDiagnosticsClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MontageOutlierDiagnosticsClient.java index e47681861..b167184da 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MontageOutlierDiagnosticsClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MontageOutlierDiagnosticsClient.java @@ -160,7 +160,7 @@ private void run() // Z,Max PM Residual,Tile 1,Tile 2,PM X,PM Y,Mean Tile Pair Residual Is Outlier final String[] v = line.split(","); if (v.length > 3) { - final Double z = new Double(v[0]); + final Double z = Double.valueOf(v[0]); if ((zValues == null) || (zValues.contains(z))) { final List pairList = zToOutlierPairsMap.computeIfAbsent(z, diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/StackClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/StackClient.java index 334360ff2..016633520 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/StackClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/StackClient.java @@ -241,7 +241,7 @@ private void cloneStackVersion() if (parameters.zValues != null) { zValues = new ArrayList<>(parameters.zValues.size()); for (final String zString : parameters.zValues) { - zValues.add(new Double(zString)); + zValues.add(Double.valueOf(zString)); } } @@ -304,9 +304,9 @@ private void deleteStack() renderDataClient.deleteStackSection(stack, parameters.sectionId); } } else { - Double z; + double z; for (final String zString : parameters.zValues) { - z = new Double(zString); + z = Double.parseDouble(zString); renderDataClient.deleteStack(stack, z); } if (parameters.sectionId != null) { diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/TerraceLayersClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/TerraceLayersClient.java index e56a22166..692136434 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/TerraceLayersClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/TerraceLayersClient.java @@ -104,7 +104,7 @@ public static class Parameters extends CommandLineParameters { public String roughMatchCollection; public String getTargetStack() { - if ((targetStack == null) || (targetStack.trim().length() == 0)) { + if ((targetStack == null) || (targetStack.trim().isEmpty())) { targetStack = stack; } return targetStack; @@ -147,7 +147,7 @@ private TerraceLayersClient(final Parameters parameters) this.sourceRenderDataClient = parameters.renderWeb.getDataClient(); if ((parameters.targetProject == null) || - (parameters.targetProject.trim().length() == 0) || + (parameters.targetProject.trim().isEmpty()) || (parameters.targetProject.equals(parameters.renderWeb.project))) { this.targetRenderDataClient = sourceRenderDataClient; } else { @@ -254,14 +254,14 @@ private void mapMinCoordinatesByConnectedCluster(final List sortedZs, for (final Set connectedGroupIdSet : connectedGroupIdSets) { Bounds connectedBounds = null; for (final String groupId : connectedGroupIdSet) { - final Double z = new Double(groupId); + final Double z = Double.valueOf(groupId); final Bounds layerBounds = zToBoundsMap.get(z); connectedBounds = connectedBounds == null ? layerBounds : connectedBounds.union(layerBounds); connectedLayerCount++; } for (final String groupId : connectedGroupIdSet) { - final Double z = new Double(groupId); + final Double z = Double.valueOf(groupId); zToBoundsMap.put(z, connectedBounds); zToMinCoordinateMap.put(z, minCoordinate); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ValidateTilesClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ValidateTilesClient.java index f1dcc9c44..0c3ae69b6 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ValidateTilesClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ValidateTilesClient.java @@ -67,7 +67,7 @@ public void runClient(final String[] args) throws Exception { final ValidateTilesClient client = new ValidateTilesClient(parameters); for (final String z : parameters.zValues) { - client.validateTilesForZ(new Double(z)); + client.validateTilesForZ(Double.valueOf(z)); } client.completeStackIfNecessary(); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/destreak/StreakCorrection_Plugin.java b/render-ws-java-client/src/main/java/org/janelia/render/client/destreak/StreakCorrection_Plugin.java index 29cd73a50..952b117c1 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/destreak/StreakCorrection_Plugin.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/destreak/StreakCorrection_Plugin.java @@ -22,7 +22,7 @@ import org.janelia.render.client.RenderDataClient; import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.parameter.RenderWebServiceParameters; -import spim.Threads; +import net.preibisch.mvrecon.Threads; import javax.swing.SwingUtilities; import java.awt.KeyboardFocusManager; diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/intensityadjust/IntensityCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/intensityadjust/IntensityCorrectionClient.java index c11987cc0..418ff578c 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/intensityadjust/IntensityCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/intensityadjust/IntensityCorrectionClient.java @@ -25,6 +25,7 @@ public void runClient(final String[] args) throws Exception { final IntensityAdjustParameters parameters = new IntensityAdjustParameters(); parameters.parse(args); + parameters.algorithmic.initDefaultValues(); LOG.info("runClient: entry, parameters={}", parameters); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/solver/DistributedSolveMultiThread.java b/render-ws-java-client/src/main/java/org/janelia/render/client/solver/DistributedSolveMultiThread.java index 9d69da039..fd2418f8d 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/solver/DistributedSolveMultiThread.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/solver/DistributedSolveMultiThread.java @@ -17,7 +17,7 @@ import bdv.util.BdvStackSource; import mpicbg.models.Affine2D; -import mpicbg.spim.io.IOFunctions; +import net.preibisch.legacy.io.IOFunctions; import net.imglib2.multithreading.SimpleMultiThreading; public class DistributedSolveMultiThread extends DistributedSolve diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/ThicknessCorrectionData.java b/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/ThicknessCorrectionData.java index 2442d22a7..684120a30 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/ThicknessCorrectionData.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/ThicknessCorrectionData.java @@ -138,8 +138,8 @@ public ThicknessCorrectionData(final List correctionDataLines) for (int linesIndex = 0; linesIndex < correctionDataLines.size(); linesIndex++) { final Matcher matcher = linePattern.matcher(correctionDataLines.get(linesIndex)); if (matcher.matches()) { - final Integer stackZ = new Integer(matcher.group(1)); - final Double correctedZ = new Double(matcher.group(2)); + final Integer stackZ = Integer.valueOf(matcher.group(1)); + final Double correctedZ = Double.valueOf(matcher.group(2)); if ((lastCorrectedZ != null) && (correctedZ <= lastCorrectedZ)) { throw new IllegalArgumentException("out of order or duplicate correctedZ value on line " + (linesIndex + 1)); @@ -156,8 +156,8 @@ public ThicknessCorrectionData(final List correctionDataLines) // pin first and last corrected z values to first and last stack z values unless they are very different final double maxDelta = 0.5; - final Double firstCorrectedZ = orderedCorrectedZValues.get(0); - final Integer firstStackZ = orderedStackZValues.get(0); + final Double firstCorrectedZ = orderedCorrectedZValues.getFirst(); + final Integer firstStackZ = orderedStackZValues.getFirst(); double delta = Math.abs(firstCorrectedZ - firstStackZ); if (delta < maxDelta) { orderedCorrectedZValues.set(0, firstStackZ.doubleValue()); @@ -180,11 +180,11 @@ public ThicknessCorrectionData(final List correctionDataLines) } public double getFirstCorrectedZ() { - return orderedCorrectedZValues.get(0); + return orderedCorrectedZValues.getFirst(); } public double getLastCorrectedZ() { - return orderedCorrectedZValues.get(orderedCorrectedZValues.size() - 1); + return orderedCorrectedZValues.getLast(); } /** diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/HeadlessZPositionCorrectionTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/HeadlessZPositionCorrectionTest.java index 3de5e39fc..dfae22c43 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/HeadlessZPositionCorrectionTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/HeadlessZPositionCorrectionTest.java @@ -294,7 +294,7 @@ private static void testEstimationsForRenderSlices() throws IOException { final int minZ = 10; final int maxZ = minZ + 9; // inclusive final List sortedZList = IntStream.rangeClosed(minZ, maxZ) - .boxed().map(Double::new).collect(Collectors.toList()); + .boxed().map(Double::valueOf).collect(Collectors.toList()); // for 19m VNC, layers were rendered at scale 0.125 final double renderScale = 0.25; diff --git a/render-ws-spark-client/pom.xml b/render-ws-spark-client/pom.xml index 1fe0ddb57..cea51c579 100644 --- a/render-ws-spark-client/pom.xml +++ b/render-ws-spark-client/pom.xml @@ -6,7 +6,7 @@ modules-root org.janelia.render - 4.3.0-SNAPSHOT + 5.0.0-SNAPSHOT Render Web Service Spark Client @@ -94,7 +94,7 @@ - sc.fiji:SPIM_Registration + net.preibisch:multiview-reconstruction META-INF/json/** @@ -209,38 +209,22 @@ org.janelia.saalfeldlab n5-spark - 3.7.3 + + 4.1.0 - + com.esotericsoftware kryo - - javax.validation - validation-api - - - - com.google.http-client - google-http-client-xml - - - - com.kjetland - mbknor-jackson-jsonschema_2.12 - - + com.fasterxml.jackson.core jackson-databind - ${jackson-version} @@ -248,7 +232,12 @@ org.apache.spark spark-core_2.13 - 3.4.1 + + 4.0.4 provided @@ -275,9 +264,9 @@ Comment out the hadoop-client-runtime exclusion below and run mvn compile to pull it into your local .m2/repository. The compile will fail because of the enforcer no-duplicate-classes rule but the jar will get pulled. - Once $HOME/.m2/repository/org/apache/hadoop/hadoop-client-runtime/3.3.4/hadoop-client-runtime-3.3.4.jar exists, + Once $HOME/.m2/repository/org/apache/hadoop/hadoop-client-runtime/3.4.1/hadoop-client-runtime-3.4.1.jar exists, uncomment the exclusion and run mvn clean package. - Finally, add the hadoop-client-runtime-3.3.4.jar to IntelliJ run configurations to run local Spark jobs from IDE. + Finally, add the hadoop-client-runtime-3.4.1.jar to IntelliJ run configurations to run local Spark jobs from IDE. Note that the hadoop version will change with Spark version changes. Hopefully, whoever updates Spark will also update this comment with the new corresponding hadoop version and .m2 path. diff --git a/render-ws-with-mongo-db/Dockerfile b/render-ws-with-mongo-db/Dockerfile index 61711d795..6df517074 100644 --- a/render-ws-with-mongo-db/Dockerfile +++ b/render-ws-with-mongo-db/Dockerfile @@ -11,23 +11,23 @@ # # To build with CORS enabled only for https://neuroglancer-demo.appspot.com (the default build): # cd ${RENDER_REPO_ROOT_DIR} -# docker build --file render-ws-with-mongo-db/Dockerfile --no-cache --build-arg CORS_ALLOWED_ORIGINS='https://neuroglancer-demo.appspot.com' --tag ghcr.io/janeliascicomp/render-ws-with-mongodb:0.0.21 . +# docker build --file render-ws-with-mongo-db/Dockerfile --no-cache --build-arg CORS_ALLOWED_ORIGINS='https://neuroglancer-demo.appspot.com' --tag ghcr.io/janeliascicomp/render-ws-with-mongodb:1.0.1 . # # To build with CORS enabled for all origins (note tag suffix '.cors-star'): # cd ${RENDER_REPO_ROOT_DIR} -# docker build --file render-ws-with-mongo-db/Dockerfile --no-cache --build-arg CORS_ALLOWED_ORIGINS='*' --tag ghcr.io/janeliascicomp/render-ws-with-mongodb:0.0.21.cors-star . +# docker build --file render-ws-with-mongo-db/Dockerfile --no-cache --build-arg CORS_ALLOWED_ORIGINS='*' --tag ghcr.io/janeliascicomp/render-ws-with-mongodb:1.0.1.cors-star . # # To run a container in detached mode: -# docker run --publish 8080:8080 --detach --rm ghcr.io/janeliascicomp/render-ws-with-mongodb:0.0.21 +# docker run --publish 8080:8080 --detach --rm ghcr.io/janeliascicomp/render-ws-with-mongodb:1.0.1 # # To run a container in detached mode with the standard MongoDB dump directory mounted: -# docker run --publish 8080:8080 --detach --rm --mount type=bind,source=/mnt/disks/mongodb_dump_fs/dump,target=/mnt/disks/mongodb_dump_fs/dump ghcr.io/janeliascicomp/render-ws-with-mongodb:0.0.21 +# docker run --publish 8080:8080 --detach --rm --mount type=bind,source=/mnt/disks/mongodb_dump_fs/dump,target=/mnt/disks/mongodb_dump_fs/dump ghcr.io/janeliascicomp/render-ws-with-mongodb:1.0.1 # # To run a container with 4GB headroom and the test/two_z_with_four_tiles/mongodb/render collections loaded at startup: -# docker run --publish 8080:8080 --detach --rm ghcr.io/janeliascicomp/render-ws-with-mongodb:0.0.21 4 test/two_z_with_four_tiles/mongodb/render +# docker run --publish 8080:8080 --detach --rm ghcr.io/janeliascicomp/render-ws-with-mongodb:1.0.1 4 test/two_z_with_four_tiles/mongodb/render # # To launch an interactive bash shell within a new container: -# docker run --interactive --tty --publish 8080:8080 --entrypoint /bin/bash --rm ghcr.io/janeliascicomp/render-ws-with-mongodb:0.0.21 +# docker run --interactive --tty --publish 8080:8080 --entrypoint /bin/bash --rm ghcr.io/janeliascicomp/render-ws-with-mongodb:1.0.1 # # To launch an interactive bash shell within an already running container: # docker ps -a # to get the container id @@ -37,7 +37,7 @@ # docker exec --interactive --tty "$(docker ps -q)" /bin/bash # # To push the image to the janeliascicomp containers GitHub registry (see https://github.com/JaneliaSciComp/containers/tree/main): -# docker push ghcr.io/janeliascicomp/render-ws-with-mongodb:0.0.21 +# docker push ghcr.io/janeliascicomp/render-ws-with-mongodb:1.0.1 # # Published Containers Notes: # - Before pushing, your docker instance needs to be logged in to GitHub. diff --git a/render-ws/pom.xml b/render-ws/pom.xml index 6785ea5db..02afb1629 100644 --- a/render-ws/pom.xml +++ b/render-ws/pom.xml @@ -5,7 +5,7 @@ modules-root org.janelia.render - 4.3.0-SNAPSHOT + 5.0.0-SNAPSHOT Render Web Service @@ -102,11 +102,11 @@ ${n5-version} - + com.fasterxml.jackson.core jackson-databind - ${jackson-version} @@ -161,16 +161,23 @@ resteasy-servlet-initializer + com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - ${jackson-version} + ${jackson.version} jakarta.activation jakarta.activation-api + + + javax.xml.bind + jaxb-api + @@ -228,11 +235,21 @@ It is important that this flapdoodle dependency (for testing) is listed after the swagger dependencies above. Swagger currently requires a newer version of commons-lang3 than flapdoodle. + TODO: determine if this constraint still exists after upgrading to flapdoodle 4.33.0 --> de.flapdoodle.embed de.flapdoodle.embed.mongo - 3.1.4 + + 4.33.0 test @@ -247,7 +264,7 @@ 4.7.9.Final - 10.0.13 + 10.0.26 4.4.1 gpl_v2 Howard Hughes Medical Institute diff --git a/render-ws/src/main/java/org/janelia/render/service/util/ConfiguredJsonProvider.java b/render-ws/src/main/java/org/janelia/render/service/util/ConfiguredJsonProvider.java index 9429e104b..9232b7c39 100644 --- a/render-ws/src/main/java/org/janelia/render/service/util/ConfiguredJsonProvider.java +++ b/render-ws/src/main/java/org/janelia/render/service/util/ConfiguredJsonProvider.java @@ -30,19 +30,15 @@ public ConfiguredJsonProvider () { } @Override - protected ObjectMapper _locateMapperViaProvider(final Class type, - final MediaType mediaType) { - - final ObjectMapper mapper; + protected ObjectMapper _locateMapperViaProvider(final Class type, final MediaType mediaType) { + return getMapperForType(type); + } + static ObjectMapper getMapperForType(final Class type) { // for collections and lists, use fast mapper which skips pretty printing if ((type == ResolvedTileSpecCollection.class) || (type == ArrayList.class)) { - mapper = JsonUtils.FAST_MAPPER; - } else { - mapper = JsonUtils.MAPPER; + return JsonUtils.FAST_MAPPER; } - - return mapper; + return JsonUtils.MAPPER; } - } diff --git a/render-ws/src/main/java/org/janelia/render/service/util/ObjectMapperContextResolver.java b/render-ws/src/main/java/org/janelia/render/service/util/ObjectMapperContextResolver.java new file mode 100644 index 000000000..0c260db42 --- /dev/null +++ b/render-ws/src/main/java/org/janelia/render/service/util/ObjectMapperContextResolver.java @@ -0,0 +1,28 @@ +package org.janelia.render.service.util; + +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.ext.ContextResolver; +import javax.ws.rs.ext.Provider; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * RESTEasy can pick either {@link ConfiguredJsonProvider} or the library's own default JSON provider + * to handle a response. Both providers now match "application/json" equally well, so the pick + * is not deterministic. + *

+ * This class removes the need to control that pick. Both providers ask a {@code ContextResolver} for + * the {@code ObjectMapper} before they serialize a response. This class is that {@code ContextResolver}. It + * always returns the correct field-visibility mapper, no matter which provider RESTEasy chose. + */ +@Provider +@Produces(MediaType.APPLICATION_JSON) +public class ObjectMapperContextResolver implements ContextResolver { + + @Override + public ObjectMapper getContext(final Class type) { + return ConfiguredJsonProvider.getMapperForType(type); + } + +} diff --git a/render-ws/src/main/scripts/install.sh b/render-ws/src/main/scripts/install.sh index fb25d24b4..653d3fa03 100755 --- a/render-ws/src/main/scripts/install.sh +++ b/render-ws/src/main/scripts/install.sh @@ -1,6 +1,6 @@ #!/bin/bash -JETTY_VERSION="10.0.13" # NOTE: jetty version should be kept in sync with values in render/render-ws/pom.xml and render/Dockerfile +JETTY_VERSION="10.0.26" # NOTE: jetty version should be kept in sync with values in render/render-ws/pom.xml and render/Dockerfile JETTY_DIST="jetty-home-${JETTY_VERSION}" # URLs for Jetty 10, SLF4J 1.7, Logback 1.1, and Swagger 2.1 MAVEN_CENTRAL_URL="https://repo1.maven.org" @@ -91,7 +91,7 @@ JETTY_WRAPPER_SCRIPT="${JETTY_BASE}/jetty_wrapper.sh" sed " s~/opt/local/jetty_home~${JETTY_HOME}~ s~/opt/local/jetty_base~${JETTY_BASE}~ - s~/misc/sc/jdks/zulu11~${JAVA_HOME}~ + s~/misc/sc/jdks/zulu21~${JAVA_HOME}~ " "${SCRIPTS_DIR}"/jetty/jetty_wrapper.sh > "${JETTY_WRAPPER_SCRIPT}" chmod 755 "${JETTY_WRAPPER_SCRIPT}" diff --git a/render-ws/src/main/scripts/jdk-vars.sh b/render-ws/src/main/scripts/jdk-vars.sh index ca88833a7..68f4c3fa7 100755 --- a/render-ws/src/main/scripts/jdk-vars.sh +++ b/render-ws/src/main/scripts/jdk-vars.sh @@ -1,8 +1,8 @@ #!/bin/bash -export JDK_VERSION="zulu11.78.15-ca-jdk11.0.26-linux_x64" +export JDK_VERSION="zulu21.52.15-ca-jdk21.0.12-linux_x64" # URL for JDK # You can find latest Linux x64 download links at: -# https://www.azul.com/downloads/?version=java-11-lts&os=linux&architecture=x86-64-bit&package=jdk&show-old-builds=true#zulu +# https://www.azul.com/downloads/?version=java-21-lts&os=linux&architecture=x86-64-bit&package=jdk&show-old-builds=true#zulu export JDK_URL="https://cdn.azul.com/zulu/bin/${JDK_VERSION}.tar.gz" \ No newline at end of file diff --git a/render-ws/src/main/scripts/jetty/jetty_wrapper.sh b/render-ws/src/main/scripts/jetty/jetty_wrapper.sh index 81a1b0d07..1a42783be 100755 --- a/render-ws/src/main/scripts/jetty/jetty_wrapper.sh +++ b/render-ws/src/main/scripts/jetty/jetty_wrapper.sh @@ -21,7 +21,7 @@ export JETTY_STATE="${JETTY_RUN}/jetty.state" # JETTY_USER # JETTY_SHELL -export JAVA_HOME="/misc/sc/jdks/zulu11" +export JAVA_HOME="/misc/sc/jdks/zulu21" export PATH="${JAVA_HOME}/bin:${PATH}" export JAVA="${JAVA_HOME}/bin/java" export JAVA_OPTIONS="-Xms${JETTY_MIN_AND_MAX_MEMORY} -Xmx${JETTY_MIN_AND_MAX_MEMORY} -server -Djava.awt.headless=true" diff --git a/render-ws/src/test/java/org/janelia/render/service/StackMetaDataServiceTest.java b/render-ws/src/test/java/org/janelia/render/service/StackMetaDataServiceTest.java index 96f3a4df8..b624a3463 100644 --- a/render-ws/src/test/java/org/janelia/render/service/StackMetaDataServiceTest.java +++ b/render-ws/src/test/java/org/janelia/render/service/StackMetaDataServiceTest.java @@ -44,7 +44,7 @@ public class StackMetaDataServiceTest { private static RenderDao renderDao; @BeforeClass - public static void before() throws Exception { + public static void before() { loadingStackId = new StackId("flyTEM", "test_project", "test_stack"); completeStackId = new StackId("flyTEM", "test", "elastic"); @@ -75,7 +75,7 @@ public void setUp() throws Exception { } @AfterClass - public static void after() throws Exception { + public static void after() { embeddedMongoDb.stop(); } @@ -199,14 +199,14 @@ public void testSetStackState() throws Exception { Assert.assertEquals("invalid stackBounds.minZ", 3903.0, stackBounds.getMinZ(), 0.01); - Assert.assertEquals("invalid sectionCount", new Long(2), stats.getSectionCount()); - Assert.assertEquals("invalid nonIntegralSectionCount", new Long(1), stats.getNonIntegralSectionCount()); - Assert.assertEquals("invalid tileCount", new Long(14), stats.getTileCount()); - Assert.assertEquals("invalid transformCount", new Long(3), stats.getTransformCount()); - Assert.assertEquals("invalid minTileWidth", new Integer(2631), stats.getMinTileWidth()); - Assert.assertEquals("invalid maxTileWidth", new Integer(2772), stats.getMaxTileWidth()); - Assert.assertEquals("invalid minTileHeight", new Integer(2257), stats.getMinTileHeight()); - Assert.assertEquals("invalid maxTileHeight", new Integer(2414), stats.getMaxTileHeight()); + Assert.assertEquals("invalid sectionCount", Long.valueOf(2), stats.getSectionCount()); + Assert.assertEquals("invalid nonIntegralSectionCount", Long.valueOf(1), stats.getNonIntegralSectionCount()); + Assert.assertEquals("invalid tileCount", Long.valueOf(14), stats.getTileCount()); + Assert.assertEquals("invalid transformCount", Long.valueOf(3), stats.getTransformCount()); + Assert.assertEquals("invalid minTileWidth", Integer.valueOf(2631), stats.getMinTileWidth()); + Assert.assertEquals("invalid maxTileWidth", Integer.valueOf(2772), stats.getMaxTileWidth()); + Assert.assertEquals("invalid minTileHeight", Integer.valueOf(2257), stats.getMinTileHeight()); + Assert.assertEquals("invalid maxTileHeight", Integer.valueOf(2414), stats.getMaxTileHeight()); final Bounds stackBounds2 = service.getStackBounds(completeStackId.getOwner(), completeStackId.getProject(), diff --git a/render-ws/src/test/java/org/janelia/render/service/dao/MatchDaoTest.java b/render-ws/src/test/java/org/janelia/render/service/dao/MatchDaoTest.java index 1ef8a2469..0c9812aff 100644 --- a/render-ws/src/test/java/org/janelia/render/service/dao/MatchDaoTest.java +++ b/render-ws/src/test/java/org/janelia/render/service/dao/MatchDaoTest.java @@ -223,23 +223,23 @@ public void testRemoveMatches() { MatchCollectionMetaData collectionMetaData = getCollectionMetaData(deletionCollectionId); Assert.assertEquals("invalid pair count before deletions", - new Long(12), collectionMetaData.getPairCount()); + Long.valueOf(12), collectionMetaData.getPairCount()); dao.removeMatchesBetweenTiles(deletionCollectionId, "0", tileA, "0", tileB); collectionMetaData = getCollectionMetaData(deletionCollectionId); Assert.assertEquals("invalid pair count after removing one tile pair", - new Long(11), collectionMetaData.getPairCount()); + Long.valueOf(11), collectionMetaData.getPairCount()); dao.removeMatchesBetweenGroups(deletionCollectionId, "0", "7"); collectionMetaData = getCollectionMetaData(deletionCollectionId); Assert.assertEquals("invalid pair count after removing pairs between groups 0 and 7", - new Long(10), collectionMetaData.getPairCount()); + Long.valueOf(10), collectionMetaData.getPairCount()); dao.removeMatchesOutsideGroup(deletionCollectionId, "0"); collectionMetaData = getCollectionMetaData(deletionCollectionId); Assert.assertEquals("invalid pair count after removing pairs outside group 0", - new Long(8), collectionMetaData.getPairCount()); + Long.valueOf(8), collectionMetaData.getPairCount()); dao.removeAllMatches(deletionCollectionId); collectionMetaData = getCollectionMetaData(deletionCollectionId); @@ -269,39 +269,40 @@ public void testRenameMatchCollection() { public void testMatchTrial() { final String json = - "{\n" + - " \"parameters\" : {\n" + - " \"featureAndMatchParameters\" : {\n" + - " \"siftFeatureParameters\" : {\n" + - " \"fdSize\" : 4,\n" + - " \"minScale\" : 0.5,\n" + - " \"maxScale\" : 1.0,\n" + - " \"steps\" : 3\n" + - " },\n" + - " \"matchDerivationParameters\" : {\n" + - " \"matchRod\" : 0.95,\n" + - " \"matchModelType\" : \"AFFINE\",\n" + - " \"matchIterations\" : 1000,\n" + - " \"matchMaxEpsilon\" : 5.0,\n" + - " \"matchMinInlierRatio\" : 0.0,\n" + - " \"matchMinNumInliers\" : 6,\n" + - " \"matchMaxTrust\" : 30.0,\n" + - " \"matchFilter\" : \"AGGREGATED_CONSENSUS_SETS\"\n" + - " }\n" + - " },\n" + - " \"pRenderParametersUrl\" : \"http://renderer-dev:8080/render-ws/v1/owner/flyTEM/project/spc_mm2_sample_rough_test_1_tier_3/stack/0016x0017_000118/tile/z_1015.0_box_5632_6656_1024_1024_0.500000/render-parameters?excludeMask=true&normalizeForMatching=true&filter=true&fillWithNoise=true\",\n" + - " \"qRenderParametersUrl\" : \"http://renderer-dev:8080/render-ws/v1/owner/flyTEM/project/spc_mm2_sample_rough_test_1_tier_3/stack/0016x0017_000118/tile/z_1016.0_box_5632_6656_1024_1024_0.500000/render-parameters?excludeMask=true&normalizeForMatching=true&filter=true&fillWithNoise=true\"\n" + - " },\n" + - " \"matches\" : [ ],\n" + - " \"stats\" : {\n" + - " \"pFeatureCount\" : 996,\n" + - " \"pFeatureDerivationMilliseconds\" : 1415,\n" + - " \"qFeatureCount\" : 1133,\n" + - " \"qFeatureDerivationMilliseconds\" : 1279,\n" + - " \"consensusSetSizes\" : [ 0 ],\n" + - " \"matchDerivationMilliseconds\" : 324\n" + - " }\n" + - "}"; + """ + { + "parameters" : { + "featureAndMatchParameters" : { + "siftFeatureParameters" : { + "fdSize" : 4, + "minScale" : 0.5, + "maxScale" : 1.0, + "steps" : 3 + }, + "matchDerivationParameters" : { + "matchRod" : 0.95, + "matchModelType" : "AFFINE", + "matchIterations" : 1000, + "matchMaxEpsilon" : 5.0, + "matchMinInlierRatio" : 0.0, + "matchMinNumInliers" : 6, + "matchMaxTrust" : 30.0, + "matchFilter" : "AGGREGATED_CONSENSUS_SETS" + } + }, + "pRenderParametersUrl" : "http://renderer-dev:8080/render-ws/v1/owner/flyTEM/project/spc_mm2_sample_rough_test_1_tier_3/stack/0016x0017_000118/tile/z_1015.0_box_5632_6656_1024_1024_0.500000/render-parameters?excludeMask=true&normalizeForMatching=true&filter=true&fillWithNoise=true", + "qRenderParametersUrl" : "http://renderer-dev:8080/render-ws/v1/owner/flyTEM/project/spc_mm2_sample_rough_test_1_tier_3/stack/0016x0017_000118/tile/z_1016.0_box_5632_6656_1024_1024_0.500000/render-parameters?excludeMask=true&normalizeForMatching=true&filter=true&fillWithNoise=true" + }, + "matches" : [ ], + "stats" : { + "pFeatureCount" : 996, + "pFeatureDerivationMilliseconds" : 1415, + "qFeatureCount" : 1133, + "qFeatureDerivationMilliseconds" : 1279, + "consensusSetSizes" : [ 0 ], + "matchDerivationMilliseconds" : 324 + } + }"""; final MatchTrial matchTrial = MatchTrial.fromJson(json); diff --git a/render-ws/src/test/java/org/janelia/render/service/dao/RenderDaoTest.java b/render-ws/src/test/java/org/janelia/render/service/dao/RenderDaoTest.java index bc9311531..004ef1fa7 100644 --- a/render-ws/src/test/java/org/janelia/render/service/dao/RenderDaoTest.java +++ b/render-ws/src/test/java/org/janelia/render/service/dao/RenderDaoTest.java @@ -44,7 +44,7 @@ public class RenderDaoTest { private static RenderDao dao; @BeforeClass - public static void before() throws Exception { + public static void before() { stackId = new StackId("flyTEM", "test", "elastic"); embeddedMongoDb = new EmbeddedMongoDb(RenderDao.RENDER_DB_NAME); dao = new RenderDao(embeddedMongoDb.getMongoClient()); @@ -177,7 +177,7 @@ public void testCloneStack() { zValues = dao.getZValues(filteredStackId); Assert.assertEquals("invalid number of z values after clone filter", 1, zValues.size()); - Assert.assertEquals("invalid z value after clone filter", newZValue, zValues.get(0)); + Assert.assertEquals("invalid z value after clone filter", newZValue, zValues.getFirst()); } @Test @@ -210,17 +210,17 @@ public void testSaveStackMetaDataAndDeriveStats() { final Bounds expectedBounds = new Bounds(1094.0, 1769.0, 3903.0, 9917.0, 8301.0, 3903.1); Assert.assertEquals("invalid bounds", expectedBounds.toJson(), stats.getStackBounds().toJson()); - Assert.assertEquals("invalid tile count", new Long(14), stats.getTileCount()); + Assert.assertEquals("invalid tile count", Long.valueOf(14), stats.getTileCount()); // test getSectionData after section collection was created by call to ensureIndexesAndDeriveStats final List list = dao.getSectionData(stackId, null, null); Assert.assertNotNull("null list retrieved", list); Assert.assertEquals("invalid number of sections found", 3, list.size()); - final SectionData sectionData = list.get(0); + final SectionData sectionData = list.getFirst(); Assert.assertEquals("invalid sectionId for first section", "3903.0", sectionData.getSectionId()); Assert.assertEquals("invalid z for section 3903.0", 3903, sectionData.getZ(), 0.01); - Assert.assertEquals("invalid tileCount for section 3903.0", new Long(2), sectionData.getTileCount()); + Assert.assertEquals("invalid tileCount for section 3903.0", Long.valueOf(2), sectionData.getTileCount()); final List filteredList = dao.getSectionData(stackId, 3902.0, 3903.0); @@ -237,8 +237,7 @@ public void testRemoveStack() { final List zValuesBeforeRemove = dao.getZValues(stackId); Assert.assertNotNull("zValues null for " + stackId + " before removal", zValuesBeforeRemove); - Assert.assertTrue("zValues missing for " + stackId + " before removal", - zValuesBeforeRemove.size() > 0); + Assert.assertFalse("zValues missing for " + stackId + " before removal", zValuesBeforeRemove.isEmpty()); dao.removeStack(stackId, true); final StackMetaData stackMetaAfterRemove = dao.getStackMetaData(stackId); diff --git a/render-ws/src/test/java/org/janelia/test/EmbeddedMongoDb.java b/render-ws/src/test/java/org/janelia/test/EmbeddedMongoDb.java index bf3615de4..e50452cb1 100644 --- a/render-ws/src/test/java/org/janelia/test/EmbeddedMongoDb.java +++ b/render-ws/src/test/java/org/janelia/test/EmbeddedMongoDb.java @@ -11,24 +11,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.flapdoodle.embed.mongo.Command; -import de.flapdoodle.embed.mongo.MongoImportExecutable; -import de.flapdoodle.embed.mongo.MongoImportStarter; -import de.flapdoodle.embed.mongo.MongodExecutable; -import de.flapdoodle.embed.mongo.MongodProcess; -import de.flapdoodle.embed.mongo.MongodStarter; -import de.flapdoodle.embed.mongo.config.Defaults; -import de.flapdoodle.embed.mongo.config.ImmutableMongoCmdOptions; -import de.flapdoodle.embed.mongo.config.MongoCmdOptions; -import de.flapdoodle.embed.mongo.config.MongoImportConfig; -import de.flapdoodle.embed.mongo.config.MongodConfig; -import de.flapdoodle.embed.mongo.config.Net; +import de.flapdoodle.embed.mongo.commands.MongoImportArguments; +import de.flapdoodle.embed.mongo.commands.MongodArguments; +import de.flapdoodle.embed.mongo.commands.ServerAddress; import de.flapdoodle.embed.mongo.distribution.IFeatureAwareVersion; import de.flapdoodle.embed.mongo.distribution.Version; -import de.flapdoodle.embed.process.config.ImmutableRuntimeConfig; -import de.flapdoodle.embed.process.config.RuntimeConfig; -import de.flapdoodle.embed.process.config.io.ProcessOutput; -import de.flapdoodle.embed.process.runtime.Network; +import de.flapdoodle.embed.mongo.transitions.ExecutedMongoImportProcess; +import de.flapdoodle.embed.mongo.transitions.MongoImport; +import de.flapdoodle.embed.mongo.transitions.Mongod; +import de.flapdoodle.embed.mongo.transitions.RunningMongodProcess; +import de.flapdoodle.embed.process.io.ProcessOutput; +import de.flapdoodle.reverse.StateID; +import de.flapdoodle.reverse.Transition; +import de.flapdoodle.reverse.TransitionWalker; +import de.flapdoodle.reverse.Transitions; +import de.flapdoodle.reverse.transitions.Start; /** * Manages an embedded mongo database for use in testing. @@ -39,34 +36,25 @@ public class EmbeddedMongoDb { private final IFeatureAwareVersion version; - private final int port; - private final MongodExecutable mongodExecutable; - private final MongodProcess mongodProcess; + private final TransitionWalker.ReachedState runningMongod; + private final ServerAddress serverAddress; private final MongoClient mongoClient; private final MongoDatabase db; - public EmbeddedMongoDb(final String dbName) - throws IOException { + public EmbeddedMongoDb(final String dbName) { this.version = Version.Main.V4_0; - this.port = Network.freeServerPort(Network.getLocalHost()); - // use ephemeralForTest storage engine to fix super slow run times on Mac - // see https://github.com/flapdoodle-oss/de.flapdoodle.embed.mongo/issues/166 - final ImmutableMongoCmdOptions mongoCmdOptions = - MongoCmdOptions.builder().storageEngine("ephemeralForTest").build(); + // flapdoodle picks a free port for us, so ask the running process where it ended up + this.runningMongod = SILENT_MONGOD.start(version); - final MongodConfig mongodConfig = MongodConfig.builder() - .version(version) - .net(new Net(port, Network.localhostIsIPv6())) - .cmdOptions(mongoCmdOptions) - .build(); - - this.mongodExecutable = STARTER.prepare(mongodConfig); - - this.mongodProcess = mongodExecutable.start(); + // mongod is started without --bind_ip, so it only listens on the loopback interface. + // Reuse only the port here because the host flapdoodle reports comes from + // InetAddress.getLocalHost(), which can be this machine's external name. + final int port = runningMongod.current().getServerAddress().getPort(); + this.serverAddress = ServerAddress.of("localhost", port); - final ConnectionString connectionString = new ConnectionString("mongodb://localhost:" + port); + final ConnectionString connectionString = new ConnectionString("mongodb://" + serverAddress); this.mongoClient = MongoClients.create(connectionString); this.db = mongoClient.getDatabase(dbName); @@ -82,20 +70,15 @@ public void importCollection(final String collectionName, final Boolean upsert, final Boolean drop) throws IOException { - final MongoImportConfig mongoImportConfig = MongoImportConfig.builder() - .version(version) - .net(new Net(port, Network.localhostIsIPv6())) + final MongoImportArguments importArguments = MongoImportArguments.builder() .databaseName(db.getName()) .collectionName(collectionName) - .isUpsertDocuments(upsert) - .isDropCollection(drop) - .isJsonArray(jsonArray) .importFile(jsonFile.getAbsolutePath()) + .isJsonArray(jsonArray) + .upsertDocuments(upsert) + .dropCollection(drop) .build(); - final MongoImportExecutable mongoImportExecutable = - MongoImportStarter.getInstance(MONGO_IMPORT_RUNTIME_CONFIG).prepare(mongoImportConfig); - // Occasionally during GitHub Action builds, imports will fail with // java.io.IOException: error=26, Text file busy // @@ -110,11 +93,9 @@ public void importCollection(final String collectionName, for (int i = 0; i < maxRetries; i++) { try { // to see flapdoodle files on Mac: ls -al /var/folders/*/*/*/*mongo* - // uncomment following line if flapdoodle raises "could not run process" error after debugger exit - // mongoImportExecutable.stop(); - mongoImportExecutable.start(); + runImport(importArguments); i = maxRetries; // break out of retry loop upon success - } catch (final IOException e) { + } catch (final RuntimeException e) { final int numberOfAttempts = i + 1; if (numberOfAttempts < maxRetries) { LOG.warn("importCollection: sleeping {}ms before next retry after catching exception {}", @@ -129,12 +110,34 @@ public void importCollection(final String collectionName, } else { LOG.warn("importCollection: failed {} times to import {}, giving up and re-raising exception", numberOfAttempts, jsonFile); - throw e; + throw new IOException("failed to import " + jsonFile + " into " + collectionName, e); } } } } + /** + * Runs mongoimport against the running mongod, raising a {@link RuntimeException} if it fails. + * The import process is started and stopped within this method so that its executable is released + * before the next retry (see the retry comments in {@link #importCollection}). + */ + private void runImport(final MongoImportArguments importArguments) { + + final Transitions importTransitions = SILENT_MONGO_IMPORT + .transitions(version) + .replace(Start.to(MongoImportArguments.class).initializedWith(importArguments)) + .addAll(Start.to(ServerAddress.class).initializedWith(serverAddress)); + + try (final TransitionWalker.ReachedState executed = + importTransitions.walker().initState(StateID.of(ExecutedMongoImportProcess.class))) { + + final int returnCode = executed.current().returnCode(); + if (returnCode != 0) { + throw new IllegalStateException("mongoimport exited with return code " + returnCode); + } + } + } + public void stop() { try { @@ -144,31 +147,49 @@ public void stop() { } try { - mongodProcess.stop(); + mongoClient.close(); } catch (final Throwable t) { - LOG.warn("failed to stop mongod process", t); + LOG.warn("failed to close mongo client", t); } try { - mongodExecutable.stop(); + runningMongod.close(); } catch (final Throwable t) { - LOG.warn("failed to stop mongod executable", t); + LOG.warn("failed to stop mongod process", t); } } private static final Logger LOG = LoggerFactory.getLogger(EmbeddedMongoDb.class); - private static final RuntimeConfig MONGO_IMPORT_RUNTIME_CONFIG = ImmutableRuntimeConfig.builder() - .processOutput(ProcessOutput.silent()) - .artifactStore(Defaults.extractedArtifactStoreFor(Command.MongoImport)) - .isDaemonProcess(false) // make sure import processes are not daemons to avoid shutdown issues (see https://github.com/flapdoodle-oss/de.flapdoodle.embed.mongo/issues/191 ) - .build(); - - private static final RuntimeConfig MONGOD_RUNTIME_CONFIG = ImmutableRuntimeConfig.builder() - .processOutput(ProcessOutput.silent()) - .artifactStore(Defaults.extractedArtifactStoreFor(Command.MongoD)) - .build(); - - private static final MongodStarter STARTER = MongodStarter.getInstance(MONGOD_RUNTIME_CONFIG); + /** + * Mongod configured to keep its process output off the console + * (the flapdoodle 4.x replacement for the 3.x silent RuntimeConfig). + */ + private static final Mongod SILENT_MONGOD = new Mongod() { + @Override + public Transition mongodArguments() { + // use ephemeralForTest storage engine to fix super slow run times on Mac + // see https://github.com/flapdoodle-oss/de.flapdoodle.embed.mongo/issues/166 + return Start.to(MongodArguments.class) + .initializedWith(MongodArguments.defaults().withStorageEngine("ephemeralForTest")); + } + @Override + public Transition processOutput() { + return Start.to(ProcessOutput.class).initializedWith(ProcessOutput.silent()); + } + }; + + /** + * MongoImport configured to keep its process output off the console. + * Import processes are not daemons (flapdoodle's default ProcessConfig sets daemonProcess to false), + * which avoids the shutdown issues described in + * flapdoodle issue 191 . + */ + private static final MongoImport SILENT_MONGO_IMPORT = new MongoImport() { + @Override + public Transition processOutput() { + return Start.to(ProcessOutput.class).initializedWith(ProcessOutput.silent()); + } + }; } diff --git a/trakem2-scripts/pom.xml b/trakem2-scripts/pom.xml index a43e02f89..c5668c0a4 100644 --- a/trakem2-scripts/pom.xml +++ b/trakem2-scripts/pom.xml @@ -5,7 +5,7 @@ modules-root org.janelia.render - 4.3.0-SNAPSHOT + 5.0.0-SNAPSHOT TrakEM2 Scripts @@ -256,11 +256,11 @@ ${n5-version} - + com.fasterxml.jackson.core jackson-databind - ${jackson-version} diff --git a/trakem2-scripts/src/main/java/org/janelia/render/trakem2/ExportToRenderUsingBasisStack_Plugin.java b/trakem2-scripts/src/main/java/org/janelia/render/trakem2/ExportToRenderUsingBasisStack_Plugin.java index 78a543168..8bfcfae8b 100644 --- a/trakem2-scripts/src/main/java/org/janelia/render/trakem2/ExportToRenderUsingBasisStack_Plugin.java +++ b/trakem2-scripts/src/main/java/org/janelia/render/trakem2/ExportToRenderUsingBasisStack_Plugin.java @@ -36,9 +36,9 @@ /** * This plug-in exports TrakEM patch data into a render web services stack using a basis stack * to identify shared lens correction transformations. - * + * - * WARNING: this is a hack! - * + * - * Make sure transformation logic in {@link #exportPatches()} matches your use case before using. * * @author Eric Trautman @@ -85,9 +85,9 @@ private void exportPatches() true, true); - if (displayableList.size() > 0) { + if (! displayableList.isEmpty()) { - final Patch firstPatch = (Patch) displayableList.get(0); + final Patch firstPatch = (Patch) displayableList.getFirst(); final Matcher m = TILE_SECTION_ID_PATTERN.matcher(firstPatch.getTitle()); if (! m.matches()) { throw new IllegalArgumentException("cannot parse sectionId from patch title (tileId): " + @@ -142,8 +142,7 @@ private void exportPatches() final CoordinateTransform stageTransform = flattenedTransformList.get(stageTransformIndex); final CoordinateTransform alignmentTransform = flattenedTransformList.get(alignmentTransformIndex); - if (alignmentTransform instanceof AffineModel2D) { - final AffineModel2D alignmentModel = (AffineModel2D) alignmentTransform; + if (alignmentTransform instanceof final AffineModel2D alignmentModel) { if (stageTransform instanceof TranslationModel2D) { alignmentModel.concatenate((TranslationModel2D) stageTransform); } else if (stageTransform instanceof AffineModel2D) { @@ -221,7 +220,7 @@ private String getTransformLogInfo(final List flattenedTran private static ExportData exportData = null; - private class ExportData { + private static class ExportData { private String baseDataUrl; private String basisRenderOwner; @@ -311,7 +310,8 @@ boolean collectInputsAndLoadRenderData() { try { for (final String zPair : trakZToTargetZMapString.split(",")) { final String[] zPairArray = zPair.split("="); - trakZToTargetZMap.put(new Double(zPairArray[0]), new Double(zPairArray[1])); + trakZToTargetZMap.put(Double.parseDouble(zPairArray[0]), + Double.parseDouble(zPairArray[1])); } } catch (final Throwable t) { throw new IllegalArgumentException( @@ -349,7 +349,7 @@ boolean collectInputsAndLoadRenderData() { } boolean hasMappedZValues() { - return trakZToTargetZMapString.length() > 0; + return !trakZToTargetZMapString.isEmpty(); } Double getRenderStackZ(final Double forTrakZ) { diff --git a/trakem2-scripts/src/main/java/org/janelia/render/trakem2/ImportFromRender_Plugin.java b/trakem2-scripts/src/main/java/org/janelia/render/trakem2/ImportFromRender_Plugin.java index d0933ccf1..73fc68e07 100644 --- a/trakem2-scripts/src/main/java/org/janelia/render/trakem2/ImportFromRender_Plugin.java +++ b/trakem2-scripts/src/main/java/org/janelia/render/trakem2/ImportFromRender_Plugin.java @@ -69,7 +69,7 @@ public void run(final String arg) { final List projectList = Project.getProjects(); final boolean isNewProject = PluginArgument.NEW_PROJECT.equals(pluginArgument) || - (projectList.size() == 0); + (projectList.isEmpty()); final boolean isImportLayer = PluginArgument.IMPORT_LAYER.equals(pluginArgument); @@ -102,7 +102,7 @@ public void run(final String arg) { final LayerSet trakLayerSet; final Display front = Display.getFront(); if (front == null) { - trakProject = projectList.get(0); + trakProject = projectList.getFirst(); trakLayerSet = trakProject.getRootLayerSet(); } else { trakLayerSet = front.getLayerSet(); @@ -170,7 +170,7 @@ private void convertTileSpecsToPatches() { try { final List allChannelSpecs = importData.channels.isEmpty() ? tileSpec.getAllChannels() : tileSpec.getChannels(importData.channels); - final ChannelSpec firstChannelSpec = allChannelSpecs.get(0); + final ChannelSpec firstChannelSpec = allChannelSpecs.getFirst(); final ImageAndMask imageAndMask = firstChannelSpec.getFirstMipmapEntry().getValue(); final String imageFilePath = imageAndMask.getImageFilePath(); @@ -255,7 +255,7 @@ private void convertTileSpecsToPatches() { Utils.log("\nWARNING: Check console window for details about tile conversion failures!\n"); } - if (skippedMasksCsvData.length() > 0) { + if (! skippedMasksCsvData.isEmpty()) { final String storageFolder = importData.trakProject.getLoader().getStorageFolder(); final String csvFileName = "skipped_masks_" + new Date().getTime() + ".csv"; final Path csvPath = Paths.get(storageFolder, csvFileName); @@ -293,7 +293,7 @@ public static PluginArgument checkedValueOf(final String arg, } - private class ImportData { + private static class ImportData { private Project trakProject; private LayerSet trakLayerSet; @@ -302,7 +302,7 @@ private class ImportData { private String renderOwner; private String renderProject; private String renderStack; - private Set channels; + private final Set channels; private double minZ; private double maxZ; private boolean splitSections; @@ -325,7 +325,7 @@ private class ImportData { renderOwner = "flyTEM"; renderProject = "FAFB00"; renderStack = "v12_acquire_merged"; - channels = new HashSet(); + channels = new HashSet<>(); minZ = 1.0; maxZ = minZ; imagePlusType = ImagePlus.GRAY8; @@ -376,13 +376,13 @@ boolean setParametersFromDialog() { renderOwner = dialog.getNextString(); renderProject = dialog.getNextString(); renderStack = dialog.getNextString(); - String channelString = dialog.getNextString(); - if (channelString != null && channelString.length() > 0) { + final String channelString = dialog.getNextString(); + if (channelString != null && (! channelString.isEmpty())) { channels.add(channelString); } minZ = dialog.getNextNumber(); maxZ = dialog.getNextNumber(); - imagePlusType = new Double(dialog.getNextNumber()).intValue(); + imagePlusType = Double.valueOf(dialog.getNextNumber()).intValue(); loadMasks = dialog.getNextBoolean(); splitSections = dialog.getNextBoolean(); replaceLastWithStage = dialog.getNextBoolean(); @@ -420,7 +420,7 @@ boolean loadRenderData(final boolean isNewProject) { zValues.add(sectionData.getZ()); } - if (zValues.size() == 0) { + if (zValues.isEmpty()) { throw new IllegalArgumentException( "The " + stackId + " does not contain any layers with z values between " + minZ + " and " + maxZ); @@ -433,7 +433,7 @@ boolean loadRenderData(final boolean isNewProject) { if (isNewProject) { - final TileSpec firstTileSpec = layerRenderParametersList.get(0).getTileSpecs().get(0); + final TileSpec firstTileSpec = layerRenderParametersList.getFirst().getTileSpecs().getFirst(); @SuppressWarnings("WrapperTypeMayBePrimitive") final Double meshCellSize = firstTileSpec.getMeshCellSize(); trakProject.setProperty("mesh_resolution", String.valueOf(meshCellSize.intValue()));