From 596e1fdf9ae6de7eac16f21a37484adadb3f9c15 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 6 Nov 2017 20:13:02 -0500 Subject: [PATCH] Refactor render client parameters to use ParameterDelegate references so that common parameter sets can be easily shared by different clients. Add parameter parsing unit tests for each client. --- .idea/misc.xml | 3 +- .../match/RenderableCanvasIdPairs.java | 30 +++ .../org/janelia/alignment/util}/FileUtil.java | 30 ++- .../alignment/MipmapGeneratorTest.java | 26 +-- .../mipmap/BoxMipmapGeneratorTest.java | 28 +-- .../client/AcquisitionDataImportClient.java | 43 ++-- .../client/LowLatencyMontageClient.java | 123 ++++++---- .../client/MontageAcquisitionClient.java | 131 +++++++---- .../acquire/client/MontageParameters.java | 2 +- .../janelia/acquire/client/SolverOptions.java | 69 ------ .../org/janelia/render/client/BoxClient.java | 23 +- .../janelia/render/client/BoxGenerator.java | 211 ++++-------------- .../render/client/BoxRemovalClient.java | 33 ++- .../render/client/CoordinateClient.java | 92 +++++--- .../render/client/CopyStackClient.java | 123 +++++----- .../client/FixAutoLoaderScaleClient.java | 40 ++-- .../render/client/ImportJsonClient.java | 43 ++-- .../render/client/ImportMETClient.java | 52 +++-- .../render/client/ImportMatchClient.java | 8 +- .../client/ImportTileZValuesClient.java | 29 ++- .../client/ImportTransformChangesClient.java | 35 +-- .../janelia/render/client/MipmapClient.java | 127 ++--------- .../render/client/PointMatchClient.java | 20 +- .../client/PointMatchOptimizerClient.java | 92 ++------ .../janelia/render/client/RenderClient.java | 143 +++++++----- .../render/client/RenderClientParameters.java | 131 ----------- .../render/client/RenderSectionClient.java | 147 +++++++----- .../render/client/SectionUpdateClient.java | 21 +- .../janelia/render/client/StackClient.java | 191 ++++++++++------ .../client/StorePotentialTilePairsClient.java | 41 +--- .../janelia/render/client/TilePairClient.java | 140 ++++-------- .../render/client/TileRemovalClient.java | 34 +-- .../render/client/TransformSectionClient.java | 70 ++++-- .../render/client/ValidateTilesClient.java | 36 +-- .../render/client/WarpTransformClient.java | 72 +++--- .../CommandLineParameters.java | 31 ++- .../parameter/LayerBoundsParameters.java | 59 +++++ .../MatchClipParameters.java | 11 +- .../MatchDerivationParameters.java | 43 ++-- .../MatchRenderParameters.java | 22 +- .../MatchWebServiceParameters.java} | 21 +- .../parameter/MaterializedBoxParameters.java | 131 +++++++++++ .../client/parameter/MipmapParameters.java | 90 ++++++++ .../RenderWebServiceParameters.java} | 25 +-- .../TileSpecValidatorParameters.java} | 16 +- .../client/parameter/WarpStackParameters.java | 97 ++++++++ .../client/parameter/ZRangeParameters.java | 27 +++ .../client/LowLatencyMontageClientTest.java | 6 + .../janelia/render/client/BoxClientTest.java | 18 ++ .../render/client/BoxRemovalClientTest.java | 18 ++ .../render/client/CoordinateClientTest.java | 11 +- .../render/client/CopyStackClientTest.java | 18 ++ .../client/FixAutoLoaderScaleClientTest.java | 18 ++ .../render/client/ImportJsonClientTest.java | 18 ++ .../render/client/ImportMETClientTest.java | 18 ++ .../render/client/ImportMatchClientTest.java | 18 ++ .../client/ImportTileZValuesClientTest.java | 18 ++ .../ImportTransformChangesClientTest.java | 18 ++ .../render/client/MipmapClientTest.java | 53 ++--- .../render/client/PointMatchClientTest.java | 9 +- .../client/PointMatchOptimizerClientTest.java | 18 ++ .../render/client/RenderClientTest.java | 18 ++ .../client/RenderSectionClientTest.java | 18 ++ ...ebServiceParametersWithValidatorTest.java} | 7 +- .../client/SectionUpdateClientTest.java | 18 ++ .../render/client/StackClientTest.java | 18 ++ .../StorePotentialTilePairsClientTest.java | 18 ++ .../render/client/TilePairClientTest.java | 8 +- .../render/client/TileRemovalClientTest.java | 18 ++ .../client/TransformSectionClientTest.java | 18 ++ .../client/ValidateTilesClientTest.java | 18 ++ .../render/client/spark/AddMaskUrlClient.java | 131 +++++------ .../render/client/spark/BoxClient.java | 73 +++--- .../render/client/spark/BoxRemovalClient.java | 68 +++--- .../render/client/spark/CopyStackClient.java | 65 +++--- .../client/spark/DMeshPointMatchClient.java | 14 +- .../client/spark/FixMipmapUrlClient.java | 191 ++++++++-------- .../render/client/spark/MipmapClient.java | 70 +++--- .../RenderableCanvasIdPairsUtilities.java | 27 --- .../client/spark/SIFTPointMatchClient.java | 19 +- .../render/client/spark/ScapeClient.java | 177 +++++++-------- .../client/spark/TransformSectionClient.java | 89 +++----- .../client/spark/WarpTransformClient.java | 135 +++-------- .../client/spark/AddMaskUrlClientTest.java | 18 ++ .../render/client/spark/BoxClientTest.java | 18 ++ .../client/spark/BoxRemovalClientTest.java | 18 ++ .../client/spark/CopyStackClientTest.java | 18 ++ .../spark/DMeshPointMatchClientTest.java | 18 ++ .../client/spark/FixMipmapUrlClientTest.java | 18 ++ .../render/client/spark/MipmapClientTest.java | 18 ++ .../RenderableCanvasIdPairsUtilitiesTest.java | 2 +- .../render/client/spark/ScapeClientTest.java | 18 ++ .../spark/TransformSectionClientTest.java | 18 ++ .../client/spark/WarpTransformClientTest.java | 18 ++ 94 files changed, 2595 insertions(+), 2076 deletions(-) rename {render-ws-java-client/src/main/java/org/janelia/render/client => render-app/src/main/java/org/janelia/alignment/util}/FileUtil.java (81%) delete mode 100644 render-ws-java-client/src/main/java/org/janelia/acquire/client/SolverOptions.java delete mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/RenderClientParameters.java rename render-ws-java-client/src/main/java/org/janelia/render/client/{ => parameter}/CommandLineParameters.java (65%) create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/parameter/LayerBoundsParameters.java rename render-ws-java-client/src/main/java/org/janelia/render/client/{parameters => parameter}/MatchClipParameters.java (71%) rename render-ws-java-client/src/main/java/org/janelia/render/client/{parameters => parameter}/MatchDerivationParameters.java (76%) rename render-ws-java-client/src/main/java/org/janelia/render/client/{parameters => parameter}/MatchRenderParameters.java (75%) rename render-ws-java-client/src/main/java/org/janelia/render/client/{parameters/MatchDataClientParameters.java => parameter/MatchWebServiceParameters.java} (56%) create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MaterializedBoxParameters.java create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MipmapParameters.java rename render-ws-java-client/src/main/java/org/janelia/render/client/{RenderDataClientParameters.java => parameter/RenderWebServiceParameters.java} (50%) rename render-ws-java-client/src/main/java/org/janelia/render/client/{RenderDataClientParametersWithValidator.java => parameter/TileSpecValidatorParameters.java} (84%) create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/parameter/WarpStackParameters.java create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/parameter/ZRangeParameters.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/BoxClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/BoxRemovalClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/CopyStackClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/FixAutoLoaderScaleClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/ImportJsonClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/ImportMETClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/ImportMatchClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/ImportTileZValuesClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/ImportTransformChangesClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/PointMatchOptimizerClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/RenderClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/RenderSectionClientTest.java rename render-ws-java-client/src/test/java/org/janelia/render/client/{RenderDataClientParametersWithValidatorTest.java => RenderWebServiceParametersWithValidatorTest.java} (83%) create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/SectionUpdateClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/StackClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/StorePotentialTilePairsClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/TileRemovalClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/TransformSectionClientTest.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/ValidateTilesClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/AddMaskUrlClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/BoxClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/BoxRemovalClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/CopyStackClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/DMeshPointMatchClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/FixMipmapUrlClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/MipmapClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/ScapeClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/TransformSectionClientTest.java create mode 100644 render-ws-spark-client/src/test/java/org/janelia/render/client/spark/WarpTransformClientTest.java diff --git a/.idea/misc.xml b/.idea/misc.xml index 5d3d392c0..d28c11d5c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -2,8 +2,9 @@ - + + diff --git a/render-app/src/main/java/org/janelia/alignment/match/RenderableCanvasIdPairs.java b/render-app/src/main/java/org/janelia/alignment/match/RenderableCanvasIdPairs.java index f2982abf1..9b0e76521 100644 --- a/render-app/src/main/java/org/janelia/alignment/match/RenderableCanvasIdPairs.java +++ b/render-app/src/main/java/org/janelia/alignment/match/RenderableCanvasIdPairs.java @@ -1,10 +1,15 @@ package org.janelia.alignment.match; +import java.io.IOException; import java.io.Reader; import java.io.Serializable; +import java.nio.file.FileSystems; +import java.nio.file.Path; import java.util.List; import org.janelia.alignment.json.JsonUtils; +import org.janelia.alignment.util.FileUtil; +import org.slf4j.LoggerFactory; /** * List of {@link OrderedCanvasIdPair} objects with a common render parameters URL template. @@ -61,6 +66,29 @@ public List getNeighborPairs() { return neighborPairs; } + /** + * @return pairs object loaded from the specified file. + */ + public static RenderableCanvasIdPairs load(final String dataFile) + throws IOException, IllegalArgumentException { + + final RenderableCanvasIdPairs renderableCanvasIdPairs; + + final Path path = FileSystems.getDefault().getPath(dataFile).toAbsolutePath(); + + LOG.info("load: entry, path={}", path); + + try (final Reader reader = FileUtil.DEFAULT_INSTANCE.getExtensionBasedReader(path.toString())) { + renderableCanvasIdPairs = RenderableCanvasIdPairs.fromJson(reader); + } + + LOG.info("load: exit, loaded {} pairs", renderableCanvasIdPairs.size()); + + + return renderableCanvasIdPairs; + } + + public static RenderableCanvasIdPairs fromJson(final Reader json) { return JSON_HELPER.fromJson(json); } @@ -68,4 +96,6 @@ public static RenderableCanvasIdPairs fromJson(final Reader json) { private static final JsonUtils.Helper JSON_HELPER = new JsonUtils.Helper<>(RenderableCanvasIdPairs.class); + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(RenderableCanvasIdPairs.class); + } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/FileUtil.java b/render-app/src/main/java/org/janelia/alignment/util/FileUtil.java similarity index 81% rename from render-ws-java-client/src/main/java/org/janelia/render/client/FileUtil.java rename to render-app/src/main/java/org/janelia/alignment/util/FileUtil.java index 756418ab4..ada30d62b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/FileUtil.java +++ b/render-app/src/main/java/org/janelia/alignment/util/FileUtil.java @@ -1,4 +1,4 @@ -package org.janelia.render.client; +package org.janelia.alignment.util; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -24,7 +24,7 @@ import org.slf4j.LoggerFactory; /** - * Utility for reading and writing files. + * Shared file management utilities. * * @author Eric Trautman */ @@ -110,7 +110,33 @@ public static void ensureWritableDirectory(final File directory) { } } + public static boolean deleteRecursive(final File file) { + + // could use Apache Commons FileUtils.delete but to have the logging, need to implement here + + boolean deleteSuccessful = true; + + if (file.isDirectory()){ + final File[] files = file.listFiles(); + if (files != null) { + for (final File f : files) { + deleteSuccessful = deleteSuccessful && deleteRecursive(f); + } + } + } + + if (file.delete()) { + LOG.info("deleted " + file.getAbsolutePath()); + } else { + LOG.warn("failed to delete " + file.getAbsolutePath()); + deleteSuccessful = false; + } + + return deleteSuccessful; + } + private static final Logger LOG = LoggerFactory.getLogger(FileUtil.class); private static final int DEFAULT_BUFFER_SIZE = 65536; + } diff --git a/render-app/src/test/java/org/janelia/alignment/MipmapGeneratorTest.java b/render-app/src/test/java/org/janelia/alignment/MipmapGeneratorTest.java index 6299bec83..95e719158 100644 --- a/render-app/src/test/java/org/janelia/alignment/MipmapGeneratorTest.java +++ b/render-app/src/test/java/org/janelia/alignment/MipmapGeneratorTest.java @@ -4,6 +4,7 @@ import org.janelia.alignment.spec.ChannelSpec; import org.janelia.alignment.spec.TileSpec; +import org.janelia.alignment.util.FileUtil; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -38,7 +39,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - deleteRecursive(baseMipmapDirectory); + FileUtil.deleteRecursive(baseMipmapDirectory); } @Test @@ -125,29 +126,6 @@ private void validateMask(final ChannelSpec channelSpec, } } - private static boolean deleteRecursive(final File file) { - - boolean deleteSuccessful = true; - - if (file.isDirectory()){ - final File[] files = file.listFiles(); - if (files != null) { - for (final File f : files) { - deleteSuccessful = deleteSuccessful && deleteRecursive(f); - } - } - } - - if (file.delete()) { - LOG.info("deleted " + file.getAbsolutePath()); - } else { - LOG.warn("failed to delete " + file.getAbsolutePath()); - deleteSuccessful = false; - } - - return deleteSuccessful; - } - private static final Logger LOG = LoggerFactory.getLogger(MipmapGeneratorTest.class); } diff --git a/render-app/src/test/java/org/janelia/alignment/mipmap/BoxMipmapGeneratorTest.java b/render-app/src/test/java/org/janelia/alignment/mipmap/BoxMipmapGeneratorTest.java index 28943e155..83d9dae55 100644 --- a/render-app/src/test/java/org/janelia/alignment/mipmap/BoxMipmapGeneratorTest.java +++ b/render-app/src/test/java/org/janelia/alignment/mipmap/BoxMipmapGeneratorTest.java @@ -28,6 +28,7 @@ import org.janelia.alignment.ArgbRenderer; import org.janelia.alignment.RenderParameters; +import org.janelia.alignment.util.FileUtil; import org.janelia.alignment.Utils; import org.janelia.alignment.spec.Bounds; import org.janelia.alignment.util.LabelImageProcessorCache; @@ -85,7 +86,8 @@ public void setup() throws Exception { @After public void tearDown() throws Exception { - deleteRecursive(testDirectory); + + FileUtil.deleteRecursive(testDirectory); } @Test @@ -219,29 +221,5 @@ private BoxMipmapGenerator validateNextLevel(final BoxMipmapGenerator boxMipmapG return nextLevelGenerator; } - // TODO: replace with Apache Commons FileUtils.delete - public static boolean deleteRecursive(final File file) { - - boolean deleteSuccessful = true; - - if (file.isDirectory()){ - final File[] files = file.listFiles(); - if (files != null) { - for (final File f : files) { - deleteSuccessful = deleteSuccessful && deleteRecursive(f); - } - } - } - - if (file.delete()) { - LOG.info("deleted " + file.getAbsolutePath()); - } else { - LOG.warn("failed to delete " + file.getAbsolutePath()); - deleteSuccessful = false; - } - - return deleteSuccessful; - } - private static final Logger LOG = LoggerFactory.getLogger(BoxMipmapGeneratorTest.class); } diff --git a/render-ws-java-client/src/main/java/org/janelia/acquire/client/AcquisitionDataImportClient.java b/render-ws-java-client/src/main/java/org/janelia/acquire/client/AcquisitionDataImportClient.java index 12740c671..a38bcfb67 100644 --- a/render-ws-java-client/src/main/java/org/janelia/acquire/client/AcquisitionDataImportClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/acquire/client/AcquisitionDataImportClient.java @@ -1,7 +1,23 @@ package org.janelia.acquire.client; import com.beust.jcommander.Parameter; -import org.janelia.acquire.client.model.*; +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.janelia.acquire.client.model.Acquisition; +import org.janelia.acquire.client.model.AcquisitionTile; +import org.janelia.acquire.client.model.AcquisitionTileIdList; +import org.janelia.acquire.client.model.AcquisitionTileList; +import org.janelia.acquire.client.model.AcquisitionTileState; +import org.janelia.acquire.client.model.Calibration; import org.janelia.alignment.spec.ResolvedTileSpecCollection; import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.TransformSpec; @@ -12,13 +28,12 @@ import org.janelia.render.client.ClientRunner; import org.janelia.render.client.ImportJsonClient; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParametersWithValidator; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.*; - /** * Java client that pulls tile data from an acquisition system and stores it in the render database. * @@ -26,11 +41,13 @@ */ public class AcquisitionDataImportClient { - @SuppressWarnings("ALL") - static class Parameters extends RenderDataClientParametersWithValidator { + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - // NOTE: --baseDataUrl, --owner, --project, --validatorClass, and --validatorData parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); @Parameter(names = "--finalStackState", description = "State render stack should have after import (default is COMPLETE)", required = false) StackMetaData.StackState finalStackState; @@ -85,7 +102,7 @@ public static void main(final String[] args) { @Override public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, AcquisitionDataImportClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -121,7 +138,7 @@ public void runClient(final String[] args) throws Exception { public AcquisitionDataImportClient(final Parameters parameters) throws IOException { this.parameters = parameters; - this.tileSpecValidator = parameters.getValidatorInstance(); + this.tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); this.acquisitionDataClient = new AcquisitionDataClient(parameters.baseAcquisitionUrl); @@ -199,7 +216,7 @@ public void processAcquisition(final Acquisition acquisition) { try { String ownerName = acquisition.getProjectOwner(); if (parameters.overrideOwnerFromParameter) { - ownerName = parameters.owner; + ownerName = parameters.renderWeb.owner; } if (ownerName == null || ownerName.trim().length() == 0) { throw new IllegalArgumentException("Owner for acquisition " + acqId + " is null"); @@ -218,7 +235,7 @@ public void processAcquisition(final Acquisition acquisition) { final String acquireStackName = baseStackName + "_acquire"; - final RenderDataClient renderDataClient = new RenderDataClient(parameters.baseDataUrl, + final RenderDataClient renderDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, ownerName, projectName); diff --git a/render-ws-java-client/src/main/java/org/janelia/acquire/client/LowLatencyMontageClient.java b/render-ws-java-client/src/main/java/org/janelia/acquire/client/LowLatencyMontageClient.java index 299bc18f2..ff3ee8afe 100644 --- a/render-ws-java-client/src/main/java/org/janelia/acquire/client/LowLatencyMontageClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/acquire/client/LowLatencyMontageClient.java @@ -1,6 +1,7 @@ package org.janelia.acquire.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.File; import java.io.IOException; @@ -29,7 +30,9 @@ import org.janelia.render.client.ClientRunner; import org.janelia.render.client.ImportJsonClient; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParametersWithValidator; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,41 +44,73 @@ */ public class LowLatencyMontageClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParametersWithValidator { - - // NOTE: --baseDataUrl, --owner, --project, --validatorClass, and --validatorData parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator - - @Parameter(names = "--finalStackState", description = "State render stack should have after import (default is COMPLETE)", required = false) - private StackMetaData.StackState finalStackState; - - @Parameter(names = "--transformFile", description = "File containing shared JSON transform specs (.json, .gz, or .zip)", required = false) - private String transformFile; - - @Parameter(names = "--baseAcquisitionUrl", description = "Base URL for acquisiiton data (e.g. http://host[:port]/???/v1)", required = true) - private String baseAcquisitionUrl; - - @Parameter(names = "--acquisitionId", description = "Acquisition identifier for limiting tiles to a known acquisition", required = false) - private Long acquisitionId; - - @Parameter(names = "--acquisitionTileState", description = "Only process acquisition tiles that are in this state (default is READY)", required = false) - private AcquisitionTileState acquisitionTileState = AcquisitionTileState.READY; - - @Parameter(names = "--acquisitionTileCount", description = "Maximum number of acquisition tiles to retrieve in each request", required = false) - private Integer acquisitionTileCount = 1; - - @Parameter(names = "--waitSeconds", description = "Number of seconds to wait before checking for newly acquired tiles (default 5)", required = false) - private int waitSeconds = 5; - - @Parameter(names = "--montageScript", description = "Full path of the montage generator script (e.g. /groups/flyTEM/.../montage_section_SL)", required = false) - private String montageScript; - - @Parameter(names = "--montageParametersFile", description = "File containing JSON parameters for montage generation", required = false) - private String montageParametersFile; - - @Parameter(names = "--montageWorkDirectory", description = "Parent directory for montage input data files (e.g. /nobackup/flyTEM/montage_work)", required = false) - private String montageWorkDirectory; + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); + + @Parameter( + names = "--finalStackState", + description = "State render stack should have after import (default is COMPLETE)", + required = false) + public StackMetaData.StackState finalStackState; + + @Parameter( + names = "--transformFile", + description = "File containing shared JSON transform specs (.json, .gz, or .zip)", + required = false) + public String transformFile; + + @Parameter( + names = "--baseAcquisitionUrl", + description = "Base URL for acquisiiton data (e.g. http://host[:port]/???/v1)", + required = true) + public String baseAcquisitionUrl; + + @Parameter( + names = "--acquisitionId", + description = "Acquisition identifier for limiting tiles to a known acquisition", + required = false) + public Long acquisitionId; + + @Parameter( + names = "--acquisitionTileState", + description = "Only process acquisition tiles that are in this state (default is READY)", + required = false) + public AcquisitionTileState acquisitionTileState = AcquisitionTileState.READY; + + @Parameter( + names = "--acquisitionTileCount", + description = "Maximum number of acquisition tiles to retrieve in each request", + required = false) + public Integer acquisitionTileCount = 1; + + @Parameter( + names = "--waitSeconds", + description = "Number of seconds to wait before checking for newly acquired tiles (default 5)", + required = false) + public int waitSeconds = 5; + + @Parameter( + names = "--montageScript", + description = "Full path of the montage generator script (e.g. /groups/flyTEM/.../montage_section_SL)", + required = false) + public String montageScript; + + @Parameter( + names = "--montageParametersFile", + description = "File containing JSON parameters for montage generation", + required = false) + public String montageParametersFile; + + @Parameter( + names = "--montageWorkDirectory", + description = "Parent directory for montage input data files (e.g. /nobackup/flyTEM/montage_work)", + required = false) + public String montageWorkDirectory; public void validate() throws IllegalArgumentException { @@ -99,7 +134,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, LowLatencyMontageClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -136,7 +171,7 @@ public void runClient(final String[] args) throws Exception { public LowLatencyMontageClient(final Parameters parameters) throws IOException { this.parameters = parameters; - this.tileSpecValidator = parameters.getValidatorInstance(); + this.tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); this.acquisitionDataClient = new AcquisitionDataClient(parameters.baseAcquisitionUrl); @@ -228,7 +263,7 @@ public void processAcquisition(final Acquisition acquisition) { final String acquireStackName = baseStackName + "_acquire"; final String montageStackName = baseStackName + "_montage"; - final RenderDataClient renderDataClient = new RenderDataClient(parameters.baseDataUrl, + final RenderDataClient renderDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, ownerName, projectName); @@ -314,13 +349,13 @@ public Set importAcquisitionData(final RenderDataClient renderDataClient transformSpecsCopy.addAll(transformSpecs); final ResolvedTileSpecCollection resolvedTiles = new ResolvedTileSpecCollection(transformSpecsCopy, - new ArrayList(8192)); + new ArrayList<>(8192)); resolvedTiles.setTileSpecValidator(tileSpecValidator); final AcquisitionTileIdList completedTileIds = new AcquisitionTileIdList(AcquisitionTileState.COMPLETE, - new ArrayList(8192)); + new ArrayList<>(8192)); final AcquisitionTileIdList failedTileIds = new AcquisitionTileIdList(AcquisitionTileState.FAILED, - new ArrayList()); + new ArrayList<>()); final ProcessTimer timer = new ProcessTimer(); AcquisitionTileList acquisitionTileList; @@ -443,11 +478,11 @@ public void invokeMontageProcessor(final String owner, final MontageParameters montageParameters = MontageParameters.load(parameters.montageParametersFile); montageParameters.setSourceCollection( - new AlignmentRenderCollection(parameters.baseDataUrl, + new AlignmentRenderCollection(parameters.renderWeb.baseDataUrl, owner, project, acquireStack)); montageParameters.setTargetCollection( - new AlignmentRenderCollection(parameters.baseDataUrl, + new AlignmentRenderCollection(parameters.renderWeb.baseDataUrl, owner, project, montageStack)); final File montageWorkStackDir = Paths.get(parameters.montageWorkDirectory, diff --git a/render-ws-java-client/src/main/java/org/janelia/acquire/client/MontageAcquisitionClient.java b/render-ws-java-client/src/main/java/org/janelia/acquire/client/MontageAcquisitionClient.java index 07c400c16..3e963a0dc 100644 --- a/render-ws-java-client/src/main/java/org/janelia/acquire/client/MontageAcquisitionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/acquire/client/MontageAcquisitionClient.java @@ -1,6 +1,7 @@ package org.janelia.acquire.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.File; import java.io.IOException; @@ -17,7 +18,9 @@ import org.janelia.alignment.spec.stack.StackVersion; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParametersWithValidator; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,42 +31,78 @@ */ public class MontageAcquisitionClient { - @SuppressWarnings("ALL") - static class Parameters extends RenderDataClientParametersWithValidator { - - // NOTE: --baseDataUrl, --owner, --project, --validatorClass, and --validatorData parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator - - @Parameter(names = "--stackId", description = "identifier of the stack to be processed", required = false) - private String stackId; - - @Parameter(names = "--stackFilter", description = "Regex the must be satisified by the stacks that need to be processed") - private String stackFilter = "\\d+_acquire"; - - @Parameter(names = "--aggregatedAcquireStack", description = "The name of the aggregated acquire stack", required = true) - private String aggregatedAcquireStack = null; - - @Parameter(names = "--targetOwner", description = "owner of the montaged collection", required = false) - private String targetOwner = null; - - @Parameter(names = "--targetProject", description = "project of the montaged collection", required = false) - private String targetProject = null; - - @Parameter(names = "--targetStack", description = "stack of the montaged collection", required = false) - private String targetStack = null; - - @Parameter(names = "--montageScript", description = "Full path of the montage generator script (e.g. /groups/flyTEM/.../montage_section_SL)", required = false) - private String montageScript; - - @Parameter(names = "--montageParametersFile", description = "File containing JSON parameters for montage generation", required = false) - private String montageParametersFile; - - @Parameter(names = "--montageWorkDirectory", description = "Parent directory for montage input data files (e.g. /nobackup/flyTEM/montage_work)", required = false) - private String montageWorkDirectory; - - @Parameter(names = "--waitSeconds", description = "Number of seconds to wait before checking for newly acquired tiles (default 5)", required = false) - private int waitSeconds = 5; - @Parameter(names = "--doNotAggregateSources", arity = 0, description = "If the parameter is true it leaves the source collection as is and it will not aggregate it into a bigger collection", required = false) + static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); + + @Parameter( + names = "--stackId", + description = "identifier of the stack to be processed", + required = false) + public String stackId; + + @Parameter( + names = "--stackFilter", + description = "Regex the must be satisified by the stacks that need to be processed") + public String stackFilter = "\\d+_acquire"; + + @Parameter( + names = "--aggregatedAcquireStack", + description = "The name of the aggregated acquire stack", + required = true) + public String aggregatedAcquireStack = null; + + @Parameter( + names = "--targetOwner", + description = "owner of the montaged collection", + required = false) + public String targetOwner = null; + + @Parameter( + names = "--targetProject", + description = "project of the montaged collection", + required = false) + public String targetProject = null; + + @Parameter( + names = "--targetStack", + description = "stack of the montaged collection", + required = false) + public String targetStack = null; + + @Parameter( + names = "--montageScript", + description = "Full path of the montage generator script (e.g. /groups/flyTEM/.../montage_section_SL)", + required = false) + public String montageScript; + + @Parameter( + names = "--montageParametersFile", + description = "File containing JSON parameters for montage generation", + required = false) + public String montageParametersFile; + + @Parameter( + names = "--montageWorkDirectory", + description = "Parent directory for montage input data files (e.g. /nobackup/flyTEM/montage_work)", + required = false) + public String montageWorkDirectory; + + @Parameter( + names = "--waitSeconds", + description = "Number of seconds to wait before checking for newly acquired tiles (default 5)", + required = false) + public int waitSeconds = 5; + + @Parameter( + names = "--doNotAggregateSources", + arity = 0, + description = "If the parameter is true it leaves the source collection as is and it will not aggregate it into a bigger collection", + required = false) Boolean doNotAggregateSources; Boolean aggregateSources() { @@ -88,7 +127,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, MontageAcquisitionClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); parameters.validate(); @@ -122,9 +161,7 @@ public void runClient(final String[] args) throws Exception { public MontageAcquisitionClient(final Parameters parameters) throws IOException { - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.renderDataClient = parameters.renderWeb.getDataClient(); this.parameters = parameters; this.failedStackIds = new HashSet<>(); } @@ -154,7 +191,7 @@ public void processStacks() public List retrieveStackIds(final MontageAcquisitionClient.Parameters stackFilter) throws IOException { final List stackIds = renderDataClient.getOwnerStacks(); return stackIds.stream() - .filter(s -> s.getProject().equals(stackFilter.project) && s.getStack().matches(stackFilter.stackFilter)) + .filter(s -> s.getProject().equals(stackFilter.renderWeb.project) && s.getStack().matches(stackFilter.stackFilter)) .filter(s -> stackFilter.stackId == null || s.getStack().equals(stackFilter.stackId)) .collect(Collectors.toList()); } @@ -168,7 +205,11 @@ public void processStack(final StackId stackId) throws IOException { final MontageParameters montageParameters = MontageParameters.load(parameters.montageParametersFile); montageParameters.setSourceCollection( - new AlignmentRenderCollection(parameters.baseDataUrl, parameters.owner, parameters.project, stackId.getStack())); + new AlignmentRenderCollection(parameters.renderWeb.baseDataUrl, + parameters.renderWeb.owner, + parameters.renderWeb.project, + stackId.getStack())); + if (parameters.targetOwner != null && parameters.targetOwner.trim().length() > 0) { montageParameters.getTargetCollection().setOwner(parameters.targetOwner.trim()); } @@ -188,8 +229,8 @@ public void processStack(final StackId stackId) throws IOException { } final File montageWorkStackDir = Paths.get(parameters.montageWorkDirectory, - parameters.project, - stackId.getStack()).toAbsolutePath().toFile(); + parameters.renderWeb.project, + stackId.getStack()).toAbsolutePath().toFile(); if (! montageWorkStackDir.exists()) { if (! montageWorkStackDir.mkdirs()) { diff --git a/render-ws-java-client/src/main/java/org/janelia/acquire/client/MontageParameters.java b/render-ws-java-client/src/main/java/org/janelia/acquire/client/MontageParameters.java index 5f8e078aa..b891131ef 100644 --- a/render-ws-java-client/src/main/java/org/janelia/acquire/client/MontageParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/acquire/client/MontageParameters.java @@ -8,7 +8,7 @@ import java.util.Map; import org.janelia.alignment.json.JsonUtils; -import org.janelia.render.client.FileUtil; +import org.janelia.alignment.util.FileUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/render-ws-java-client/src/main/java/org/janelia/acquire/client/SolverOptions.java b/render-ws-java-client/src/main/java/org/janelia/acquire/client/SolverOptions.java deleted file mode 100644 index b53871bc7..000000000 --- a/render-ws-java-client/src/main/java/org/janelia/acquire/client/SolverOptions.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.janelia.acquire.client; - -/** - * Solver options for alignment tools. - * - * See - * https://github.com/khaledkhairy/EM_aligner/blob/master/matlab_compiled/sample_montage_input.json - * - * - *

- * For "montage" runs: - *

    - *
  • - * These parameters will always stay unchanged: - * solver, nbrs, xs_weight, translation fac, stvec_flag, conn_comp, distributed. - *
    • - *
  • - * These parameters can be experimented with, but normally shouldn't change: - * min_tiles, outlier_lambda, min_points, small_region, small_region_lambda, calc_confidence. - *
    • - *
  • - * These parameters need to be specified for each run: - * lambda, edge_lambda. - *
    • - *
- * - *

- * - * @author Eric Trautman - */ -@SuppressWarnings({"FieldCanBeLocal", "unused"}) -public class SolverOptions { - - private final Integer min_tiles; - private final Integer degree; - private final Double outlier_lambda; - private final String solver; - private final Integer min_points; - private final Integer nbrs; - private final Double xs_weight; - private final Integer stvec_flag; - private final Integer conn_comp; - private final Integer distributed; - private final Double lambda; - private final Double edge_lambda; - private final Double small_region_lambda; - private final Integer small_region; - private final Integer calc_confidence; - private final Integer translation_fac; - - public SolverOptions() { - this.min_tiles = 3; - this.degree = 1; - this.outlier_lambda = 1000.0; - this.solver = "backslash"; - this.min_points = 5; - this.nbrs = 1; - this.xs_weight = 0.05; - this.stvec_flag = 0; - this.conn_comp = 1; - this.distributed = 0; - this.lambda = 0.01; - this.edge_lambda = 0.01; - this.small_region_lambda = 10.0; - this.small_region = 5; - this.calc_confidence = 1; - this.translation_fac = 1; - } -} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/BoxClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/BoxClient.java index 98964aa6c..047ebf191 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/BoxClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/BoxClient.java @@ -1,9 +1,13 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.util.List; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MaterializedBoxParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,13 +19,18 @@ */ public class BoxClient { - @SuppressWarnings("ALL") - private static class Parameters extends BoxGenerator.Parameters { + public static class Parameters extends CommandLineParameters { - // NOTE: almost everything is defined in BoxGenerator.Parameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter(description = "Z values for layers to render", required = true) - private List zValues; + @ParametersDelegate + public MaterializedBoxParameters box = new MaterializedBoxParameters(); + + @Parameter( + description = "Z values for layers to render", + required = true) + public List zValues; } @@ -34,11 +43,11 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, BoxClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); - final BoxGenerator boxGenerator = new BoxGenerator(parameters); + final BoxGenerator boxGenerator = new BoxGenerator(parameters.renderWeb, parameters.box); boxGenerator.createEmptyImageFile(); for (final Double z : parameters.zValues) { diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/BoxGenerator.java b/render-ws-java-client/src/main/java/org/janelia/render/client/BoxGenerator.java index 3228ce7b1..fa53faf32 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/BoxGenerator.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/BoxGenerator.java @@ -1,7 +1,5 @@ package org.janelia.render.client; -import com.beust.jcommander.Parameter; - import java.awt.Color; import java.awt.Graphics2D; import java.awt.image.BufferedImage; @@ -29,6 +27,8 @@ import org.janelia.alignment.spec.stack.StackStats; import org.janelia.alignment.util.ImageProcessorCache; import org.janelia.alignment.util.LabelImageProcessorCache; +import org.janelia.render.client.parameter.MaterializedBoxParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,133 +53,8 @@ */ public class BoxGenerator implements Serializable { - @SuppressWarnings("ALL") - public static class Parameters extends RenderDataClientParameters { - - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - - @Parameter( - names = "--stack", - description = "Stack name", - required = true) - public String stack; - - @Parameter( - names = "--rootDirectory", - description = "Root directory for rendered tiles (e.g. /tier2/flyTEM/nobackup/rendered_boxes)", - required = true) - private String rootDirectory; - - @Parameter( - names = "--width", - description = "Width of each box", - required = true) - private Integer width; - - @Parameter( - names = "--height", - description = "Height of each box", - required = true) - private Integer height; - - @Parameter( - names = "--maxLevel", - description = "Maximum mipmap level to generate", - required = false) - private Integer maxLevel = 0; - - @Parameter( - names = "--format", - description = "Format for rendered boxes", - required = false) - private String format = Utils.PNG_FORMAT; - - @Parameter( - names = "--maxOverviewWidthAndHeight", - description = "Max width and height of layer overview image (omit or set to zero to disable overview generation)", - required = false) - private Integer maxOverviewWidthAndHeight; - - @Parameter( - names = "--skipInterpolation", - description = "skip interpolation (e.g. for DMG data)", - required = false, - arity = 0) - private boolean skipInterpolation = false; - - @Parameter( - names = "--binaryMask", - description = "use binary mask (e.g. for DMG data)", - required = false, - arity = 0) - private boolean binaryMask = false; - - @Parameter( - names = "--label", - description = "Generate single color tile labels instead of actual tile images", - required = false, - arity = 0) - private boolean label = false; - - @Parameter( - names = "--createIGrid", - description = "create an IGrid file", - required = false, - arity = 0) - private boolean createIGrid = false; - - @Parameter( - names = "--forceGeneration", - description = "Regenerate boxes even if they already exist", - required = false, - arity = 0) - private boolean forceGeneration = false; - - @Parameter( - names = "--renderGroup", - description = "Index (1-n) that identifies portion of layer to render (omit if only one job is being used)", - required = false) - public Integer renderGroup; - - @Parameter( - names = "--numberOfRenderGroups", - description = "Total number of parallel jobs being used to render this layer (omit if only one job is being used)", - required = false) - public Integer numberOfRenderGroups; - - public boolean isOverviewNeeded() { - return ((maxOverviewWidthAndHeight != null) && (maxOverviewWidthAndHeight > 0)); - } - - public Parameters getInstanceForRenderGroup(final int group, - final int numberOfGroups) { - Parameters p = new Parameters(); - - p.baseDataUrl = this.baseDataUrl; - p.owner = this.owner; - p.project = this.project; - p.stack = this.stack; - p.rootDirectory = this.rootDirectory; - p.width = this.width; - p.height = this.height; - p.maxLevel = this.maxLevel; - p.format = this.format; - p.maxOverviewWidthAndHeight = this.maxOverviewWidthAndHeight; - p.skipInterpolation = this.skipInterpolation; - p.binaryMask = this.binaryMask; - p.label = this.label; - p.createIGrid = this.createIGrid; - p.forceGeneration = this.forceGeneration; - - p.renderGroup = group; - p.numberOfRenderGroups = numberOfGroups; - - return p; - } - - } - - private final Parameters parameters; + private final RenderWebServiceParameters renderWebParameters; + private final MaterializedBoxParameters boxParameters; private final String stack; private final String format; @@ -192,27 +67,29 @@ public Parameters getInstanceForRenderGroup(final int group, private transient RenderDataClient rdc; - public BoxGenerator(final Parameters parameters) + public BoxGenerator(final RenderWebServiceParameters renderWebParameters, + final MaterializedBoxParameters boxParameters) throws IOException { - this.parameters = parameters; + this.renderWebParameters = renderWebParameters; + this.boxParameters = boxParameters; - this.stack = parameters.stack; - this.format = parameters.format; - this.boxWidth = parameters.width; - this.boxHeight = parameters.height; + this.stack = boxParameters.stack; + this.format = boxParameters.format; + this.boxWidth = boxParameters.width; + this.boxHeight = boxParameters.height; String boxName = this.boxWidth + "x" + this.boxHeight; - if (parameters.label) { + if (boxParameters.label) { boxName += "-label"; this.backgroundRGBColor = Color.WHITE.getRGB(); } else { this.backgroundRGBColor = null; } - final Path boxPath = Paths.get(parameters.rootDirectory, - parameters.project, - parameters.stack, + final Path boxPath = Paths.get(boxParameters.rootDirectory, + renderWebParameters.project, + boxParameters.stack, boxName).toAbsolutePath(); this.boxDirectory = boxPath.toFile(); @@ -235,24 +112,24 @@ public BoxGenerator(final Parameters parameters) this.emptyImageFile = new File(boxDirectory.getAbsolutePath(), "empty." + format.toLowerCase()); - if (parameters.renderGroup != null) { + if (boxParameters.renderGroup != null) { - if (parameters.numberOfRenderGroups == null) { + if (boxParameters.numberOfRenderGroups == null) { throw new IllegalArgumentException( "numberOfRenderGroups must be specified when renderGroup is specified"); } - if (parameters.renderGroup < 1) { + if (boxParameters.renderGroup < 1) { throw new IllegalArgumentException("renderGroup values start at 1"); } - if (parameters.renderGroup > parameters.numberOfRenderGroups) { + if (boxParameters.renderGroup > boxParameters.numberOfRenderGroups) { throw new IllegalArgumentException( - "numberOfRenderGroups (" + parameters.numberOfRenderGroups + - ") must be greater than the renderGroup (" + parameters.renderGroup + ")"); + "numberOfRenderGroups (" + boxParameters.numberOfRenderGroups + + ") must be greater than the renderGroup (" + boxParameters.renderGroup + ")"); } - } else if (parameters.numberOfRenderGroups != null) { + } else if (boxParameters.numberOfRenderGroups != null) { throw new IllegalArgumentException( "renderGroup (1-n) must be specified when numberOfRenderGroups are specified"); } @@ -268,9 +145,9 @@ public BoxGenerator(final Parameters parameters) public RenderDataClient getRenderDataClient() { if (rdc == null) { - rdc = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + rdc = new RenderDataClient(renderWebParameters.baseDataUrl, + renderWebParameters.owner, + renderWebParameters.project); } return rdc; } @@ -286,7 +163,7 @@ public void createEmptyImageFile() final BufferedImage emptyImage = new BufferedImage(boxWidth, boxHeight, BufferedImage.TYPE_INT_ARGB); - if (parameters.label) { + if (boxParameters.label) { final Graphics2D targetGraphics = emptyImage.createGraphics(); targetGraphics.setBackground(new Color(backgroundRGBColor)); @@ -321,13 +198,13 @@ public void generateBoxesForZ(final Double z) final Bounds layerBounds = getRenderDataClient().getLayerBounds(stack, z); final SectionBoxBounds boxBounds = new SectionBoxBounds(z, boxWidth, boxHeight, layerBounds); - if (parameters.renderGroup != null) { - boxBounds.setRenderGroup(parameters.renderGroup, parameters.numberOfRenderGroups, parameters.maxLevel); + if (boxParameters.renderGroup != null) { + boxBounds.setRenderGroup(boxParameters.renderGroup, boxParameters.numberOfRenderGroups, boxParameters.maxLevel); } final int tileCount; final ImageProcessorCache imageProcessorCache; - if (parameters.label) { + if (boxParameters.label) { // retrieve all tile specs for layer so that imageUrls can be consistently mapped to label colors // (this allows label runs to be resumed after failures) @@ -352,7 +229,7 @@ public void generateBoxesForZ(final Double z) z, layerBounds, boxBounds, tileCount); BoxMipmapGenerator boxMipmapGenerator = new BoxMipmapGenerator(z.intValue(), - parameters.label, + boxParameters.label, format, boxWidth, boxHeight, @@ -362,9 +239,9 @@ public void generateBoxesForZ(final Double z) boxBounds.getLastRow(), boxBounds.getFirstColumn(), boxBounds.getLastColumn(), - parameters.forceGeneration); + boxParameters.forceGeneration); final IGridPaths iGridPaths; - if (parameters.createIGrid) { + if (boxParameters.createIGrid) { iGridPaths = new IGridPaths(boxBounds.getNumberOfRows(), boxBounds.getNumberOfColumns()); } else { iGridPaths = null; @@ -385,16 +262,16 @@ public void generateBoxesForZ(final Double z) final Path overviewDirPath = Paths.get(boxDirectory.getAbsolutePath(), "small"); final String overviewFileName = z.intValue() + "." + format.toLowerCase(); final File overviewFile = new File(overviewDirPath.toFile(), overviewFileName).getAbsoluteFile(); - boolean isOverviewGenerated = (! parameters.forceGeneration) && overviewFile.exists(); + boolean isOverviewGenerated = (! boxParameters.forceGeneration) && overviewFile.exists(); if (isOverviewGenerated) { LOG.info("generateBoxesForZ: {}, overview {} already generated", z, overviewFile.getAbsolutePath()); } - for (int level = 0; level < parameters.maxLevel; level++) { + for (int level = 0; level < boxParameters.maxLevel; level++) { boxMipmapGenerator = boxMipmapGenerator.generateNextLevel(); - if (parameters.isOverviewNeeded() && (! isOverviewGenerated)) { - isOverviewGenerated = boxMipmapGenerator.generateOverview(parameters.maxOverviewWidthAndHeight, + if (boxParameters.isOverviewNeeded() && (! isOverviewGenerated)) { + isOverviewGenerated = boxMipmapGenerator.generateOverview(boxParameters.maxOverviewWidthAndHeight, stackBounds, overviewFile); } @@ -434,7 +311,7 @@ private void generateLevelZero(final Double z, row, column); - if (parameters.forceGeneration || (!levelZeroFile.exists())) { + if (boxParameters.forceGeneration || (!levelZeroFile.exists())) { renderParameters = generateLevelZeroBox(x, y, z, imageProcessorCache, @@ -518,14 +395,14 @@ private RenderParameters generateLevelZeroBox(final int x, final int column) throws IOException { - final String parametersUrl = + final String boxParametersUrl = getRenderDataClient().getRenderParametersUrlString(stack, x, y, z, boxWidth, boxHeight, 1.0); - LOG.info("generateLevelZeroBox: loading {}", parametersUrl); + LOG.info("generateLevelZeroBox: loading {}", boxParametersUrl); - final RenderParameters renderParameters = RenderParameters.loadFromUrl(parametersUrl); - renderParameters.setSkipInterpolation(parameters.skipInterpolation); - renderParameters.setBinaryMask(parameters.binaryMask); + final RenderParameters renderParameters = RenderParameters.loadFromUrl(boxParametersUrl); + renderParameters.setSkipInterpolation(boxParameters.skipInterpolation); + renderParameters.setBinaryMask(boxParameters.binaryMask); renderParameters.setBackgroundRGBColor(backgroundRGBColor); if (renderParameters.hasTileSpecs()) { @@ -536,7 +413,7 @@ private RenderParameters generateLevelZeroBox(final int x, BoxMipmapGenerator.saveImage(levelZeroImage, levelZeroFile, - parameters.label, + boxParameters.label, format); boxMipmapGenerator.addSource(row, column, levelZeroFile); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/BoxRemovalClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/BoxRemovalClient.java index 2144bf071..898ad4b4b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/BoxRemovalClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/BoxRemovalClient.java @@ -7,6 +7,7 @@ import java.util.List; import org.apache.commons.io.FileUtils; +import org.janelia.render.client.parameter.CommandLineParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,20 +21,30 @@ */ public class BoxRemovalClient implements Serializable { - @SuppressWarnings("ALL") - private static class Parameters extends CommandLineParameters { + public static class Parameters extends CommandLineParameters { - @Parameter(names = "--stackDirectory", description = "Stack directory containing boxes to remove (e.g. /tier2/flyTEM/nobackup/rendered_boxes/FAFB00/v7_align_tps/8192x8192)", required = true) - private String stackDirectory; + @Parameter( + names = "--stackDirectory", + description = "Stack directory containing boxes to remove (e.g. /tier2/flyTEM/nobackup/rendered_boxes/FAFB00/v7_align_tps/8192x8192)", + required = true) + public String stackDirectory; - @Parameter(names = "--minLevel", description = "Minimum mipmap level to remove", required = false) - private int minLevel = 0; + @Parameter( + names = "--minLevel", + description = "Minimum mipmap level to remove", + required = false) + public int minLevel = 0; - @Parameter(names = "--maxLevel", description = "Maximum mipmap level to remove (values > 8 will also delete small overview images)", required = false) - private int maxLevel = 9; + @Parameter( + names = "--maxLevel", + description = "Maximum mipmap level to remove (values > 8 will also delete small overview images)", + required = false) + public int maxLevel = 9; - @Parameter(description = "Z values for layers to remove", required = true) - private List zValues; + @Parameter( + description = "Z values for layers to remove", + required = true) + public List zValues; } @@ -46,7 +57,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, BoxRemovalClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/CoordinateClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/CoordinateClient.java index c1196d615..8c797cb14 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/CoordinateClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/CoordinateClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -29,7 +30,10 @@ import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.stack.StackVersion; +import org.janelia.alignment.util.FileUtil; import org.janelia.alignment.util.ProcessTimer; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,63 +47,88 @@ */ public class CoordinateClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter(names = "--stack", description = "Stack name", required = true) - private String stack; - - @Parameter(names = "--z", description = "Z value for all source coordinates", required = false) - private Double z; + @Parameter( + names = "--stack", + description = "Stack name", + required = true) + public String stack; @Parameter( names = "--toOwner", description = "Name of target stack owner (for round trip mapping, default is source owner)", required = false) - private String toOwner; + public String toOwner; @Parameter( names = "--toProject", description = "Name of target stack project (for round trip mapping, default is source owner)", required = false) - private String toProject; + public String toProject; @Parameter( names = "--toStack", description = "Name of target stack (for round trip mapping, omit for one way mapping)", required = false) - private String toStack; - - @Parameter(names = "--fromSwcDirectory", description = "directory containing source .swc files", required = false) - private String fromSwcDirectory; + public String toStack; - @Parameter(names = "--toSwcDirectory", description = "directory to write target .swc files with mapped coordinates", required = false) - private String toSwcDirectory; + @Parameter( + names = "--z", + description = "Z value for all source coordinates", + required = false) + public Double z; - @Parameter(names = "--fromJson", description = "JSON file containing coordinates to be mapped (.json, .gz, or .zip)", required = false) - private String fromJson; + @Parameter( + names = "--fromJson", + description = "JSON file containing coordinates to be mapped (.json, .gz, or .zip)", + required = false) + public String fromJson; - @Parameter(names = "--toJson", description = "JSON file where mapped coordinates are to be stored (.json, .gz, or .zip)", required = false) + @Parameter( + names = "--toJson", + description = "JSON file where mapped coordinates are to be stored (.json, .gz, or .zip)", + required = false) private String toJson; - @Parameter(names = "--localToWorld", description = "Convert from local to world coordinates (default is to convert from world to local)", required = false, arity = 0) - private boolean localToWorld = false; + @Parameter( + names = "--localToWorld", + description = "Convert from local to world coordinates (default is to convert from world to local)", + required = false, + arity = 0) + public boolean localToWorld = false; - @Parameter(names = "--numberOfThreads", description = "Number of threads to use for conversion", required = false) - private int numberOfThreads = 1; + @Parameter( + names = "--fromSwcDirectory", + description = "directory containing source .swc files", + required = false) + public String fromSwcDirectory; + + @Parameter( + names = "--toSwcDirectory", + description = "directory to write target .swc files with mapped coordinates", + required = false) + public String toSwcDirectory; + + @Parameter( + names = "--numberOfThreads", + description = "Number of threads to use for conversion", + required = false) + public int numberOfThreads = 1; public String getToOwner() { if (toOwner == null) { - toOwner = owner; + toOwner = renderWeb.owner; } return toOwner; } public String getToProject() { if (toProject == null) { - toProject = project; + toProject = renderWeb.project; } return toProject; } @@ -170,14 +199,12 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, CoordinateClient.class); + parameters.parse(args); parameters.validateInputAndOutput(); LOG.info("runClient: entry, parameters={}", parameters); - final RenderDataClient renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient renderDataClient = parameters.renderWeb.getDataClient(); final CoordinateClient client = new CoordinateClient(parameters.stack, parameters.z, @@ -200,9 +227,10 @@ public void runClient(final String[] args) throws Exception { } else { - final RenderDataClient targetRenderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.getToOwner(), - parameters.getToProject()); + final RenderDataClient targetRenderDataClient = + new RenderDataClient(parameters.renderWeb.baseDataUrl, + parameters.getToOwner(), + parameters.getToProject()); final CoordinateClient targetClient = new CoordinateClient(parameters.toStack, null, diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/CopyStackClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/CopyStackClient.java index a051b023d..2c5b20f10 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/CopyStackClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/CopyStackClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.util.Collections; @@ -11,6 +12,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import mpicbg.trakem2.transform.AffineModel2D; @@ -25,6 +27,9 @@ import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.stack.StackMetaData.StackState; import org.janelia.alignment.util.ProcessTimer; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.LayerBoundsParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,13 +40,16 @@ */ public class CopyStackClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter(names = "--fromStack", description = "Name of source stack", required = true) - private String fromStack; + @Parameter( + names = "--fromStack", + description = "Name of source stack", + required = true) + public String fromStack; @Parameter( names = "--toOwner", @@ -55,82 +63,63 @@ private static class Parameters extends RenderDataClientParameters { required = false) private String toProject; - @Parameter(names = "--toStack", description = "Name of target stack", required = true) + @Parameter( + names = "--toStack", + description = "Name of target stack", + required = true) private String toStack; - @Parameter(names = "--z", description = "Z value of section to be copied", required = true) - private List zValues; - - @Parameter(names = "--minX", description = "Minimum X value for all tiles", required = false) - private Double minX; - - @Parameter(names = "--maxX", description = "Maximum X value for all tiles", required = false) - private Double maxX; - - @Parameter(names = "--minY", description = "Minimum Y value for all tiles", required = false) - private Double minY; + @Parameter( + names = "--z", + description = "Z value of section to be copied", + required = true) + public List zValues; - @Parameter(names = "--maxY", description = "Maximum Y value for all tiles", required = false) - private Double maxY; + @ParametersDelegate + public LayerBoundsParameters layerBounds = new LayerBoundsParameters(); @Parameter( names = "--keepExisting", description = "Keep any existing target stack tiles with the specified z (default is to remove them)", - required = false, arity = 0) - private boolean keepExisting = false; + required = false, + arity = 0) + public boolean keepExisting = false; @Parameter( names = "--completeToStackAfterCopy", description = "Complete the to stack after copying all layers", - required = false, arity = 0) - private boolean completeToStackAfterCopy = false; + required = false, + arity = 0) + public boolean completeToStackAfterCopy = false; @Parameter( names = "--replaceLastTransformWithStage", description = "Replace the last transform in each tile space with a 'stage identity' transform", - required = false, arity = 0) - private boolean replaceLastTransformWithStage = false; + required = false, + arity = 0) + public boolean replaceLastTransformWithStage = false; @Parameter( names = "--splitMergedSections", description = "Reset z values for tiles so that original sections are separated", - required = false, arity = 0) - private boolean splitMergedSections = false; + required = false, + arity = 0) + public boolean splitMergedSections = false; public String getToOwner() { if (toOwner == null) { - toOwner = owner; + toOwner = renderWeb.owner; } return toOwner; } public String getToProject() { if (toProject == null) { - toProject = project; + toProject = renderWeb.project; } return toProject; } - public void validateStackBounds() throws IllegalArgumentException { - - if ((minX != null) || (maxX != null) || (minY != null) || (maxY != null)) { - - if ((minX == null) || (maxX == null) || (minY == null) || (maxY == null)) { - throw new IllegalArgumentException("since one or more of minX (" + minX + "), maxX (" + maxX + - "), minY (" + minY + "), maxY (" + maxY + - ") is specified, all must be specified"); - } - - if (minX > maxX) { - throw new IllegalArgumentException("minX (" + minX + ") is greater than maxX (" + maxX + ")"); - } - - if (minY > maxY) { - throw new IllegalArgumentException("minY (" + minY + ") is greater than maxY (" + maxY + ")"); - } - } - - } } public static void main(final String[] args) { @@ -139,8 +128,8 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, CopyStackClient.class); - parameters.validateStackBounds(); + parameters.parse(args); + parameters.layerBounds.validate(); LOG.info("runClient: entry, parameters={}", parameters); @@ -169,11 +158,9 @@ public CopyStackClient(final Parameters parameters) throws Exception { this.parameters = parameters; - this.fromDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.fromDataClient = parameters.renderWeb.getDataClient(); - this.toDataClient = new RenderDataClient(parameters.baseDataUrl, + this.toDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, parameters.getToOwner(), parameters.getToProject()); @@ -198,7 +185,7 @@ public void copyLayer(final Double z) throws Exception { final ResolvedTileSpecCollection sourceCollection = fromDataClient.getResolvedTiles(parameters.fromStack, z); - if (parameters.minX != null) { + if (parameters.layerBounds.minX != null) { final Set tileIdsToKeep = getIdsForTilesInBox(z); sourceCollection.filterSpecs(tileIdsToKeep); } @@ -239,10 +226,12 @@ private Set getIdsForTilesInBox(final Double z) throws Exception { final Set tileIdsToKeep = new HashSet<>(tileBoundsList.size()); - for (final TileBounds tileBounds : tree.findTilesInBox(parameters.minX, parameters.minY, - parameters.maxX, parameters.maxY)) { - tileIdsToKeep.add(tileBounds.getTileId()); - } + tileIdsToKeep.addAll( + tree.findTilesInBox(parameters.layerBounds.minX, + parameters.layerBounds.minY, + parameters.layerBounds.maxX, + parameters.layerBounds.maxY).stream().map( + TileBounds::getTileId).collect(Collectors.toList())); if (tileBoundsList.size() > tileIdsToKeep.size()) { LOG.info("getIdsForTilesInBox: removed {} tiles outside of bounding box", @@ -284,16 +273,12 @@ private void replaceLastTransformWithStage(final ResolvedTileSpecCollection sour private Map getSectionIdToIntegralZMap() throws IOException { - final Comparator sectionComparator = new Comparator() { - @Override - public int compare(final SectionData o1, - final SectionData o2) { - int result = o1.getZ().compareTo(o2.getZ()); - if (result == 0) { - result = o1.getSectionId().compareTo(o2.getSectionId()); - } - return result; + final Comparator sectionComparator = (o1, o2) -> { + int result = o1.getZ().compareTo(o2.getZ()); + if (result == 0) { + result = o1.getSectionId().compareTo(o2.getSectionId()); } + return result; }; final List orderedSectionDataList = diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/FixAutoLoaderScaleClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/FixAutoLoaderScaleClient.java index 24cb28b1e..7eb5cc78b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/FixAutoLoaderScaleClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/FixAutoLoaderScaleClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.util.List; @@ -9,6 +10,9 @@ import org.janelia.alignment.spec.ResolvedTileSpecCollection; import org.janelia.alignment.spec.TransformSpec; import org.janelia.alignment.spec.validator.TileSpecValidator; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,44 +23,46 @@ */ public class FixAutoLoaderScaleClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParametersWithValidator { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); @Parameter( names = "--stack", description = "Name of source stack containing base tile specifications", required = true) - private String stack; + public String stack; @Parameter( names = "--targetProject", description = "Name of target project that will contain imported transforms (default is to reuse source project)", required = false) - private String targetProject; + public String targetProject; @Parameter( names = "--targetStack", description = "Name of target (align, montage, etc.) stack that will contain imported transforms", required = true) - private String targetStack; + public String targetStack; @Parameter( names = "--newScale", description = "Corrected scale", required = true) - private Double newScale; + public Double newScale; @Parameter( names = "--oldScale", description = "Current scale to be replaced", required = false) - private Double oldScale = 0.935; + public Double oldScale = 0.935; @Parameter(description = "Z values of layers with tiles needing correction", required = true) - private List zValues; + public List zValues; } public static void main(final String[] args) { @@ -65,7 +71,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, FixAutoLoaderScaleClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -91,19 +97,17 @@ public void runClient(final String[] args) throws Exception { public FixAutoLoaderScaleClient(final Parameters parameters) { this.parameters = parameters; - this.tileSpecValidator = parameters.getValidatorInstance(); + this.tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); - this.sourceRenderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.sourceRenderDataClient = parameters.renderWeb.getDataClient(); if ((parameters.targetProject == null) || (parameters.targetProject.trim().length() == 0) || - (parameters.targetProject.equals(parameters.project))){ + (parameters.targetProject.equals(parameters.renderWeb.project))){ this.targetRenderDataClient = sourceRenderDataClient; } else { - this.targetRenderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, + this.targetRenderDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, + parameters.renderWeb.owner, parameters.targetProject); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportJsonClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportJsonClient.java index 82722b0da..dfec19bd9 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportJsonClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportJsonClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Reader; @@ -14,7 +15,11 @@ import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.TransformSpec; import org.janelia.alignment.spec.validator.TileSpecValidator; +import org.janelia.alignment.util.FileUtil; import org.janelia.alignment.util.ProcessTimer; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,20 +30,30 @@ */ public class ImportJsonClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParametersWithValidator { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter(names = "--stack", description = "Name of stack for imported data", required = true) - private String stack; + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); - @Parameter(names = "--transformFile", description = "file containing shared JSON transform specs (.json, .gz, or .zip)", required = false) - private String transformFile; + @Parameter( + names = "--stack", + description = "Name of stack for imported data", + required = true) + public String stack; - @Parameter(description = "list of tile spec files (.json, .gz, or .zip)", required = true) - private List tileFiles; + @Parameter( + names = "--transformFile", + description = "file containing shared JSON transform specs (.json, .gz, or .zip)", + required = false) + public String transformFile; + + @Parameter( + description = "list of tile spec files (.json, .gz, or .zip)", + required = true) + public List tileFiles; } public static void main(final String[] args) { @@ -47,7 +62,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, ImportJsonClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -70,11 +85,9 @@ public void runClient(final String[] args) throws Exception { public ImportJsonClient(final Parameters parameters) throws IOException { this.parameters = parameters; - this.tileSpecValidator = parameters.getValidatorInstance(); + this.tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.renderDataClient = parameters.renderWeb.getDataClient(); this.renderDataClient.ensureStackIsInLoadingState(parameters.stack, null); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportMETClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportMETClient.java index e65929f86..a84e54982 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportMETClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportMETClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.BufferedReader; import java.io.IOException; @@ -26,6 +27,8 @@ import org.janelia.alignment.spec.validator.TileSpecValidator; import org.janelia.alignment.util.ProcessTimer; import org.janelia.render.client.ImportTransformChangesClient.ChangeMode; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,13 +39,13 @@ */ public class ImportMETClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); @Parameter(names = "--stack", description = "Name of source stack containing base tile specifications", required = true) - private String stack; + public String stack; @Parameter( names = "--targetOwner", @@ -56,34 +59,47 @@ private static class Parameters extends RenderDataClientParameters { required = false) private String targetProject; - @Parameter(names = "--targetStack", description = "Name of target (align, montage, etc.) stack that will contain imported transforms", required = true) + @Parameter( + names = "--targetStack", + description = "Name of target (align, montage, etc.) stack that will contain imported transforms", + required = true) private String targetStack; - @Parameter(names = "--metFile", description = "MET file for section", required = true) - private String metFile; + @Parameter( + names = "--metFile", + description = "MET file for section", + required = true) + public String metFile; - @Parameter(names = "--formatVersion", description = "MET format version ('v1', v2', 'v3', ...)", required = false) - private String formatVersion = "v1"; + @Parameter( + names = "--formatVersion", + description = "MET format version ('v1', v2', 'v3', ...)", + required = false) + public String formatVersion = "v1"; @Parameter( names = "--changeMode", description = "Specifies how the transforms should be applied to existing data", required = false) - private ChangeMode changeMode = ChangeMode.REPLACE_LAST; + public ChangeMode changeMode = ChangeMode.REPLACE_LAST; - @Parameter(names = "--disableValidation", description = "Disable flyTEM tile validation", required = false, arity = 0) - private boolean disableValidation; + @Parameter( + names = "--disableValidation", + description = "Disable flyTEM tile validation", + required = false, + arity = 0) + public boolean disableValidation; public String getTargetOwner() { if (targetOwner == null) { - targetOwner = owner; + targetOwner = renderWeb.owner; } return targetOwner; } public String getTargetProject() { if (targetProject == null) { - targetProject = project; + targetProject = renderWeb.project; } return targetProject; } @@ -96,7 +112,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, ImportMETClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -125,11 +141,9 @@ public ImportMETClient(final Parameters parameters) { this.tileSpecValidator = new TemTileSpecValidator(); } - this.sourceRenderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.sourceRenderDataClient = parameters.renderWeb.getDataClient(); - this.targetRenderDataClient = new RenderDataClient(parameters.baseDataUrl, + this.targetRenderDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, parameters.getTargetOwner(), parameters.getTargetProject()); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportMatchClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportMatchClient.java index 31a8cc695..d50ab1efc 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportMatchClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportMatchClient.java @@ -10,8 +10,10 @@ import java.util.List; import org.janelia.alignment.match.CanvasMatches; +import org.janelia.alignment.util.FileUtil; import org.janelia.alignment.util.ProcessTimer; -import org.janelia.render.client.parameters.MatchDataClientParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MatchWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +27,7 @@ public class ImportMatchClient { public static class Parameters extends CommandLineParameters { @ParametersDelegate - public MatchDataClientParameters matchClient = new MatchDataClientParameters(); + public MatchWebServiceParameters matchClient = new MatchWebServiceParameters(); @Parameter( names = "--batchSize", @@ -45,7 +47,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, ImportMatchClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportTileZValuesClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportTileZValuesClient.java index b6628b90e..b2f88e2f8 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportTileZValuesClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportTileZValuesClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.BufferedReader; import java.io.IOException; @@ -14,6 +15,8 @@ import java.util.Map; import java.util.regex.Pattern; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,16 +27,22 @@ */ public class ImportTileZValuesClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter(names = "--stack", description = "Name of stack containing tile specifications", required = true) - private String stack; + @Parameter( + names = "--stack", + description = "Name of stack containing tile specifications", + required = true) + public String stack; - @Parameter(names = "--dataFile", description = "File containing tileId to z value mappings", required = true) - private String dataFile; + @Parameter( + names = "--dataFile", + description = "File containing tileId to z value mappings", + required = true) + public String dataFile; } @@ -43,7 +52,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, ImportTileZValuesClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -65,9 +74,7 @@ public ImportTileZValuesClient(final Parameters parameters) throws IOException { this.parameters = parameters; - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.renderDataClient = parameters.renderWeb.getDataClient(); this.renderDataClient.ensureStackIsInLoadingState(parameters.stack, null); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportTransformChangesClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportTransformChangesClient.java index 95eaea98c..b017ae059 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ImportTransformChangesClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ImportTransformChangesClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Reader; @@ -16,7 +17,11 @@ import org.janelia.alignment.spec.TileTransform; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.validator.TileSpecValidator; +import org.janelia.alignment.util.FileUtil; import org.janelia.alignment.util.ProcessTimer; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,17 +34,19 @@ public class ImportTransformChangesClient { public enum ChangeMode { APPEND, REPLACE_LAST, REPLACE_ALL } - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParametersWithValidator { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); @Parameter( names = "--stack", description = "Name of source stack containing base tile specifications", required = true) - private String stack; + public String stack; @Parameter(names = "--targetOwner", @@ -63,24 +70,24 @@ private static class Parameters extends RenderDataClientParametersWithValidator names = "--transformFile", description = "File containing list of transform changes (.json, .gz, or .zip). For best performance, changes for all tiles with the same z should be grouped into the same file.", required = true) - private String transformFile; + public String transformFile; @Parameter( names = "--changeMode", description = "Specifies how the transforms should be applied to existing data", required = false) - private ChangeMode changeMode = ChangeMode.REPLACE_LAST; + public ChangeMode changeMode = ChangeMode.REPLACE_LAST; public String getTargetOwner() { if (targetOwner == null) { - targetOwner = owner; + targetOwner = renderWeb.owner; } return targetOwner; } public String getTargetProject() { if (targetProject == null) { - targetProject = project; + targetProject = renderWeb.project; } return targetProject; } @@ -93,7 +100,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, ImportTransformChangesClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -116,13 +123,11 @@ public void runClient(final String[] args) throws Exception { public ImportTransformChangesClient(final Parameters parameters) { this.parameters = parameters; - this.tileSpecValidator = parameters.getValidatorInstance(); + this.tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); - this.sourceRenderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.sourceRenderDataClient = parameters.renderWeb.getDataClient(); - this.targetRenderDataClient = new RenderDataClient(parameters.baseDataUrl, + this.targetRenderDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, parameters.getTargetOwner(), parameters.getTargetProject()); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MipmapClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MipmapClient.java index 8b72d94a0..10128ce32 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MipmapClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MipmapClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import ij.ImagePlus; import ij.io.Opener; @@ -11,7 +12,6 @@ import java.io.File; import java.io.IOException; import java.net.URI; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -25,6 +25,9 @@ import org.janelia.alignment.spec.stack.MipmapPathBuilder; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.stack.StackVersion; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MipmapParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,112 +39,18 @@ */ public class MipmapClient { - @SuppressWarnings("ALL") - public static class CommonParameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter( - names = "--stack", - description = "Stack name", - required = true) - public String stack; + @ParametersDelegate + public MipmapParameters mipmap = new MipmapParameters(); @Parameter( - names = "--rootDirectory", - description = "Root directory for mipmaps (e.g. /nrs/flyTEM/rendered_mipmaps/FAFB00)", + description = "Z values for layers to render", required = true) - protected String rootDirectory; - - @Parameter( - names = "--minLevel", - description = "Minimum mipmap level to generate", - required = false) - protected Integer minLevel = 1; - - @Parameter(names = "--maxLevel", description = "Maximum mipmap level to generate", required = false) - protected Integer maxLevel = 6; - - @Parameter(names = "--format", description = "Format for mipmaps (tiff, jpg, png)", required = false) - protected String format = Utils.TIFF_FORMAT; - - @Parameter(names = "--forceGeneration", description = "Regenerate mipmaps even if they already exist", required = false, arity = 0) - protected boolean forceGeneration = false; - - public CommonParameters() { - this(null, null, null, null, null, Utils.TIFF_FORMAT, 1, 6, false); - } - - public CommonParameters(final String rootDirectory, - final Integer maxLevel) { - this(null, null, null, null, rootDirectory, Utils.TIFF_FORMAT, 1, maxLevel, false); - } - - public CommonParameters(final String baseDataUrl, - final String owner, - final String project, - final String stack, - final String rootDirectory, - final String format, - final Integer minLevel, - final Integer maxLevel, - final boolean forceGeneration) { - super(baseDataUrl, owner, project); - this.stack = stack; - this.rootDirectory = rootDirectory; - this.format = format; - this.minLevel = minLevel; - this.maxLevel = maxLevel; - this.forceGeneration = forceGeneration; - } - - public MipmapPathBuilder getMipmapPathBuilder() - throws IOException { - - final File dir = new File(rootDirectory).getCanonicalFile(); - - if (! dir.exists()) { - throw new IOException("missing root directory " + rootDirectory); - } - - if (! dir.canWrite()) { - throw new IOException("not allowed to write to root directory " + rootDirectory); - } - - String extension = format; - // map 'tiff' format to 'tif' extension so that {@link ij.io.Opener#openURL(String)} method will work. - if (Utils.TIFF_FORMAT.equals(format)) { - extension = "tif"; - } - - return new MipmapPathBuilder(dir.getPath(), maxLevel, extension); - } - } - - @SuppressWarnings("ALL") - public static class Parameters extends CommonParameters { - - @Parameter(names = "--renderGroup", description = "Index (1-n) that identifies portion of layer to render (omit if only one job is being used)", required = false) - private Integer renderGroup = 1; - - @Parameter(names = "--numberOfRenderGroups", description = "Total number of parallel jobs being used to render this layer (omit if only one job is being used)", required = false) - private Integer numberOfRenderGroups = 1; - - @Parameter(description = "Z values for layers to render", required = true) - private List zValues; - - public Parameters() { - this(new CommonParameters(), new ArrayList()); - } - - public Parameters(final CommonParameters commonParameters, - final List zValues) { - super(commonParameters.baseDataUrl, - commonParameters.owner, commonParameters.project, commonParameters.stack, - commonParameters.rootDirectory, commonParameters.format, - commonParameters.minLevel, commonParameters.maxLevel, commonParameters.forceGeneration); - this.zValues = zValues; - } + public List zValues; } @@ -154,11 +63,12 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, MipmapClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); - final MipmapClient client = new MipmapClient(parameters); + final MipmapClient client = new MipmapClient(parameters.renderWeb, + parameters.mipmap); for (final Double z : parameters.zValues) { client.generateMipmapsForZ(z); } @@ -168,13 +78,14 @@ public void runClient(final String[] args) throws Exception { clientRunner.run(); } - private final Parameters parameters; + private final MipmapParameters parameters; private final String stack; private final MipmapPathBuilder mipmapPathBuilder; private final RenderDataClient renderDataClient; - public MipmapClient(final Parameters parameters) + public MipmapClient(final RenderWebServiceParameters renderWebParameters, + final MipmapParameters parameters) throws IOException { this.parameters = parameters; @@ -204,9 +115,7 @@ public MipmapClient(final Parameters parameters) "renderGroup (1-n) must be specified when numberOfRenderGroups are specified"); } - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.renderDataClient = renderWebParameters.getDataClient(); } public MipmapPathBuilder getMipmapPathBuilder() { diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchClient.java index 094ef3631..9492df094 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchClient.java @@ -28,9 +28,10 @@ import org.janelia.alignment.spec.Bounds; import org.janelia.alignment.spec.LayoutData; import org.janelia.alignment.spec.TileSpec; -import org.janelia.render.client.parameters.MatchClipParameters; -import org.janelia.render.client.parameters.MatchDataClientParameters; -import org.janelia.render.client.parameters.MatchDerivationParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MatchClipParameters; +import org.janelia.render.client.parameter.MatchDerivationParameters; +import org.janelia.render.client.parameter.MatchWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +75,7 @@ public enum RenderFileFormat { public static class Parameters extends CommandLineParameters { @ParametersDelegate - public MatchDataClientParameters matchClient = new MatchDataClientParameters(); + public MatchWebServiceParameters matchClient = new MatchWebServiceParameters(); @ParametersDelegate public MatchDerivationParameters match = new MatchDerivationParameters(); @@ -85,22 +86,19 @@ public static class Parameters extends CommandLineParameters { @Parameter( names = "--renderScale", description = "Render canvases at this scale", - required = false, - order = 10) + required = false) public Double renderScale = 1.0; @Parameter(names = "--fillWithNoise", description = "Fill each canvas image with noise before rendering to improve point match derivation", required = false, - arity = 1, - order = 15) + arity = 1) public boolean fillWithNoise = true; @Parameter( names = "--renderFileFormat", description = "Format for saved canvases (only relevant if debugDirectory is specified)", - required = false, - order = 16) + required = false) public RenderFileFormat renderFileFormat = RenderFileFormat.JPG; @Parameter( @@ -235,7 +233,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, PointMatchClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchOptimizerClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchOptimizerClient.java index eafd7229f..e399ddd3d 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchOptimizerClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchOptimizerClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.util.ArrayList; import java.util.HashMap; @@ -14,7 +15,8 @@ import org.janelia.alignment.match.CanvasFeatureExtractor; import org.janelia.alignment.match.CanvasFeatureMatchResult; import org.janelia.alignment.match.CanvasFeatureMatcher; -import org.janelia.alignment.match.ModelType; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MatchDerivationParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,7 +28,6 @@ */ public class PointMatchOptimizerClient { - @SuppressWarnings("ALL") public static class Parameters extends CommandLineParameters { @Parameter( @@ -51,79 +52,28 @@ public static class Parameters extends CommandLineParameters { names = "--renderScaleStep", description = "Amount to adjust render scale for each iteration during optimization", required = false) - private Double renderScaleStep = 0.1; + public Double renderScaleStep = 0.1; @Parameter( names = "--minFeatureCount", description = "Minimum number features for optimal render scale", required = false) - private Integer minFeatureCount = 3000; + public Integer minFeatureCount = 3000; @Parameter( names = "--maxFeatureCount", description = "Maximum number features for optimal render scale", required = false) - private Integer maxFeatureCount = 6000; + public Integer maxFeatureCount = 6000; - @Parameter( - names = "--SIFTfdSize", - description = "SIFT feature descriptor size: how many samples per row and column", - required = false) - private Integer fdSize = 8; - - @Parameter( - names = "--SIFTsteps", - description = "SIFT steps per scale octave", - required = false) - private Integer steps = 2; - - @Parameter( - names = "--matchModelType", - description = "Type of model for match filtering", - required = false) - private ModelType matchModelType = ModelType.AFFINE; - - @Parameter( - names = "--matchIterations", - description = "Match filter iterations", - required = false) - private Integer matchIterations = 1000; - - @Parameter( - names = "--matchMaxEpsilon", - description = "Minimal allowed transfer error for match filtering", - required = false) - private Float matchMaxEpsilon = 20.0f; - - @Parameter( - names = "--matchMinInlierRatio", - description = "Minimal ratio of inliers to candidates for match filtering", - required = false) - private Float matchMinInlierRatio = 0.0f; - - @Parameter( - names = "--matchMinNumInliers", - description = "Minimal absolute number of inliers for match filtering", - required = false) - private Integer matchMinNumInliers = 4; - - @Parameter( - names = "--matchMaxTrust", - description = "Reject match candidates with a cost larger than maxTrust * median cost", - required = false) - private Double matchMaxTrust = 3.0; - - @Parameter( - names = "--matchMaxNumInliers", - description = "Maximum number of inliers for match filtering", - required = false) - private Integer matchMaxNumInliers = 20; + @ParametersDelegate + public MatchDerivationParameters match = new MatchDerivationParameters(); @Parameter( names = "--matchRodStep", description = "Amount to adjust ratio of distances for each iteration during optimization", required = false) - private Double matchRodStep = 0.05; + public Double matchRodStep = 0.05; } @@ -136,7 +86,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, PointMatchOptimizerClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -225,21 +175,21 @@ public void run() throws Exception { LOG.info("run: testing match rod {}", rod); matcher = new CanvasFeatureMatcher(rod, - parameters.matchModelType, - parameters.matchIterations, - parameters.matchMaxEpsilon, - parameters.matchMinInlierRatio, - parameters.matchMinNumInliers, - parameters.matchMaxTrust, + parameters.match.matchModelType, + parameters.match.matchIterations, + parameters.match.matchMaxEpsilon, + parameters.match.matchMinInlierRatio, + parameters.match.matchMinNumInliers, + parameters.match.matchMaxTrust, null, true); matchResult = matcher.deriveMatchResult(pFeatureList, qFeatureList); inlierCount = matchResult.getInlierPointMatchList().size(); - if (inlierCount < parameters.matchMinNumInliers) { + if (inlierCount < parameters.match.matchMinNumInliers) { - if (previousInlierCount >= parameters.matchMinNumInliers) { + if (previousInlierCount >= parameters.match.matchMinNumInliers) { optimalRod = previousRod; inlierCount = previousInlierCount; } else { @@ -247,7 +197,7 @@ public void run() throws Exception { rod += parameters.matchRodStep; } - } else if (inlierCount > parameters.matchMaxNumInliers) { + } else if (inlierCount > parameters.match.matchMaxNumInliers) { previousRod = rod; rod -= parameters.matchRodStep; @@ -277,8 +227,8 @@ private int extractFeaturesForScale(final double renderScale) { LOG.info("extractFeaturesForScale: entry, scale={}", renderScale); final FloatArray2DSIFT.Param siftParameters = new FloatArray2DSIFT.Param(); - siftParameters.fdSize = parameters.fdSize; - siftParameters.steps = parameters.steps; + siftParameters.fdSize = parameters.match.fdSize; + siftParameters.steps = parameters.match.steps; final CanvasFeatureExtractor extractor = new CanvasFeatureExtractor(siftParameters, renderScale - 0.02, diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderClient.java index 05516f51c..b1df0fba2 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderClient.java @@ -1,62 +1,93 @@ package org.janelia.render.client; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.LinkedHashMap; +import java.util.Map; + import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.janelia.alignment.RenderParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.janelia.render.client.response.BufferedImageResponseHandler; import org.janelia.render.client.response.FileResponseHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.LinkedHashMap; -import java.util.Map; - /** - * Java client for accessing Render Web Services. + * Simple client for rendering based upon a JSON parameters spec. * * @author Eric Trautman */ public class RenderClient { - public static final String DEFAULT_FORMAT = RenderClientParameters.JPEG_FORMAT.toLowerCase(); + public static final String JPEG_FORMAT = "jpeg"; + public static final String PNG_FORMAT = "png"; - /** - * @param args see {@link RenderClientParameters} for command line argument details. - */ - public static void main(String[] args) { - try { - final RenderClientParameters clientParameters = RenderClientParameters.parseCommandLineArgs(args); - - if (clientParameters.displayHelp()) { - clientParameters.showUsage(); - } else { - final RenderClient client = new RenderClient(clientParameters.getBaseUri(), - clientParameters.getProjectId()); - if (clientParameters.renderInWindow()) { - client.renderInWindow(clientParameters.getIn(), - clientParameters.getFormat()); + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @Parameter(names = "--in", + description = "Path to render parameters json file", + required = true) + public String in; + + @Parameter(names = "--out", + description = "Path for the output image file (if omitted, image will be displayed in window)", + required = false) + public String out; + + @Parameter(names = "--format", + description = "Format for output image (jpeg or png, default is jpeg)", + required = false) + public String format; + + public boolean renderInWindow() { + return (out == null); + } + + } + + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + + final Parameters parameters = new Parameters(); + parameters.parse(args); + + LOG.info("runClient: entry, parameters={}", parameters); + + final RenderClient client = new RenderClient(parameters.renderWeb.baseDataUrl, + parameters.renderWeb.project); + + if (parameters.renderInWindow()) { + client.renderInWindow(parameters.in, + parameters.out); } else { - client.renderToFile(clientParameters.getIn(), - clientParameters.getOut(), - clientParameters.getFormat()); + client.renderToFile(parameters.in, + parameters.out, + parameters.format); } } - } catch (Throwable t) { - LOG.error("main: caught exception", t); - System.exit(1); - } + }; + clientRunner.run(); } - private Map formatToRenderUriMap; - private CloseableHttpClient httpClient; + private final Map formatToRenderUriMap; + private final CloseableHttpClient httpClient; /** * @param baseUriString base URI for web service (e.g. http://renderer.int.janelia.org/render-ws/v1). @@ -65,8 +96,8 @@ public static void main(String[] args) { * @throws IllegalArgumentException * if the render URI instances cannot be constructed. */ - public RenderClient(String baseUriString, - String owner) throws IllegalArgumentException { + public RenderClient(final String baseUriString, + final String owner) throws IllegalArgumentException { final String trimmedBaseUriString; if (baseUriString.endsWith("/")) { @@ -79,8 +110,8 @@ public RenderClient(String baseUriString, final URI jpegUri = createUri(projectUriString + "/jpeg-image"); final URI pngUri = createUri(projectUriString + "/png-image"); final Map map = new LinkedHashMap<>(); - map.put(RenderClientParameters.JPEG_FORMAT.toLowerCase(), jpegUri); - map.put(RenderClientParameters.PNG_FORMAT.toLowerCase(), pngUri); + map.put(JPEG_FORMAT.toLowerCase(), jpegUri); + map.put(PNG_FORMAT.toLowerCase(), pngUri); this.formatToRenderUriMap = map; this.httpClient = HttpClients.createDefault(); @@ -91,11 +122,11 @@ public RenderClient(String baseUriString, * * @return render URI for the specified format. */ - public URI getRenderUriForFormat(String format) { + public URI getRenderUriForFormat(final String format) { - String lowerCaseFormat; + final String lowerCaseFormat; if (format == null) { - lowerCaseFormat = DEFAULT_FORMAT; + lowerCaseFormat = JPEG_FORMAT; } else { lowerCaseFormat = format.toLowerCase(); } @@ -103,8 +134,8 @@ public URI getRenderUriForFormat(String format) { URI uri = formatToRenderUriMap.get(lowerCaseFormat); if (uri == null) { LOG.warn("getRenderUriForFormat: unknown format '" + format + "' requested, using '" + - DEFAULT_FORMAT + "' instead, known formats are: " + formatToRenderUriMap.keySet()); - uri = formatToRenderUriMap.get(DEFAULT_FORMAT); + JPEG_FORMAT + "' instead, known formats are: " + formatToRenderUriMap.keySet()); + uri = formatToRenderUriMap.get(JPEG_FORMAT); } return uri; @@ -126,9 +157,9 @@ public String toString() { * @throws IOException * if the render request fails for any reason. */ - public void renderToFile(String renderParametersPath, - String outputFilePath, - String outputFormat) + public void renderToFile(final String renderParametersPath, + final String outputFilePath, + final String outputFormat) throws IOException { final File parametersFile = new File(renderParametersPath); @@ -150,9 +181,9 @@ public void renderToFile(String renderParametersPath, * @throws IOException * if the render request fails for any reason. */ - public void renderToFile(RenderParameters renderParameters, - File outputFile, - String outputFormat) + public void renderToFile(final RenderParameters renderParameters, + final File outputFile, + final String outputFormat) throws IOException { LOG.info("renderToFile: entry, renderParameters={}, outputFile={}, outputFormat={}", @@ -199,8 +230,8 @@ public void renderToFile(RenderParameters renderParameters, * @throws IOException * if the render request fails for any reason. */ - public void renderInWindow(String renderParametersPath, - String outputFormat) + public void renderInWindow(final String renderParametersPath, + final String outputFormat) throws IOException { final File parametersFile = new File(renderParametersPath); final RenderParameters renderParameters = RenderParameters.parseJson(parametersFile); @@ -217,8 +248,8 @@ public void renderInWindow(String renderParametersPath, * @throws IOException * if the render request fails for any reason. */ - public void renderInWindow(RenderParameters renderParameters, - String outputFormat) + public void renderInWindow(final RenderParameters renderParameters, + final String outputFormat) throws IOException { LOG.info("renderInWindow: entry, renderParameters={}, outputFormat={}", renderParameters, outputFormat); @@ -238,11 +269,11 @@ public void renderInWindow(RenderParameters renderParameters, LOG.info("renderInWindow: exit, created {}", image); } - private URI createUri(String uriString) throws IllegalArgumentException { - URI uri; + private URI createUri(final String uriString) throws IllegalArgumentException { + final URI uri; try { uri = new URI(uriString); - } catch (URISyntaxException e) { + } catch (final URISyntaxException e) { throw new IllegalArgumentException("failed to parse URI string: " + uriString, e); } return uri; diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderClientParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderClientParameters.java deleted file mode 100644 index 787019e9e..000000000 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderClientParameters.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.janelia.render.client; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -/** - * Render client command line parameters. - * - * @author Eric Trautman - */ -@Parameters -public class RenderClientParameters { - - public static final String JPEG_FORMAT = "jpeg"; - public static final String PNG_FORMAT = "png"; - - @Parameter(names = "--help", - description = "Display this note", - help = true) - private transient boolean help; - - @Parameter(names = "--baseUri", - description = "Base URI for web services requests", - required = true) - private String baseUri; - - @Parameter( names = "--projectId", - description = "Project ID", - required = true) - private String projectId; - - @Parameter(names = "--in", - description = "Path to render parameters json file", - required = true) - private String in; - - @Parameter(names = "--out", - description = "Path for the output image file (if omitted, image will be displayed in window)", - required = false) - private String out; - - @Parameter(names = "--format", - description = "Format for output image (jpeg or png, default is jpeg)", - required = false) - private String format; - - private transient JCommander jCommander; - - public RenderClientParameters() { - this.help = false; - this.baseUri = null; - this.projectId = null; - this.in = null; - this.out = null; - this.format = JPEG_FORMAT; - this.jCommander = null; - } - - /** - * @param args arguments to parse. - * - * @return parameters instance populated by parsing the specified arguments. - * - * @throws IllegalArgumentException - * if any invalid arguments are specified. - */ - public static RenderClientParameters parseCommandLineArgs(String[] args) throws IllegalArgumentException { - RenderClientParameters parameters = new RenderClientParameters(); - parameters.setCommander(); - try { - parameters.jCommander.parse(args); - } catch (Throwable t) { - throw new IllegalArgumentException("failed to parse command line arguments", t); - } - return parameters; - } - - public boolean displayHelp() { - return help; - } - - public String getBaseUri() { - return baseUri; - } - - public String getProjectId() { - return projectId; - } - - public String getIn() { - return in; - } - - public String getOut() { - return out; - } - - public String getFormat() { - return format; - } - - public boolean renderInWindow() { - return (out == null); - } - - @Override - public String toString() { - return "RenderClientParameters{baseUri='" + baseUri + '\'' + - ", projectId='" + projectId + '\'' + - ", in='" + in + '\'' + - ", out='" + out + '\'' + - ", format='" + format + '\'' + - '}'; - } - - /** - * Displays command usage information on the console (standard-out). - */ - public void showUsage() { - if (jCommander == null) { - setCommander(); - } - jCommander.usage(); - } - - private void setCommander() { - jCommander = new JCommander(this); - jCommander.setProgramName("java -jar current-ws-standalone.jar"); - } -} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderSectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderSectionClient.java index fd47a3a07..a5e414e9c 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderSectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderSectionClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import ij.process.ByteProcessor; @@ -14,7 +15,10 @@ import org.janelia.alignment.RenderParameters; import org.janelia.alignment.Utils; import org.janelia.alignment.spec.Bounds; +import org.janelia.alignment.util.FileUtil; import org.janelia.alignment.util.ImageProcessorCache; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,52 +30,95 @@ */ public class RenderSectionClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { - - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - - @Parameter(names = "--stack", description = "Stack name", required = true) - private String stack; - - @Parameter(names = "--rootDirectory", description = "Root directory for rendered layers (e.g. /tier2/flyTEM/nobackup/rendered_boxes)", required = true) - private String rootDirectory; - - @Parameter(names = "--scale", description = "Scale for each rendered layer", required = false) - private Double scale = 0.02; - - @Parameter(names = "--format", description = "Format for rendered boxes", required = false) - private String format = Utils.PNG_FORMAT; - - @Parameter(names = "--doFilter", description = "Use ad hoc filter to support alignment", required = false, arity = 1) - private boolean doFilter = true; - - @Parameter(names = "--channels", description = "Specify channel(s) and weights to render (e.g. 'DAPI' or 'DAPI__0.7__TdTomato__0.3')", required = false) - private String channels; - - @Parameter(names = "--fillWithNoise", description = "Fill image with noise before rendering to improve point match derivation", required = false, arity = 1) - private boolean fillWithNoise = true; - - @Parameter(description = "Z values for sections to render", required = true) - private List zValues; - - @Parameter(names = "--bounds", description = "Bounds used for all layers: xmin, xmax, ymin,ymax", required = false) - private List bounds; - - @Parameter(names = "--customOutputFolder", description = "Custom named folder for output. Overrides the default format 'sections_at_#' folder", required = false) - private String customOutputFolder; - - @Parameter(names = "--customSubFolder", description = "Name for subfolder to customOutputFolder, if used", required = false) - private String customSubFolder; - - @Parameter(names = "--padFileNamesWithZeros", description = "Pad outputfilenames with leading zeroes, i.e. 12.tiff -> 00012.tiff", required = false) - private boolean padFileNameWithZeroes; - - @Parameter(names = "--maxIntensity",description = "Max intensity to render image", required = false) - private Integer maxIntensity; - - @Parameter(names = "--minIntensity",description = "Min intensity to render image", required = false) - private Integer minIntensity; + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @Parameter( + names = "--stack", + description = "Stack name", + required = true) + public String stack; + + @Parameter( + names = "--rootDirectory", + description = "Root directory for rendered layers (e.g. /tier2/flyTEM/nobackup/rendered_boxes)", + required = true) + public String rootDirectory; + + @Parameter( + names = "--scale", + description = "Scale for each rendered layer", + required = false) + public Double scale = 0.02; + + @Parameter( + names = "--format", + description = "Format for rendered boxes", + required = false) + public String format = Utils.PNG_FORMAT; + + @Parameter( + names = "--doFilter", + description = "Use ad hoc filter to support alignment", + required = false, + arity = 1) + public boolean doFilter = true; + + @Parameter( + names = "--channels", + description = "Specify channel(s) and weights to render (e.g. 'DAPI' or 'DAPI__0.7__TdTomato__0.3')", + required = false) + public String channels; + + @Parameter( + names = "--fillWithNoise", + description = "Fill image with noise before rendering to improve point match derivation", + required = false, + arity = 1) + public boolean fillWithNoise = true; + + @Parameter( + description = "Z values for sections to render", + required = true) + public List zValues; + + @Parameter( + names = "--bounds", + description = "Bounds used for all layers: xmin, xmax, ymin,ymax", + required = false) + public List bounds; + + @Parameter( + names = "--customOutputFolder", + description = "Custom named folder for output. Overrides the default format 'sections_at_#' folder", + required = false) + public String customOutputFolder; + + @Parameter( + names = "--customSubFolder", + description = "Name for subfolder to customOutputFolder, if used", + required = false) + public String customSubFolder; + + @Parameter( + names = "--padFileNamesWithZeros", + description = "Pad outputfilenames with leading zeroes, i.e. 12.tiff -> 00012.tiff", + required = false) + public boolean padFileNameWithZeroes; + + @Parameter( + names = "--maxIntensity", + description = "Max intensity to render image", + required = false) + public Integer maxIntensity; + + @Parameter( + names = "--minIntensity", + description = "Min intensity to render image", + required = false) + public Integer minIntensity; } /** @@ -83,7 +130,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, RenderSectionClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -120,7 +167,7 @@ public RenderSectionClient(final Parameters clientParameters) { } else { final String sectionsAtScaleName = "sections_at_" + clientParameters.scale; sectionPath = Paths.get(clientParameters.rootDirectory, - clientParameters.project, + clientParameters.renderWeb.project, clientParameters.stack, sectionsAtScaleName); } @@ -133,9 +180,7 @@ public RenderSectionClient(final Parameters clientParameters) { final int maxCachedPixels = 50 * 1000000; this.imageProcessorCache = new ImageProcessorCache(maxCachedPixels, false, false); - this.renderDataClient = new RenderDataClient(clientParameters.baseDataUrl, - clientParameters.owner, - clientParameters.project); + this.renderDataClient = clientParameters.renderWeb.getDataClient(); } public void generateImageForZ(final Double z) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/SectionUpdateClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/SectionUpdateClient.java index a6aeb86eb..1140556d8 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/SectionUpdateClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/SectionUpdateClient.java @@ -1,7 +1,10 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,19 +15,19 @@ */ public class SectionUpdateClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); @Parameter(names = "--stack", description = "Stack name", required = true) - private String stack; + public String stack; @Parameter(names = "--sectionId", description = "Section ID", required = true) - private String sectionId; + public String sectionId; @Parameter(names = "--z", description = "Z value", required = true) - private Double z; + public Double z; } /** @@ -36,7 +39,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, SectionUpdateClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -52,9 +55,7 @@ public void runClient(final String[] args) throws Exception { public SectionUpdateClient(final Parameters parameters) { this.parameters = parameters; - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.renderDataClient = parameters.renderWeb.getDataClient(); } public void updateZ() diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/StackClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/StackClient.java index 33e157835..ffbdc8450 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/StackClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/StackClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.util.ArrayList; @@ -10,6 +11,8 @@ import org.janelia.alignment.spec.stack.StackId; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.stack.StackVersion; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,64 +27,120 @@ public class StackClient { public enum Action { CREATE, CLONE, RENAME, SET_STATE, DELETE } - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { - - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - - @Parameter(names = "--stack", description = "Stack name", required = true) - private String stack; - - @Parameter(names = "--action", description = "Management action to perform", required = true) - private Action action; - - @Parameter(names = "--stackState", description = "New state for stack", required = false) - private StackState stackState; - - @Parameter(names = "--versionNotes", description = "Notes about the version being created", required = false) - private String versionNotes; - - @Parameter(names = "--cycleNumber", description = "Processing cycle number", required = false) - private Integer cycleNumber; - - @Parameter(names = "--cycleStepNumber", description = "Processing cycle step number", required = false) - private Integer cycleStepNumber; - - @Parameter(names = "--stackResolutionX", description = "X resoution (in nanometers) for the stack", required = false) - private Double stackResolutionX; - - @Parameter(names = "--stackResolutionY", description = "Y resoution (in nanometers) for the stack", required = false) - private Double stackResolutionY; - - @Parameter(names = "--stackResolutionZ", description = "Z resoution (in nanometers) for the stack", required = false) - private Double stackResolutionZ; - - @Parameter(names = "--materializedBoxRootPath", description = "Root path for materialized boxes", required = false) - private String materializedBoxRootPath; - - @Parameter(names = "--cloneResultProject", description = "Name of project for stack created by clone operation (default is to use source project)", required = false) - private String cloneResultProject; - - @Parameter(names = "--cloneResultStack", description = "Name of stack created by clone operation", required = false) - private String cloneResultStack; - - @Parameter(names = "--renamedOwner", description = "Name of renamed stack owner (default is to use source owner)", required = false) - private String renamedOwner; - - @Parameter(names = "--renamedProject", description = "Name of renamed stack project (default is to use source project)", required = false) - private String renamedProject; - - @Parameter(names = "--renamedStack", description = "Name of renamed stack", required = false) - private String renamedStack; - - @Parameter(names = "--sectionId", description = "The sectionId to delete", required = false) - private String sectionId; - - @Parameter(names = "--skipSharedTransformClone", description = "Only clone tiles, skipping clone of shared transforms (default is false)", required = false, arity = 0) - private Boolean skipSharedTransformClone; - - @Parameter(names = "--zValues", description = "Z values for filtering", required = false, variableArity = true) - private List zValues; + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @Parameter( + names = "--stack", + description = "Stack name", + required = true) + public String stack; + + @Parameter( + names = "--action", + description = "Management action to perform", + required = true) + public Action action; + + @Parameter( + names = "--stackState", + description = "New state for stack", + required = false) + public StackState stackState; + + @Parameter( + names = "--versionNotes", + description = "Notes about the version being created", + required = false) + public String versionNotes; + + @Parameter( + names = "--cycleNumber", + description = "Processing cycle number", + required = false) + public Integer cycleNumber; + + @Parameter( + names = "--cycleStepNumber", + description = "Processing cycle step number", + required = false) + public Integer cycleStepNumber; + + @Parameter( + names = "--stackResolutionX", + description = "X resoution (in nanometers) for the stack", + required = false) + public Double stackResolutionX; + + @Parameter( + names = "--stackResolutionY", + description = "Y resoution (in nanometers) for the stack", + required = false) + public Double stackResolutionY; + + @Parameter( + names = "--stackResolutionZ", + description = "Z resoution (in nanometers) for the stack", + required = false) + public Double stackResolutionZ; + + @Parameter( + names = "--materializedBoxRootPath", + description = "Root path for materialized boxes", + required = false) + public String materializedBoxRootPath; + + @Parameter( + names = "--cloneResultProject", + description = "Name of project for stack created by clone operation (default is to use source project)", + required = false) + public String cloneResultProject; + + @Parameter( + names = "--cloneResultStack", + description = "Name of stack created by clone operation", + required = false) + public String cloneResultStack; + + @Parameter( + names = "--renamedOwner", + description = "Name of renamed stack owner (default is to use source owner)", + required = false) + public String renamedOwner; + + @Parameter( + names = "--renamedProject", + description = "Name of renamed stack project (default is to use source project)", + required = false) + public String renamedProject; + + @Parameter( + names = "--renamedStack", + description = "Name of renamed stack", + required = false) + public String renamedStack; + + @Parameter( + names = "--sectionId", + description = "The sectionId to delete", + required = false) + public String sectionId; + + @Parameter( + names = "--skipSharedTransformClone", + description = "Only clone tiles, skipping clone of shared transforms (default is false)", + required = false, + arity = 0) + public Boolean skipSharedTransformClone; + + @Parameter( + names = "--zValues", + description = "Z values for filtering", + required = false, + variableArity = true) + public List zValues; } @@ -94,7 +153,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, StackClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -127,9 +186,7 @@ public StackClient(final Parameters parameters) { this.parameters = parameters; this.stack = parameters.stack; - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.renderDataClient = parameters.renderWeb.getDataClient(); } public void createStackVersion() @@ -194,14 +251,18 @@ public void renameStack() throw new IllegalArgumentException("missing --renamedStack value"); } - final String toOwner = parameters.renamedOwner == null ? parameters.owner : parameters.renamedOwner; - final String toProject = parameters.renamedProject == null ? parameters.project : parameters.renamedProject; + final String toOwner = parameters.renamedOwner == null ? + parameters.renderWeb.owner : parameters.renamedOwner; + final String toProject = parameters.renamedProject == null ? + parameters.renderWeb.project : parameters.renamedProject; final StackId toStackId = new StackId(toOwner, toProject, parameters.renamedStack); renderDataClient.renameStack(stack, toStackId); - final RenderDataClient renamedDataClient = new RenderDataClient(parameters.baseDataUrl, toOwner, toProject); + final RenderDataClient renamedDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, + toOwner, + toProject); logMetaData("renameStack: after rename", renamedDataClient, parameters.renamedStack); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/StorePotentialTilePairsClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/StorePotentialTilePairsClient.java index e13953b55..0d205b336 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/StorePotentialTilePairsClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/StorePotentialTilePairsClient.java @@ -4,10 +4,7 @@ import com.beust.jcommander.ParametersDelegate; import java.io.IOException; -import java.io.Reader; import java.net.URISyntaxException; -import java.nio.file.FileSystems; -import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -16,7 +13,8 @@ import org.janelia.alignment.match.Matches; import org.janelia.alignment.match.OrderedCanvasIdPair; import org.janelia.alignment.match.RenderableCanvasIdPairs; -import org.janelia.render.client.parameters.MatchDataClientParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MatchWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,9 +29,12 @@ public class StorePotentialTilePairsClient { public static class Parameters extends CommandLineParameters { @ParametersDelegate - public MatchDataClientParameters matchClient = new MatchDataClientParameters(); + public MatchWebServiceParameters matchClient = new MatchWebServiceParameters(); - @Parameter(names = "--pairJson", description = "JSON file where potential tile pairs are stored (.json, .gz, or .zip)", required = true) + @Parameter( + names = "--pairJson", + description = "JSON file where potential tile pairs are stored (.json, .gz, or .zip)", + required = true) public List pairJson; } @@ -45,7 +46,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, StorePotentialTilePairsClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -80,7 +81,7 @@ private void storeEmptyMatchesForPairFile(final String pairJsonFileName) LOG.info("storeEmptyMatchesForPairFile: pairJsonFileName is {}", pairJsonFileName); - final RenderableCanvasIdPairs renderableCanvasIdPairs = load(pairJsonFileName); + final RenderableCanvasIdPairs renderableCanvasIdPairs = RenderableCanvasIdPairs.load(pairJsonFileName); final List pairList = renderableCanvasIdPairs.getNeighborPairs(); final List matchesList = new ArrayList<>(pairList.size()); final Matches emptyMatches = new Matches(new double[1][0], new double[1][0], new double[0]); @@ -100,29 +101,5 @@ private void storeEmptyMatchesForPairFile(final String pairJsonFileName) LOG.info("storeEmptyMatchesForPairFile: exit"); } - /** - * @return pairs object loaded from the specified file. - */ - private static RenderableCanvasIdPairs load(final String dataFile) - throws IOException, IllegalArgumentException { - - // TODO: this was cut-and-pasted from spark client module, but should be refactored into render-app for common use - - final RenderableCanvasIdPairs renderableCanvasIdPairs; - - final Path path = FileSystems.getDefault().getPath(dataFile).toAbsolutePath(); - - LOG.info("load: entry, path={}", path); - - try (final Reader reader = FileUtil.DEFAULT_INSTANCE.getExtensionBasedReader(path.toString())) { - renderableCanvasIdPairs = RenderableCanvasIdPairs.fromJson(reader); - } - - LOG.info("load: exit, loaded {} pairs", renderableCanvasIdPairs.size()); - - - return renderableCanvasIdPairs; - } - private static final Logger LOG = LoggerFactory.getLogger(StorePotentialTilePairsClient.class); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/TilePairClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/TilePairClient.java index 0ac92c436..824898078 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/TilePairClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/TilePairClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.File; import java.io.IOException; @@ -27,6 +28,10 @@ import org.janelia.alignment.spec.TileBounds; import org.janelia.alignment.spec.TileBoundsRTree; import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.util.FileUtil; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.LayerBoundsParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,13 +42,13 @@ */ public class TilePairClient { - @SuppressWarnings("ALL") - public static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); @Parameter(names = "--stack", description = "Stack name", required = true) - private String stack; + public String stack; @Parameter(names = "--baseOwner", @@ -64,135 +69,100 @@ public static class Parameters extends RenderDataClientParameters { private String baseStack; @Parameter(names = "--minZ", description = "Minimum Z value for all tiles", required = true) - private Double minZ; + public Double minZ; @Parameter(names = "--maxZ", description = "Maximum Z value for all tiles", required = true) - private Double maxZ; + public Double maxZ; @Parameter( names = "--xyNeighborFactor", description = "Multiply this by max(width, height) of each tile to determine radius for locating neighbor tiles", required = false) - private Double xyNeighborFactor = 0.9; + public Double xyNeighborFactor = 0.9; @Parameter( names = "--explicitRadius", description = "Explit radius in full scale pixels for locating neighbor tiles (if set, will override --xyNeighborFactor)", required = false) - private Double explicitRadius; + public Double explicitRadius; @Parameter( names = "--zNeighborDistance", description = "Look for neighbor tiles with z values less than or equal to this distance from the current tile's z value", required = false) - private Integer zNeighborDistance = 2; + public Integer zNeighborDistance = 2; @Parameter( names = "--excludeCornerNeighbors", description = "Exclude neighbor tiles whose center x and y is outside the source tile's x and y range respectively", required = false, arity = 1) - private boolean excludeCornerNeighbors = true; + public boolean excludeCornerNeighbors = true; @Parameter( names = "--excludeCompletelyObscuredTiles", description = "Exclude tiles that are completely obscured by reacquired tiles", required = false, arity = 1) - private boolean excludeCompletelyObscuredTiles = true; + public boolean excludeCompletelyObscuredTiles = true; @Parameter( names = "--excludeSameLayerNeighbors", description = "Exclude neighbor tiles in the same layer (z) as the source tile", required = false, arity = 1) - private boolean excludeSameLayerNeighbors = false; + public boolean excludeSameLayerNeighbors = false; @Parameter( names = "--excludeSameSectionNeighbors", description = "Exclude neighbor tiles with the same sectionId as the source tile", required = false, arity = 1) - private boolean excludeSameSectionNeighbors = false; + public boolean excludeSameSectionNeighbors = false; @Parameter( names = "--excludePairsInMatchCollection", description = "Name of match collection whose existing pairs should be excluded from the generated list (default is to include all pairs)", required = false) - private String excludePairsInMatchCollection; + public String excludePairsInMatchCollection; @Parameter( names = "--existingMatchOwner", description = "Owner of match collection whose existing pairs should be excluded from the generated list (default is owner)", required = false) - private String existingMatchOwner; + public String existingMatchOwner; @Parameter(names = "--minExistingMatchCount", description = "Minimum number of existing matches to trigger pair exclusion", required = false) - private Integer minExistingMatchCount = 0; + public Integer minExistingMatchCount = 0; @Parameter( names = "--onlyIncludeTilesFromStack", description = "Name of match collection whose existing pairs should be excluded from the generated list (default is to include all pairs)", required = false) - private String onlyIncludeTilesFromStack; + public String onlyIncludeTilesFromStack; @Parameter(names = "--toJson", description = "JSON file where tile pairs are to be stored (.json, .gz, or .zip)", required = true) - private String toJson; + public String toJson; @Parameter(names = "--maxPairsPerFile", description = "Maximum number of pairs to include in each file.", required = false) - private Integer maxPairsPerFile = 100000; + public Integer maxPairsPerFile = 100000; - @Parameter(names = "--minX", description = "Minimum X value for all tiles", required = false) - private Double minX; - - @Parameter(names = "--maxX", description = "Maximum X value for all tiles", required = false) - private Double maxX; - - @Parameter(names = "--minY", description = "Minimum Y value for all tiles", required = false) - private Double minY; - - @Parameter(names = "--maxY", description = "Maximum Y value for all tiles", required = false) - private Double maxY; + @ParametersDelegate + public LayerBoundsParameters bounds = new LayerBoundsParameters(); public Parameters() { } - public Parameters(final String baseDataUrl, - final String owner, - final String project, - final String stack, - final Double minZ, - final Double maxZ, - final Double xyNeighborFactor, - final Integer zNeighborDistance, - final Double minX, - final Double maxX, - final Double minY, - final Double maxY) { - this.baseDataUrl = baseDataUrl; - this.owner = owner; - this.project = project; - this.stack = stack; - this.minZ = minZ; - this.maxZ = maxZ; - this.xyNeighborFactor = xyNeighborFactor; - this.zNeighborDistance = zNeighborDistance; - this.minX = minX; - this.maxX = maxX; - this.minY = minY; - this.maxY = maxY; - } - public String getBaseOwner() { if (baseOwner == null) { - baseOwner = owner; + baseOwner = renderWeb.owner; } return baseOwner; } public String getBaseProject() { if (baseProject == null) { - baseProject = project; + baseProject = renderWeb.project; } return baseProject; } @@ -206,35 +176,11 @@ public String getBaseStack() { public String getExistingMatchOwner() { if (existingMatchOwner == null) { - existingMatchOwner = owner; + existingMatchOwner = renderWeb.owner; } return existingMatchOwner; } - public void validateStackBounds() throws IllegalArgumentException { - - if (minZ > maxZ) { - throw new IllegalArgumentException("minZ (" + minZ + ") is greater than maxX (" + maxZ + ")"); - } - - if ((minX != null) || (maxX != null) || (minY != null) || (maxY != null)) { - - if ((minX == null) || (maxX == null) || (minY == null) || (maxY == null)) { - throw new IllegalArgumentException("since one or more of minX (" + minX + "), maxX (" + maxX + - "), minY (" + minY + "), maxY (" + maxY + - ") is specified, all must be specified"); - } - - if (minX > maxX) { - throw new IllegalArgumentException("minX (" + minX + ") is greater than maxX (" + maxX + ")"); - } - - if (minY > maxY) { - throw new IllegalArgumentException("minY (" + minY + ") is greater than maxY (" + maxY + ")"); - } - } - - } } public static void main(final String[] args) { @@ -244,7 +190,8 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, TilePairClient.class); + parameters.parse(args); + parameters.bounds.validate(); File toFile = new File(parameters.toJson).getAbsoluteFile(); if (! toFile.exists()) { @@ -277,23 +224,19 @@ public void runClient(final String[] args) throws Exception { public TilePairClient(final Parameters parameters) throws IllegalArgumentException { - parameters.validateStackBounds(); - this.parameters = parameters; - this.filterTilesWithBox = (parameters.minX != null); + this.filterTilesWithBox = (parameters.bounds.minX != null); - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.renderDataClient = parameters.renderWeb.getDataClient(); if (parameters.onlyIncludeTilesFromStack == null) { includeClient = null; includeStack = null; } else { includeStack = StackId.fromNameString(parameters.onlyIncludeTilesFromStack, - parameters.owner, - parameters.project); - includeClient = new RenderDataClient(parameters.baseDataUrl, + parameters.renderWeb.owner, + parameters.renderWeb.project); + includeClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, includeStack.getOwner(), includeStack.getProject()); } @@ -316,11 +259,12 @@ public TilePairClient(final Parameters parameters) throws IllegalArgumentExcepti } public String getRenderParametersUrlTemplate() { - final RenderWebServiceUrls urls = new RenderWebServiceUrls(parameters.baseDataUrl, + final RenderWebServiceUrls urls = new RenderWebServiceUrls(parameters.renderWeb.baseDataUrl, parameters.getBaseOwner(), parameters.getBaseProject()); final String currentStackUrlString = urls.getStackUrlString(parameters.getBaseStack()); - final String relativeStackUrlString = currentStackUrlString.substring(parameters.baseDataUrl.length()); + final String relativeStackUrlString = + currentStackUrlString.substring(parameters.renderWeb.baseDataUrl.length()); return RenderableCanvasIdPairs.TEMPLATE_BASE_DATA_URL_TOKEN + relativeStackUrlString + "/tile/" + RenderableCanvasIdPairs. TEMPLATE_ID_TOKEN + "/render-parameters"; } @@ -479,8 +423,8 @@ public TileBoundsRTree buildRTree(final double z) final int unfilteredCount = tileBoundsList.size(); - tileBoundsList = tree.findTilesInBox(parameters.minX, parameters.minY, - parameters.maxX, parameters.maxY); + tileBoundsList = tree.findTilesInBox(parameters.bounds.minX, parameters.bounds.minY, + parameters.bounds.maxX, parameters.bounds.maxY); if (unfilteredCount > tileBoundsList.size()) { @@ -552,7 +496,7 @@ public ExistingMatchHelper(final Parameters clientParameters, sectionIdList.add(sectionData.getSectionId()); } - matchDataClient = new RenderDataClient(clientParameters.baseDataUrl, + matchDataClient = new RenderDataClient(clientParameters.renderWeb.baseDataUrl, clientParameters.getExistingMatchOwner(), clientParameters.excludePairsInMatchCollection); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/TileRemovalClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/TileRemovalClient.java index c27bb892a..e217dd790 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/TileRemovalClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/TileRemovalClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Reader; @@ -12,6 +13,9 @@ import org.janelia.alignment.json.JsonUtils; import org.janelia.alignment.spec.TileBounds; import org.janelia.alignment.spec.TileBoundsRTree; +import org.janelia.alignment.util.FileUtil; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,63 +26,63 @@ */ public class TileRemovalClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); @Parameter( names = "--stack", description = "Stack name", required = true) - private String stack; + public String stack; @Parameter( names = "--tileIdJson", description = "JSON file containing array of tileIds to be removed (.json, .gz, or .zip)", required = false) - private String tileIdJson; + public String tileIdJson; @Parameter( description = "tileIds_to_remove", required = false) - private List tileIdList; + public List tileIdList; @Parameter( names = "--hiddenTilesWithZ", description = "Z value for all hidden tiles to be removed", required = false) - private Double hiddenTilesWithZ; + public Double hiddenTilesWithZ; @Parameter( names = "--keepZ", description = "Z value for all tiles to be kept", required = false) - private Double keepZ; + public Double keepZ; @Parameter( names = "--keepMinX", description = "Minimum X value for all tiles to be kept", required = false) - private Double keepMinX; + public Double keepMinX; @Parameter( names = "--keepMinY", description = "Minimum Y value for all tiles to be kept", required = false) - private Double keepMinY; + public Double keepMinY; @Parameter( names = "--keepMaxX", description = "Maximum X value for all tiles to be kept", required = false) - private Double keepMaxX; + public Double keepMaxX; @Parameter( names = "--keepMaxY", description = "Maximum Y value for all tiles to be kept", required = false) - private Double keepMaxY; + public Double keepMaxY; private boolean isKeepBoxSpecified() { return ((keepZ != null) && (keepMinX != null) && (keepMaxX != null) && @@ -175,13 +179,11 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, TileRemovalClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); - final RenderDataClient renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient renderDataClient = parameters.renderWeb.getDataClient(); renderDataClient.ensureStackIsInLoadingState(parameters.stack, null); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/TransformSectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/TransformSectionClient.java index 377bcce5d..264ea6464 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/TransformSectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/TransformSectionClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.util.List; @@ -9,6 +10,9 @@ import org.janelia.alignment.spec.ResolvedTileSpecCollection; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.validator.TileSpecValidator; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,14 +23,19 @@ */ public class TransformSectionClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParametersWithValidator { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter(names = "--stack", description = "Stack name", required = true) - private String stack; + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); + + @Parameter( + names = "--stack", + description = "Stack name", + required = true) + public String stack; @Parameter( names = "--targetProject", @@ -40,21 +49,36 @@ private static class Parameters extends RenderDataClientParametersWithValidator required = false) private String targetStack; - @Parameter(names = "--transformId", description = "Identifier for tranformation", required = true) - private String transformId; + @Parameter( + names = "--transformId", + description = "Identifier for transformation", + required = true) + public String transformId; - @Parameter(names = "--transformClass", description = "Name of transformation implementation (java) class", required = true) - private String transformClass; + @Parameter( + names = "--transformClass", + description = "Name of transformation implementation (java) class", + required = true) + public String transformClass; // TODO: figure out less hacky way to handle spaces in transform data string - @Parameter(names = "--transformData", description = "Data with which transformation implementation should be initialized (expects values to be separated by ',' instead of ' ')", required = true) - private String transformData; + @Parameter( + names = "--transformData", + description = "Data with which transformation implementation should be initialized (expects values to be separated by ',' instead of ' ')", + required = true) + public String transformData; - @Parameter(names = "--replaceLast", description = "Replace each tile's last transform with this one (default is to append new transform)", required = false, arity = 0) - private boolean replaceLast; + @Parameter( + names = "--replaceLast", + description = "Replace each tile's last transform with this one (default is to append new transform)", + required = false, + arity = 0) + public boolean replaceLast; - @Parameter(description = "Z values", required = true) - private List zValues; + @Parameter( + description = "Z values", + required = true) + public List zValues; public String getTargetStack() { if ((targetStack == null) || (targetStack.trim().length() == 0)) { @@ -70,7 +94,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, TransformSectionClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -102,19 +126,17 @@ public TransformSectionClient(final Parameters parameters) { parameters.transformClass, parameters.transformData.replace(',', ' ')); - this.tileSpecValidator = parameters.getValidatorInstance(); + this.tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); - this.sourceRenderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.sourceRenderDataClient = parameters.renderWeb.getDataClient(); if ((parameters.targetProject == null) || (parameters.targetProject.trim().length() == 0) || - (parameters.targetProject.equals(parameters.project))) { + (parameters.targetProject.equals(parameters.renderWeb.project))) { this.targetRenderDataClient = sourceRenderDataClient; } else { - this.targetRenderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, + this.targetRenderDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, + parameters.renderWeb.owner, parameters.targetProject); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ValidateTilesClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ValidateTilesClient.java index 45f58f971..9a37761ae 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ValidateTilesClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ValidateTilesClient.java @@ -1,11 +1,15 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.util.List; import org.janelia.alignment.spec.ResolvedTileSpecCollection; import org.janelia.alignment.spec.validator.TileSpecValidator; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,17 +20,25 @@ */ public class ValidateTilesClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParametersWithValidator { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter(names = "--stack", description = "Stack name", required = true) - private String stack; + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); - @Parameter(description = "Z values", required = true) - private List zValues; + + @Parameter( + names = "--stack", + description = "Stack name", + required = true) + public String stack; + + @Parameter( + description = "Z values", + required = true) + public List zValues; } public static void main(final String[] args) { @@ -35,7 +47,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, ValidateTilesClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -55,11 +67,9 @@ public void runClient(final String[] args) throws Exception { public ValidateTilesClient(final Parameters parameters) { this.parameters = parameters; - this.tileSpecValidator = parameters.getValidatorInstance(); + this.tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.renderDataClient = parameters.renderWeb.getDataClient(); } public void validateTilesForZ(final Double z) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/WarpTransformClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/WarpTransformClient.java index 96f7e8747..b1efe0bb0 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/WarpTransformClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/WarpTransformClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.util.Collection; import java.util.List; @@ -16,6 +17,10 @@ import org.janelia.alignment.warp.AbstractWarpTransformBuilder; import org.janelia.alignment.warp.MovingLeastSquaresBuilder; import org.janelia.alignment.warp.ThinPlateSplineBuilder; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; +import org.janelia.render.client.parameter.WarpStackParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,29 +31,34 @@ */ public class WarpTransformClient { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParametersWithValidator { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter(names = "--alignStack", description = "Align stack name", required = true) - private String alignStack; + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); - @Parameter(names = "--montageStack", description = "Montage stack name", required = true) - private String montageStack; + @ParametersDelegate + public WarpStackParameters warp = new WarpStackParameters(); - @Parameter(names = "--targetStack", description = "Target stack (tps or mls) name", required = true) - private String targetStack; + @Parameter( + names = "--alpha", + description = "Alpha value for MLS transform", + required = false) + public Double alpha; - @Parameter(names = "--alpha", description = "Alpha value for MLS transform", required = false) - private Double alpha; + @Parameter( + names = "--deriveMLS", + description = "Derive moving least squares transforms instead of thin plate spline transforms", + required = false, + arity = 0) + public boolean deriveMLS; - @Parameter(names = "--deriveMLS", description = "Derive moving least squares transforms instead of thin plate spline transforms", required = false, arity = 0) - private boolean deriveMLS; - - @Parameter(description = "Z values", required = true) - private List zValues; + @Parameter( + description = "Z values", + required = true) + public List zValues; } public static void main(final String[] args) { @@ -57,7 +67,8 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, WarpTransformClient.class); + parameters.parse(args); + parameters.warp.initDefaultValues(parameters.renderWeb); LOG.info("runClient: entry, parameters={}", parameters); @@ -76,20 +87,21 @@ public void runClient(final String[] args) throws Exception { private final Parameters parameters; private final TileSpecValidator tileSpecValidator; - private final RenderDataClient renderDataClient; + private final RenderDataClient montageDataClient; + private final RenderDataClient alignDataClient; + private final RenderDataClient targetDataClient; public WarpTransformClient(final Parameters parameters) { this.parameters = parameters; - this.tileSpecValidator = parameters.getValidatorInstance(); - - this.renderDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + this.tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); + this.montageDataClient = parameters.renderWeb.getDataClient(); + this.alignDataClient = parameters.warp.getAlignDataClient(); + this.targetDataClient = parameters.warp.getTargetDataClient(); } public void setUpDerivedStack() throws Exception { - final StackMetaData montageStackMetaData = renderDataClient.getStackMetaData(parameters.montageStack); - renderDataClient.setupDerivedStack(montageStackMetaData, parameters.targetStack); + final StackMetaData montageStackMetaData = montageDataClient.getStackMetaData(parameters.warp.montageStack); + targetDataClient.setupDerivedStack(montageStackMetaData, parameters.warp.targetStack); } public void generateStackDataForZ(final Double z, @@ -98,8 +110,10 @@ public void generateStackDataForZ(final Double z, LOG.info("generateStackDataForZ: entry, z={}, alpha={}", z, alpha); - final ResolvedTileSpecCollection montageTiles = renderDataClient.getResolvedTiles(parameters.montageStack, z); - final ResolvedTileSpecCollection alignTiles = renderDataClient.getResolvedTiles(parameters.alignStack, z); + final ResolvedTileSpecCollection montageTiles = + montageDataClient.getResolvedTiles(parameters.warp.montageStack, z); + final ResolvedTileSpecCollection alignTiles = + alignDataClient.getResolvedTiles(parameters.warp.alignStack, z); final TransformSpec warpTransformSpec = buildTransform(montageTiles.getTileSpecs(), alignTiles.getTileSpecs(), @@ -125,7 +139,7 @@ public void generateStackDataForZ(final Double z, throw new IllegalStateException("no tiles left to save after filtering invalid tiles"); } - renderDataClient.saveResolvedTiles(montageTiles, parameters.targetStack, z); + targetDataClient.saveResolvedTiles(montageTiles, parameters.warp.targetStack, z); LOG.info("generateStackDataForZ: exit, saved tiles and transforms for {}", z); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/CommandLineParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CommandLineParameters.java similarity index 65% rename from render-ws-java-client/src/main/java/org/janelia/render/client/CommandLineParameters.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CommandLineParameters.java index 6ae38f57b..d559ee056 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/CommandLineParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CommandLineParameters.java @@ -1,4 +1,4 @@ -package org.janelia.render.client; +package org.janelia.render.client.parameter; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -23,9 +23,8 @@ public class CommandLineParameters implements Serializable { @Parameter( names = "--help", description = "Display this note", - help = true, - order = 0) - protected transient boolean help; + help = true) + public transient boolean help; private transient JCommander jCommander; @@ -34,11 +33,16 @@ public CommandLineParameters() { this.jCommander = null; } + public void parse(final String[] args) throws IllegalArgumentException { + parse(args, this.getClass().getEnclosingClass(), true); + } + public void parse(final String[] args, - final Class programClass) throws IllegalArgumentException { + final Class programClass, + final boolean exitOnHelpOrFailure) throws IllegalArgumentException { jCommander = new JCommander(this); - jCommander.setProgramName("java -cp current-ws-standalone.jar " + programClass.getName()); + jCommander.setProgramName("java -cp -standalone.jar " + programClass.getName()); boolean parseFailed = true; try { @@ -53,7 +57,9 @@ public void parse(final String[] args, if (help || parseFailed) { JCommander.getConsole().println(""); jCommander.usage(); - System.exit(1); + if (exitOnHelpOrFailure) { + System.exit(1); + } } } @@ -69,6 +75,17 @@ public String toString() { } } + /** + * Helper (no pun intended) for testing parameter parsing. + * + * @param parameters parameters instance to test. + */ + public static void parseHelp(final CommandLineParameters parameters) { + parameters.parse(new String[] { "--help" }, + parameters.getClass().getEnclosingClass(), + false); + } + private static final Logger LOG = LoggerFactory.getLogger(CommandLineParameters.class); } \ No newline at end of file diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/LayerBoundsParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/LayerBoundsParameters.java new file mode 100644 index 000000000..2411426dd --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/LayerBoundsParameters.java @@ -0,0 +1,59 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; + +import java.io.Serializable; + +/** + * Parameters for specifying layer bounds. + * + * @author Eric Trautman + */ +public class LayerBoundsParameters implements Serializable { + + @Parameter( + names = "--minX", + description = "Minimum X value for all tiles", + required = false) + public Double minX; + + @Parameter( + names = "--maxX", + description = "Maximum X value for all tiles", + required = false) + public Double maxX; + + @Parameter( + names = "--minY", + description = "Minimum Y value for all tiles", + required = false) + public Double minY; + + @Parameter( + names = "--maxY", + description = "Maximum Y value for all tiles", + required = false) + public Double maxY; + + public void validate() throws IllegalArgumentException { + + if ((minX != null) || (maxX != null) || (minY != null) || (maxY != null)) { + + if ((minX == null) || (maxX == null) || (minY == null) || (maxY == null)) { + throw new IllegalArgumentException("since one or more of minX (" + minX + "), maxX (" + maxX + + "), minY (" + minY + "), maxY (" + maxY + + ") is specified, all must be specified"); + } + + if (minX > maxX) { + throw new IllegalArgumentException("minX (" + minX + ") is greater than maxX (" + maxX + ")"); + } + + if (minY > maxY) { + throw new IllegalArgumentException("minY (" + minY + ") is greater than maxY (" + maxY + ")"); + } + } + + } + +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchClipParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchClipParameters.java similarity index 71% rename from render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchClipParameters.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchClipParameters.java index 148a983de..75054d001 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchClipParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchClipParameters.java @@ -1,11 +1,12 @@ -package org.janelia.render.client.parameters; +package org.janelia.render.client.parameter; import com.beust.jcommander.Parameter; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.io.Serializable; /** - * Client parameters for clipping rendered canvases for point match derivation. + * Parameters for clipping rendered canvases during point match derivation. * * @author Eric Trautman */ @@ -14,15 +15,13 @@ public class MatchClipParameters implements Serializable { @Parameter( names = "--clipWidth", description = "Number of full scale pixels to include in rendered clips of LEFT/RIGHT oriented montage tiles", - required = false, - order = 20) + required = false) public Integer clipWidth; @Parameter( names = "--clipHeight", description = "Number of full scale pixels to include in rendered clips of TOP/BOTTOM oriented montage tiles", - required = false, - order = 21) + required = false) public Integer clipHeight; } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchDerivationParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchDerivationParameters.java similarity index 76% rename from render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchDerivationParameters.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchDerivationParameters.java index c86731b8c..7922ed827 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchDerivationParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchDerivationParameters.java @@ -1,4 +1,4 @@ -package org.janelia.render.client.parameters; +package org.janelia.render.client.parameter; import com.beust.jcommander.Parameter; @@ -7,7 +7,7 @@ import org.janelia.alignment.match.ModelType; /** - * Client parameters for point match derivation. + * Parameters for point match derivation. * * @author Eric Trautman */ @@ -16,92 +16,79 @@ public class MatchDerivationParameters implements Serializable { @Parameter( names = "--SIFTfdSize", description = "SIFT feature descriptor size: how many samples per row and column", - required = false, - order = 30) + required = false) public Integer fdSize = 8; @Parameter( names = "--SIFTminScale", description = "SIFT minimum scale: minSize * minScale < size < maxSize * maxScale", - required = false, - order = 31) + required = false) public Double minScale = 0.5; @Parameter( names = "--SIFTmaxScale", description = "SIFT maximum scale: minSize * minScale < size < maxSize * maxScale", - required = false, - order = 32) + required = false) public Double maxScale = 0.85; @Parameter( names = "--SIFTsteps", description = "SIFT steps per scale octave", - required = false, - order = 33) + required = false) public Integer steps = 3; @Parameter( names = "--matchRod", description = "Ratio of distances for matches", - required = false, - order = 40) + required = false) public Float matchRod = 0.92f; @Parameter( names = "--matchModelType", description = "Type of model for match filtering", - required = false, - order = 41) + required = false) public ModelType matchModelType = ModelType.AFFINE; @Parameter( names = "--matchIterations", description = "Match filter iterations", - required = false, - order = 42) + required = false) public Integer matchIterations = 1000; @Parameter( names = "--matchMaxEpsilon", description = "Minimal allowed transfer error for match filtering", - required = false, - order = 43) + required = false) public Float matchMaxEpsilon = 20.0f; @Parameter( names = "--matchMinInlierRatio", description = "Minimal ratio of inliers to candidates for match filtering", - required = false, - order = 44) + required = false) public Float matchMinInlierRatio = 0.0f; @Parameter( names = "--matchMinNumInliers", description = "Minimal absolute number of inliers for match filtering", - required = false, - order = 45) + required = false) public Integer matchMinNumInliers = 4; @Parameter( names = "--matchMaxNumInliers", description = "Maximum number of inliers for match filtering", - required = false, - order = 46) + required = false) public Integer matchMaxNumInliers; @Parameter( names = "--matchMaxTrust", description = "Reject match candidates with a cost larger than maxTrust * median cost", - required = false, - order = 47) + required = false) public Double matchMaxTrust = 3.0; @Parameter( names = { "--maxFeatureCacheGb", "--maxImageCacheGb" }, description = "Maximum number of gigabytes of features (or DMesh images) to cache", - required = false, - order = 50) + required = false) public Integer maxCacheGb = 2; } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchRenderParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchRenderParameters.java similarity index 75% rename from render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchRenderParameters.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchRenderParameters.java index 711a15251..bbd36504c 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchRenderParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchRenderParameters.java @@ -1,11 +1,11 @@ -package org.janelia.render.client.parameters; +package org.janelia.render.client.parameter; import com.beust.jcommander.Parameter; import java.io.Serializable; /** - * Client parameters for rendering canvases for point match derivation. + * Parameters for rendering canvases during point match derivation. * * @author Eric Trautman */ @@ -14,46 +14,40 @@ public class MatchRenderParameters implements Serializable { @Parameter( names = "--renderScale", description = "Render tiles at this scale", - required = false, - order = 10) + required = false) public Double renderScale = 1.0; @Parameter( names = "--renderWithFilter", description = "Render tiles using a filter for intensity correction", required = false, - arity = 1, - order = 11) + arity = 1) public boolean renderWithFilter = true; @Parameter( names = "--renderWithoutMask", description = "Render tiles without a mask", required = false, - arity = 1, - order = 12) + arity = 1) public boolean renderWithoutMask = true; @Parameter( names = "--renderFullScaleWidth", description = "Full scale width for all rendered tiles", - required = false, - order = 13) + required = false) public Integer renderFullScaleWidth; @Parameter( names = "--renderFullScaleHeight", description = "Full scale height for all rendered tiles", - required = false, - order = 14) + required = false) public Integer renderFullScaleHeight; @Parameter( names = "--fillWithNoise", description = "Fill each canvas image with noise before rendering to improve point match derivation", required = false, - arity = 1, - order = 15) + arity = 1) public boolean fillWithNoise = true; } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchDataClientParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchWebServiceParameters.java similarity index 56% rename from render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchDataClientParameters.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchWebServiceParameters.java index c83ccadc5..91fd21a12 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameters/MatchDataClientParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchWebServiceParameters.java @@ -1,4 +1,4 @@ -package org.janelia.render.client.parameters; +package org.janelia.render.client.parameter; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; @@ -6,38 +6,29 @@ import java.io.Serializable; /** - * Base parameters for all match web service clients. + * Parameters for match web service clients. * * @author Eric Trautman */ @Parameters -public class MatchDataClientParameters implements Serializable { +public class MatchWebServiceParameters implements Serializable { @Parameter( names = "--baseDataUrl", description = "Base web service URL for data (e.g. http://host[:port]/render-ws/v1)", - required = true, - order = 1) + required = true) public String baseDataUrl; @Parameter( names = "--owner", description = "Match collection owner", - required = true, - order = 2) + required = true) public String owner; @Parameter( names = "--collection", description = "Match collection name", - required = true, - order = 3) + required = true) public String collection; - public MatchDataClientParameters() { - this.baseDataUrl = null; - this.owner = null; - this.collection = null; - } - } \ No newline at end of file diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MaterializedBoxParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MaterializedBoxParameters.java new file mode 100644 index 000000000..070c13a3b --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MaterializedBoxParameters.java @@ -0,0 +1,131 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; + +import java.io.Serializable; + +import org.janelia.alignment.Utils; + +/** + * Parameters for rendering box images to disk. + * + * @author Eric Trautman + */ +public class MaterializedBoxParameters implements Serializable { + + @Parameter( + names = "--stack", + description = "Stack name", + required = true) + public String stack; + + @Parameter( + names = "--rootDirectory", + description = "Root directory for rendered tiles (e.g. /tier2/flyTEM/nobackup/rendered_boxes)", + required = true) + public String rootDirectory; + + @Parameter( + names = "--width", + description = "Width of each box", + required = true) + public Integer width; + + @Parameter( + names = "--height", + description = "Height of each box", + required = true) + public Integer height; + + @Parameter( + names = "--maxLevel", + description = "Maximum mipmap level to generate", + required = false) + public Integer maxLevel = 0; + + @Parameter( + names = "--format", + description = "Format for rendered boxes", + required = false) + public String format = Utils.PNG_FORMAT; + + @Parameter( + names = "--maxOverviewWidthAndHeight", + description = "Max width and height of layer overview image (omit or set to zero to disable overview generation)", + required = false) + public Integer maxOverviewWidthAndHeight; + + @Parameter( + names = "--skipInterpolation", + description = "skip interpolation (e.g. for DMG data)", + required = false, + arity = 0) + public boolean skipInterpolation = false; + + @Parameter( + names = "--binaryMask", + description = "use binary mask (e.g. for DMG data)", + required = false, + arity = 0) + public boolean binaryMask = false; + + @Parameter( + names = "--label", + description = "Generate single color tile labels instead of actual tile images", + required = false, + arity = 0) + public boolean label = false; + + @Parameter( + names = "--createIGrid", + description = "create an IGrid file", + required = false, + arity = 0) + public boolean createIGrid = false; + + @Parameter( + names = "--forceGeneration", + description = "Regenerate boxes even if they already exist", + required = false, + arity = 0) + public boolean forceGeneration = false; + + @Parameter( + names = "--renderGroup", + description = "Index (1-n) that identifies portion of layer to render (omit if only one job is being used)", + required = false) + public Integer renderGroup; + + @Parameter( + names = "--numberOfRenderGroups", + description = "Total number of parallel jobs being used to render this layer (omit if only one job is being used)", + required = false) + public Integer numberOfRenderGroups; + + public boolean isOverviewNeeded() { + return ((maxOverviewWidthAndHeight != null) && (maxOverviewWidthAndHeight > 0)); + } + + public MaterializedBoxParameters getInstanceForRenderGroup(final int group, + final int numberOfGroups) { + final MaterializedBoxParameters p = new MaterializedBoxParameters(); + + p.stack = this.stack; + p.rootDirectory = this.rootDirectory; + p.width = this.width; + p.height = this.height; + p.maxLevel = this.maxLevel; + p.format = this.format; + p.maxOverviewWidthAndHeight = this.maxOverviewWidthAndHeight; + p.skipInterpolation = this.skipInterpolation; + p.binaryMask = this.binaryMask; + p.label = this.label; + p.createIGrid = this.createIGrid; + p.forceGeneration = this.forceGeneration; + + p.renderGroup = group; + p.numberOfRenderGroups = numberOfGroups; + + return p; + } +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MipmapParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MipmapParameters.java new file mode 100644 index 000000000..781b69cee --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MipmapParameters.java @@ -0,0 +1,90 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; + +import org.janelia.alignment.Utils; +import org.janelia.alignment.spec.stack.MipmapPathBuilder; + +/** + * Parameters for mipmap generation. + * + * @author Eric Trautman + */ +public class MipmapParameters implements Serializable { + + @Parameter( + names = "--stack", + description = "Stack name", + required = true) + public String stack; + + @Parameter( + names = "--rootDirectory", + description = "Root directory for mipmaps (e.g. /nrs/flyTEM/rendered_mipmaps/FAFB00)", + required = true) + public String rootDirectory; + + @Parameter( + names = "--minLevel", + description = "Minimum mipmap level to generate", + required = false) + public Integer minLevel = 1; + + @Parameter( + names = "--maxLevel", + description = "Maximum mipmap level to generate", + required = false) + public Integer maxLevel = 6; + + @Parameter( + names = "--format", + description = "Format for mipmaps (tiff, jpg, png)", + required = false) + public String format = Utils.TIFF_FORMAT; + + @Parameter( + names = "--forceGeneration", + description = "Regenerate mipmaps even if they already exist", + required = false, + arity = 0) + public boolean forceGeneration = false; + + @Parameter( + names = "--renderGroup", + description = "Index (1-n) that identifies portion of layer to render (omit if only one job is being used)", + required = false) + public Integer renderGroup = 1; + + @Parameter( + names = "--numberOfRenderGroups", + description = "Total number of parallel jobs being used to render this layer (omit if only one job is being used)", + required = false) + public Integer numberOfRenderGroups = 1; + + public MipmapPathBuilder getMipmapPathBuilder() + throws IOException { + + final File dir = new File(rootDirectory).getCanonicalFile(); + + if (! dir.exists()) { + throw new IOException("missing root directory " + rootDirectory); + } + + if (! dir.canWrite()) { + throw new IOException("not allowed to write to root directory " + rootDirectory); + } + + String extension = format; + // map 'tiff' format to 'tif' extension so that {@link ij.io.Opener#openURL(String)} method will work. + if (Utils.TIFF_FORMAT.equals(format)) { + extension = "tif"; + } + + return new MipmapPathBuilder(dir.getPath(), maxLevel, extension); + } + +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClientParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/RenderWebServiceParameters.java similarity index 50% rename from render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClientParameters.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/parameter/RenderWebServiceParameters.java index caed3cefd..21ac83f9d 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClientParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/RenderWebServiceParameters.java @@ -1,20 +1,20 @@ -package org.janelia.render.client; +package org.janelia.render.client.parameter; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.fasterxml.jackson.annotation.JsonIgnore; -import java.lang.reflect.Constructor; +import java.io.Serializable; -import org.janelia.alignment.spec.validator.TileSpecValidator; +import org.janelia.render.client.RenderDataClient; /** - * Base parameters for all render web service clients. + * Parameters for render web service clients. * * @author Eric Trautman */ @Parameters -public class RenderDataClientParameters - extends CommandLineParameters { +public class RenderWebServiceParameters implements Serializable { @Parameter( names = "--baseDataUrl", @@ -34,15 +34,8 @@ public class RenderDataClientParameters required = true) public String project; - public RenderDataClientParameters() { - this(null, null, null); - } - - public RenderDataClientParameters(final String baseDataUrl, - final String owner, - final String project) { - this.baseDataUrl = baseDataUrl; - this.owner = owner; - this.project = project; + @JsonIgnore + public RenderDataClient getDataClient() { + return new RenderDataClient(baseDataUrl, owner, project); } } \ No newline at end of file diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClientParametersWithValidator.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/TileSpecValidatorParameters.java similarity index 84% rename from render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClientParametersWithValidator.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/parameter/TileSpecValidatorParameters.java index 56656739a..8a19fc247 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClientParametersWithValidator.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/TileSpecValidatorParameters.java @@ -1,8 +1,9 @@ -package org.janelia.render.client; +package org.janelia.render.client.parameter; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import java.io.Serializable; import java.lang.reflect.Constructor; import org.janelia.alignment.spec.validator.TileSpecValidator; @@ -10,13 +11,12 @@ import org.slf4j.LoggerFactory; /** - * Base parameters for all render web service clients that support tile validation. + * Parameters for specifying a tile validator instance. * * @author Eric Trautman */ @Parameters -public class RenderDataClientParametersWithValidator - extends RenderDataClientParameters { +public class TileSpecValidatorParameters implements Serializable { @Parameter( names = "--validatorClass", @@ -77,12 +77,6 @@ public TileSpecValidator getValidatorInstance() return validatorInstance; } - public RenderDataClientParametersWithValidator() { - super(); - this.validatorClass = null; - this.validatorData = null; - } - - private static final Logger LOG = LoggerFactory.getLogger(RenderDataClientParametersWithValidator.class); + private static final Logger LOG = LoggerFactory.getLogger(TileSpecValidatorParameters.class); } \ No newline at end of file diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/WarpStackParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/WarpStackParameters.java new file mode 100644 index 000000000..09db3dc59 --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/WarpStackParameters.java @@ -0,0 +1,97 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import org.janelia.render.client.RenderDataClient; + +/** + * Parameters for deriving warp transformations. + * + * @author Eric Trautman + */ +public class WarpStackParameters { + + @Parameter( + names = "--stack", + description = "Montage stack name", + required = true) + public String montageStack; + + @Parameter( + names = "--alignOwner", + description = "Name of align stack owner (default is same as montage stack owner)", + required = false) + public String alignOwner; + + @Parameter( + names = "--alignProject", + description = "Name of align stack project (default is same as montage stack project)", + required = false) + public String alignProject; + + @Parameter( + names = "--alignStack", + description = "Align stack name", + required = true) + public String alignStack; + + @Parameter( + names = "--targetOwner", + description = "Name of target stack owner (default is same as montage stack owner)", + required = false) + public String targetOwner; + + @Parameter( + names = "--targetProject", + description = "Name of target stack project (default is same as montage stack project)", + required = false) + public String targetProject; + + @Parameter( + names = "--targetStack", + description = "Target stack name", + required = true) + public String targetStack; + + @JsonIgnore + private String baseDataUrl; + + public void initDefaultValues(final RenderWebServiceParameters renderWeb) { + + this.baseDataUrl = renderWeb.baseDataUrl; + + if (this.alignOwner == null) { + this.alignOwner = renderWeb.owner; + } + if (this.alignProject == null) { + this.alignProject = renderWeb.project; + } + + if (this.targetOwner == null) { + this.targetOwner = renderWeb.owner; + } + if (this.targetProject == null) { + this.targetProject = renderWeb.project; + } + } + + @JsonIgnore + public RenderDataClient getAlignDataClient() throws IllegalStateException { + verifyState(); + return new RenderDataClient(baseDataUrl, alignOwner, alignProject); + } + + @JsonIgnore + public RenderDataClient getTargetDataClient() throws IllegalStateException { + verifyState(); + return new RenderDataClient(baseDataUrl, targetOwner, targetProject); + } + + private void verifyState() { + if (baseDataUrl == null) { + throw new IllegalStateException( + "code error: default values not initialized, need to add call to initDefaultValues"); + } + } +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/ZRangeParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/ZRangeParameters.java new file mode 100644 index 000000000..357df9dee --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/ZRangeParameters.java @@ -0,0 +1,27 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; + +import java.io.Serializable; + +/** + * Parameters for specifying layer z ranges. + * + * @author Eric Trautman + */ +public class ZRangeParameters + implements Serializable { + + @Parameter( + names = "--minZ", + description = "Minimum Z value for layers to be processed", + required = false) + public Double minZ; + + @Parameter( + names = "--maxZ", + description = "Maximum Z value for layers to be processed", + required = false) + public Double maxZ; + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/acquire/client/LowLatencyMontageClientTest.java b/render-ws-java-client/src/test/java/org/janelia/acquire/client/LowLatencyMontageClientTest.java index b76fa3284..10fd379c0 100644 --- a/render-ws-java-client/src/test/java/org/janelia/acquire/client/LowLatencyMontageClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/acquire/client/LowLatencyMontageClientTest.java @@ -11,6 +11,7 @@ import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.stack.StackId; import org.janelia.alignment.spec.stack.StackMetaData; +import org.janelia.render.client.parameter.CommandLineParameters; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -56,6 +57,11 @@ private static void deleteMontageWorkDirectory() FileUtils.deleteDirectory(montageWorkDirectory); } + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new LowLatencyMontageClient.Parameters()); + } + @Test public void testClient() throws Exception { diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/BoxClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/BoxClientTest.java new file mode 100644 index 000000000..fc113adf8 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/BoxClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link BoxClient} class. + * + * @author Eric Trautman + */ +public class BoxClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new BoxClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/BoxRemovalClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/BoxRemovalClientTest.java new file mode 100644 index 000000000..bf20d62b6 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/BoxRemovalClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link BoxRemovalClient} class. + * + * @author Eric Trautman + */ +public class BoxRemovalClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new BoxRemovalClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/CoordinateClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/CoordinateClientTest.java index c2b618846..b7acc5677 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/CoordinateClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/CoordinateClientTest.java @@ -14,6 +14,8 @@ import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.TransformSpec; import org.janelia.alignment.spec.stack.StackVersion; +import org.janelia.alignment.util.FileUtil; +import org.janelia.render.client.parameter.CommandLineParameters; import org.junit.After; import org.junit.Assert; import org.junit.Ignore; @@ -31,10 +33,15 @@ public class CoordinateClientTest { @After public void tearDown() throws Exception { if (targetSwcDirectory != null) { - MipmapClientTest.deleteRecursive(targetSwcDirectory); + FileUtil.deleteRecursive(targetSwcDirectory); } } + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new CoordinateClient.Parameters()); + } + @Test public void testRoundTripMapping() throws Exception { testRoundTripMapping(1); @@ -216,7 +223,7 @@ private void testRoundTripMapping(final int numberOfThreads) worldListOfLists.add(Collections.singletonList(worldCoord)); } - final ResolvedTileSpecCollection tiles = new ResolvedTileSpecCollection(new ArrayList(), + final ResolvedTileSpecCollection tiles = new ResolvedTileSpecCollection(new ArrayList<>(), tileSpecList); final List> localListOfLists = client.worldToLocal(worldListOfLists, tiles); diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/CopyStackClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/CopyStackClientTest.java new file mode 100644 index 000000000..60a7412a4 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/CopyStackClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link CopyStackClient} class. + * + * @author Eric Trautman + */ +public class CopyStackClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new CopyStackClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/FixAutoLoaderScaleClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/FixAutoLoaderScaleClientTest.java new file mode 100644 index 000000000..9a08a2d75 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/FixAutoLoaderScaleClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link FixAutoLoaderScaleClient} class. + * + * @author Eric Trautman + */ +public class FixAutoLoaderScaleClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new FixAutoLoaderScaleClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/ImportJsonClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportJsonClientTest.java new file mode 100644 index 000000000..bb6405651 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportJsonClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link ImportJsonClient} class. + * + * @author Eric Trautman + */ +public class ImportJsonClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new ImportJsonClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/ImportMETClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportMETClientTest.java new file mode 100644 index 000000000..8a759511e --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportMETClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link ImportMETClient} class. + * + * @author Eric Trautman + */ +public class ImportMETClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new ImportMETClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/ImportMatchClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportMatchClientTest.java new file mode 100644 index 000000000..fb9bd90e9 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportMatchClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link ImportMatchClient} class. + * + * @author Eric Trautman + */ +public class ImportMatchClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new ImportMatchClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/ImportTileZValuesClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportTileZValuesClientTest.java new file mode 100644 index 000000000..58b559a29 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportTileZValuesClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link ImportTileZValuesClient} class. + * + * @author Eric Trautman + */ +public class ImportTileZValuesClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new ImportTileZValuesClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/ImportTransformChangesClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportTransformChangesClientTest.java new file mode 100644 index 000000000..23e41d305 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/ImportTransformChangesClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link ImportTransformChangesClient} class. + * + * @author Eric Trautman + */ +public class ImportTransformChangesClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new ImportTransformChangesClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/MipmapClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/MipmapClientTest.java index 9526dc69e..f39d9cab5 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/MipmapClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/MipmapClientTest.java @@ -21,19 +21,18 @@ import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Date; import java.util.Map; import org.janelia.alignment.ImageAndMask; import org.janelia.alignment.spec.ChannelSpec; import org.janelia.alignment.spec.TileSpec; +import org.janelia.alignment.util.FileUtil; +import org.janelia.render.client.parameter.CommandLineParameters; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Tests the {@link MipmapClient} class. @@ -51,17 +50,22 @@ public void setup() throws Exception { @After public void tearDown() throws Exception { - deleteRecursive(mipmapRootDirectory); + FileUtil.deleteRecursive(mipmapRootDirectory); + } + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new MipmapClient.Parameters()); } @Test public void testGenerateMissingMipmapFiles() throws Exception { - MipmapClient.CommonParameters commonParameters = - new MipmapClient.CommonParameters(mipmapRootDirectory.getAbsolutePath(), - 2); - MipmapClient.Parameters parameters = new MipmapClient.Parameters(commonParameters, new ArrayList()); - MipmapClient mipmapClient = new MipmapClient(parameters); + final MipmapClient.Parameters parameters = new MipmapClient.Parameters(); + parameters.mipmap.rootDirectory = mipmapRootDirectory.getAbsolutePath(); + parameters.mipmap.maxLevel = 2; + + MipmapClient mipmapClient = new MipmapClient(parameters.renderWeb, parameters.mipmap); final ImageAndMask sourceImageAndMask = new ImageAndMask("src/test/resources/col0060_row0140_cam0.tif", "src/test/resources/mask.tif"); @@ -117,10 +121,9 @@ public void testGenerateMissingMipmapFiles() throws Exception { final File previouslyGeneratedImageFile = new File(imageAndMask.getImageFilePath()); final long expectedLastModified = previouslyGeneratedImageFile.lastModified(); - commonParameters = new MipmapClient.CommonParameters(mipmapRootDirectory.getAbsolutePath(), - level); - parameters = new MipmapClient.Parameters(commonParameters, new ArrayList()); - mipmapClient = new MipmapClient(parameters); + parameters.mipmap.rootDirectory = mipmapRootDirectory.getAbsolutePath(); + parameters.mipmap.maxLevel = level; + mipmapClient = new MipmapClient(parameters.renderWeb, parameters.mipmap); tileSpec.setMipmapPathBuilder(mipmapClient.getMipmapPathBuilder()); mipmapClient.generateMissingMipmapFiles(tileSpec); @@ -162,28 +165,4 @@ public static File createTestDirectory(final String baseName) return testDirectory; } - public static boolean deleteRecursive(final File file) { - - boolean deleteSuccessful = true; - - if (file.isDirectory()){ - final File[] files = file.listFiles(); - if (files != null) { - for (final File f : files) { - deleteSuccessful = deleteSuccessful && deleteRecursive(f); - } - } - } - - if (file.delete()) { - LOG.info("deleted " + file.getAbsolutePath()); - } else { - LOG.warn("failed to delete " + file.getAbsolutePath()); - deleteSuccessful = false; - } - - return deleteSuccessful; - } - - private static final Logger LOG = LoggerFactory.getLogger(MipmapClientTest.class); } diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/PointMatchClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/PointMatchClientTest.java index 9269b0a3e..0380c690b 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/PointMatchClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/PointMatchClientTest.java @@ -5,7 +5,9 @@ import java.util.Map; import org.janelia.alignment.match.CanvasMatches; +import org.janelia.render.client.parameter.CommandLineParameters; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; /** @@ -16,6 +18,11 @@ public class PointMatchClientTest { @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new PointMatchClient.Parameters()); + } + + @Ignore public void testMultiThreadedClient() throws Exception { testClient(3); @@ -41,7 +48,7 @@ private void testClient(final int numberOfThreads) }; final PointMatchClient.Parameters clientParameters = new PointMatchClient.Parameters(); - clientParameters.parse(args, PointMatchClient.class); + clientParameters.parse(args); final PointMatchClient client = new PointMatchClient(clientParameters); diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/PointMatchOptimizerClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/PointMatchOptimizerClientTest.java new file mode 100644 index 000000000..bae20133c --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/PointMatchOptimizerClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link PointMatchOptimizerClient} class. + * + * @author Eric Trautman + */ +public class PointMatchOptimizerClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new PointMatchOptimizerClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/RenderClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/RenderClientTest.java new file mode 100644 index 000000000..4dfe05484 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/RenderClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link RenderClient} class. + * + * @author Eric Trautman + */ +public class RenderClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new RenderClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/RenderSectionClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/RenderSectionClientTest.java new file mode 100644 index 000000000..ef8586c05 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/RenderSectionClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link RenderSectionClient} class. + * + * @author Eric Trautman + */ +public class RenderSectionClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new RenderSectionClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/RenderDataClientParametersWithValidatorTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/RenderWebServiceParametersWithValidatorTest.java similarity index 83% rename from render-ws-java-client/src/test/java/org/janelia/render/client/RenderDataClientParametersWithValidatorTest.java rename to render-ws-java-client/src/test/java/org/janelia/render/client/RenderWebServiceParametersWithValidatorTest.java index 48ccc6210..66987ddf3 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/RenderDataClientParametersWithValidatorTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/RenderWebServiceParametersWithValidatorTest.java @@ -2,21 +2,22 @@ import org.janelia.alignment.spec.validator.TemTileSpecValidator; import org.janelia.alignment.spec.validator.TileSpecValidator; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.junit.Assert; import org.junit.Test; /** - * Tests the {@link RenderDataClientParametersWithValidator} class. + * Tests the {@link TileSpecValidatorParameters} class. * * @author Eric Trautman */ -public class RenderDataClientParametersWithValidatorTest { +public class RenderWebServiceParametersWithValidatorTest { @Test public void testGetValidator() throws Exception { - final RenderDataClientParametersWithValidator p = new RenderDataClientParametersWithValidator(); + final TileSpecValidatorParameters p = new TileSpecValidatorParameters(); p.validatorClass = "org.janelia.alignment.spec.validator.TemTileSpecValidator"; p.validatorData = "minCoordinate:0,maxCoordinate:100000,minSize:100,maxSize:20000"; diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/SectionUpdateClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/SectionUpdateClientTest.java new file mode 100644 index 000000000..8c5f177a0 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/SectionUpdateClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link SectionUpdateClient} class. + * + * @author Eric Trautman + */ +public class SectionUpdateClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new SectionUpdateClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/StackClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/StackClientTest.java new file mode 100644 index 000000000..4201e33f0 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/StackClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link StackClient} class. + * + * @author Eric Trautman + */ +public class StackClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new StackClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/StorePotentialTilePairsClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/StorePotentialTilePairsClientTest.java new file mode 100644 index 000000000..e0ef26a4e --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/StorePotentialTilePairsClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link StorePotentialTilePairsClient} class. + * + * @author Eric Trautman + */ +public class StorePotentialTilePairsClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new StorePotentialTilePairsClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java index 63538cfd8..460fbb0ef 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java @@ -14,6 +14,7 @@ import org.janelia.alignment.spec.TileBounds; import org.janelia.alignment.spec.TileBoundsRTree; +import org.janelia.render.client.parameter.CommandLineParameters; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -51,6 +52,11 @@ public void tearDown() throws Exception { }); } + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new TilePairClient.Parameters()); + } + @Test public void testMontageDeriveAndSaveSortedNeighborPairs() throws Exception { @@ -108,7 +114,7 @@ private static TilePairClient.Parameters getTestParameters(final int zNeighborDi "--excludeCornerNeighbors false --maxPairsPerFile 3 " + "--zNeighborDistance " + zNeighborDistance + " --toJson " + toJson; - p.parse(argString.split(" "), TilePairClient.class); + p.parse(argString.split(" ")); return p; } diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/TileRemovalClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/TileRemovalClientTest.java new file mode 100644 index 000000000..820a82ac4 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/TileRemovalClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link TileRemovalClient} class. + * + * @author Eric Trautman + */ +public class TileRemovalClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new TileRemovalClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/TransformSectionClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/TransformSectionClientTest.java new file mode 100644 index 000000000..20cd9a239 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/TransformSectionClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link TransformSectionClient} class. + * + * @author Eric Trautman + */ +public class TransformSectionClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new TransformSectionClient.Parameters()); + } + +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/ValidateTilesClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/ValidateTilesClientTest.java new file mode 100644 index 000000000..b73005dff --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/ValidateTilesClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link ValidateTilesClient} class. + * + * @author Eric Trautman + */ +public class ValidateTilesClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new ValidateTilesClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/AddMaskUrlClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/AddMaskUrlClient.java index 4baf4e5aa..14b09417f 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/AddMaskUrlClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/AddMaskUrlClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client.spark; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.awt.image.BufferedImage; import java.io.IOException; @@ -24,7 +25,9 @@ import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,16 +40,19 @@ public class AddMaskUrlClient implements Serializable { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); @Parameter( names = "--stack", description = "Name of source stack", required = true) - private String stack; + public String stack; @Parameter( names = "--targetOwner", @@ -64,42 +70,30 @@ private static class Parameters extends RenderDataClientParameters { names = "--targetStack", description = "Name of target stack", required = false) - private String targetStack; - - @Parameter( - names = "--minZ", - description = "Minimum Z value for sections to be changed", - required = false) - private Double minZ; - - @Parameter( - names = "--maxZ", - description = "Maximum Z value for sections to be changed", - required = false) - private Double maxZ; + public String targetStack; @Parameter( names = "--maskListFile", description = "File containing paths of different sized mask files", required = true) - private String maskListFile; + public String maskListFile; @Parameter( names = "--completeTargetStack", description = "Complete the target stack after adding masks", required = false, arity = 0) - private boolean completeTargetStack = false; + public boolean completeTargetStack = false; public String getTargetOwner() { if (targetOwner == null) { - targetOwner = owner; + targetOwner = renderWeb.owner; } return targetOwner; } public String getTargetProject() { if (targetProject == null) { - targetProject = project; + targetProject = renderWeb.project; } return targetProject; } @@ -139,7 +133,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, AddMaskUrlClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -185,19 +179,17 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); final List zValues = sourceDataClient.getStackZValues(parameters.stack, - parameters.minZ, - parameters.maxZ); + parameters.layerRange.minZ, + parameters.layerRange.maxZ); if (zValues.size() == 0) { throw new IllegalArgumentException("source stack does not contain any matching z values"); } - final RenderDataClient targetDataClient = new RenderDataClient(parameters.baseDataUrl, + final RenderDataClient targetDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, parameters.getTargetOwner(), parameters.getTargetProject()); @@ -209,54 +201,47 @@ public void run() final JavaRDD rddZValues = sparkContext.parallelize(zValues); - final Function addMaskFunction = new Function() { - - final - @Override - public Integer call(final Double z) - throws Exception { - - LogUtilities.setupExecutorLog4j("z " + z); - //get the source client - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); - //get the target client(which can be the same as the source) - final RenderDataClient targetDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.getTargetOwner(), - parameters.getTargetProject()); - - final ResolvedTileSpecCollection sourceCollection = - sourceDataClient.getResolvedTiles(parameters.stack, z); - - boolean updatedAtLeastOneSpec = false; - - ImageAndMask imageAndMask; - ImageAndMask fixedImageAndMask; - String maskUrl; - for (final TileSpec tileSpec : sourceCollection.getTileSpecs()) { - final Map.Entry firstMipmapEntry = tileSpec.getFirstMipmapEntry(); - if (firstMipmapEntry != null) { - imageAndMask = firstMipmapEntry.getValue(); - if (imageAndMask != null) { - maskUrl = getMaskUrl(imageAndMask, maskDataList); - fixedImageAndMask = new ImageAndMask(imageAndMask.getImageUrl(), maskUrl); - fixedImageAndMask.validate(); - for (final ChannelSpec channelSpec : tileSpec.getAllChannels()) { - channelSpec.putMipmap(firstMipmapEntry.getKey(), fixedImageAndMask); - } - updatedAtLeastOneSpec = true; + final Function addMaskFunction = (Function) z -> { + + LogUtilities.setupExecutorLog4j("z " + z); + //get the source client + final RenderDataClient sourceDataClient1 = parameters.renderWeb.getDataClient(); + + //get the target client(which can be the same as the source) + final RenderDataClient targetDataClient1 = new RenderDataClient(parameters.renderWeb.baseDataUrl, + parameters.getTargetOwner(), + parameters.getTargetProject()); + + final ResolvedTileSpecCollection sourceCollection = + sourceDataClient1.getResolvedTiles(parameters.stack, z); + + boolean updatedAtLeastOneSpec = false; + + ImageAndMask imageAndMask; + ImageAndMask fixedImageAndMask; + String maskUrl; + for (final TileSpec tileSpec : sourceCollection.getTileSpecs()) { + final Map.Entry firstMipmapEntry = tileSpec.getFirstMipmapEntry(); + if (firstMipmapEntry != null) { + imageAndMask = firstMipmapEntry.getValue(); + if (imageAndMask != null) { + maskUrl = getMaskUrl(imageAndMask, maskDataList); + fixedImageAndMask = new ImageAndMask(imageAndMask.getImageUrl(), maskUrl); + fixedImageAndMask.validate(); + for (final ChannelSpec channelSpec : tileSpec.getAllChannels()) { + channelSpec.putMipmap(firstMipmapEntry.getKey(), fixedImageAndMask); } + updatedAtLeastOneSpec = true; } } - if (updatedAtLeastOneSpec) { - targetDataClient.saveResolvedTiles(sourceCollection, parameters.getTargetStack(), z); - } else { - LOG.info("no changes necessary for z {}", z); - } - - return sourceCollection.getTileCount(); } + if (updatedAtLeastOneSpec) { + targetDataClient1.saveResolvedTiles(sourceCollection, parameters.getTargetStack(), z); + } else { + LOG.info("no changes necessary for z {}", z); + } + + return sourceCollection.getTileCount(); }; final JavaRDD rddTileCounts = rddZValues.map(addMaskFunction); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/BoxClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/BoxClient.java index 4008ed0f3..aedfba69d 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/BoxClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/BoxClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client.spark; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Serializable; @@ -19,6 +20,10 @@ import org.janelia.render.client.BoxGenerator; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MaterializedBoxParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,35 +35,28 @@ * * @author Eric Trautman */ -public class BoxClient - implements Serializable { +public class BoxClient implements Serializable { - @SuppressWarnings("ALL") - private static class Parameters extends BoxGenerator.Parameters { + public static class Parameters extends CommandLineParameters { - // NOTE: almost everything is defined in BoxGenerator.Parameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter( - names = "--minZ", - description = "Minimum Z value for sections to be rendered", - required = false) - private Double minZ; + @ParametersDelegate + public MaterializedBoxParameters box = new MaterializedBoxParameters(); - @Parameter( - names = "--maxZ", - description = "Maximum Z value for sections to be rendered", - required = false) - private Double maxZ; + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); @Parameter( names = "--z", description = "Explicit z values for sections to be rendered", required = false, variableArity = true) // e.g. --z 20.0 21.0 22.0 - private List zValues; + public List zValues; public Set getZValues() { - return (zValues == null) ? Collections.emptySet() : new HashSet(zValues); + return (zValues == null) ? Collections.emptySet() : new HashSet<>(zValues); } } @@ -69,7 +67,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, BoxClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -98,13 +96,11 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); - final List sectionDataList = sourceDataClient.getStackSectionData(parameters.stack, - parameters.minZ, - parameters.maxZ, + final List sectionDataList = sourceDataClient.getStackSectionData(parameters.box.stack, + parameters.layerRange.minZ, + parameters.layerRange.maxZ, parameters.getZValues()); if (sectionDataList.size() == 0) { throw new IllegalArgumentException("source stack does not contain any matching z values"); @@ -115,17 +111,19 @@ public void run() final LayerDistributor layerDistributor = new LayerDistributor(numberOfCores); final List> batchedZValues = layerDistributor.distribute(sectionDataList); - final List, BoxGenerator.Parameters>> zBatchWithParametersList = + final List, MaterializedBoxParameters>> zBatchWithParametersList = new ArrayList<>(batchedZValues.size()); - BoxGenerator.Parameters lastGroupParameters = parameters; + MaterializedBoxParameters lastGroupParameters = parameters.box; for (final List zBatch : batchedZValues) { - if (parameters.numberOfRenderGroups == null) { - zBatchWithParametersList.add(new Tuple2<>(zBatch, (BoxGenerator.Parameters) parameters)); + if (parameters.box.numberOfRenderGroups == null) { + zBatchWithParametersList.add(new Tuple2<>(zBatch, parameters.box)); } else { - for (int i = 0; i < parameters.numberOfRenderGroups; i++) { - final BoxGenerator.Parameters p = parameters.getInstanceForRenderGroup(i+1, parameters.numberOfRenderGroups); + for (int i = 0; i < parameters.box.numberOfRenderGroups; i++) { + final MaterializedBoxParameters p = + parameters.box.getInstanceForRenderGroup(i + 1, + parameters.box.numberOfRenderGroups); zBatchWithParametersList.add(new Tuple2<>(zBatch, p)); lastGroupParameters = p; } @@ -133,18 +131,19 @@ public void run() } // create the emtpy image file up-front - final BoxGenerator boxGenerator = new BoxGenerator(lastGroupParameters); + final BoxGenerator boxGenerator = new BoxGenerator(parameters.renderWeb, lastGroupParameters); boxGenerator.createEmptyImageFile(); - final JavaRDD, BoxGenerator.Parameters>> rddZValues = sparkContext.parallelize(zBatchWithParametersList); + final JavaRDD, MaterializedBoxParameters>> rddZValues = + sparkContext.parallelize(zBatchWithParametersList); - final Function, BoxGenerator.Parameters>, Integer> generateBoxesFunction = - (Function, BoxGenerator.Parameters>, Integer>) zBatchWithParameters -> { + final Function, MaterializedBoxParameters>, Integer> generateBoxesFunction = + (Function, MaterializedBoxParameters>, Integer>) zBatchWithParameters -> { final List zBatch = zBatchWithParameters._1; - final BoxGenerator.Parameters p = zBatchWithParameters._2; + final MaterializedBoxParameters p = zBatchWithParameters._2; - final BoxGenerator boxGenerator1 = new BoxGenerator(p); + final BoxGenerator boxGenerator1 = new BoxGenerator(parameters.renderWeb, p); for (int i = 0; i < zBatch.size(); i++) { @@ -173,7 +172,7 @@ public void run() } LOG.info("run: collected stats"); - final String renderGroupsName = parameters.numberOfRenderGroups == null ? "layers" : "render groups"; + final String renderGroupsName = parameters.box.numberOfRenderGroups == null ? "layers" : "render groups"; LOG.info("run: generated boxes for {} {}", total, renderGroupsName); sparkContext.stop(); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/BoxRemovalClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/BoxRemovalClient.java index 4b40dd9b0..b6a5c69db 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/BoxRemovalClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/BoxRemovalClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client.spark; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Serializable; @@ -13,7 +14,9 @@ import org.apache.spark.api.java.function.Function; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,45 +30,38 @@ */ public class BoxRemovalClient implements Serializable { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); @Parameter( names = "--stack", description = "Stack name", required = true) - private String stack; + public String stack; @Parameter( names = "--stackDirectory", - description = "Stack directory containing boxes to remove (e.g. /tier2/flyTEM/nobackup/rendered_boxes/FAFB00/v7_align_tps/8192x8192)", required = true) - private String stackDirectory; + description = "Stack directory containing boxes to remove (e.g. /tier2/flyTEM/nobackup/rendered_boxes/FAFB00/v7_align_tps/8192x8192)", + required = true) + public String stackDirectory; @Parameter( names = "--minLevel", description = "Minimum mipmap level to remove", required = false) - private int minLevel = 0; + public int minLevel = 0; @Parameter( names = "--maxLevel", description = "Maximum mipmap level to remove (values > 8 will also delete small overview images)", required = false) - private int maxLevel = 9; - - @Parameter( - names = "--minZ", - description = "Minimum Z value for boxes to be removed", - required = false) - private Double minZ; + public int maxLevel = 9; - @Parameter( - names = "--maxZ", - description = "Maximum Z value for boxes to be removed", - required = false) - private Double maxZ; } public static void main(final String[] args) { @@ -74,7 +70,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, BoxRemovalClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -102,13 +98,11 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); final List zValues = sourceDataClient.getStackZValues(parameters.stack, - parameters.minZ, - parameters.maxZ); + parameters.layerRange.minZ, + parameters.layerRange.maxZ); if (zValues.size() == 0) { throw new IllegalArgumentException("stack does not contain any matching z values"); @@ -117,22 +111,16 @@ public void run() final JavaRDD rddZValues = sparkContext.parallelize(zValues); - final Function generateBoxesFunction = new Function() { + final Function generateBoxesFunction = (Function) z -> { - final - @Override - public Integer call(final Double z) - throws Exception { + LogUtilities.setupExecutorLog4j("z " + z); - LogUtilities.setupExecutorLog4j("z " + z); - - final org.janelia.render.client.BoxRemovalClient boxRemovalClient = - new org.janelia.render.client.BoxRemovalClient(parameters.stackDirectory, - parameters.minLevel, - parameters.maxLevel); - boxRemovalClient.removeBoxesForZ(z); - return 1; - } + final org.janelia.render.client.BoxRemovalClient boxRemovalClient = + new org.janelia.render.client.BoxRemovalClient(parameters.stackDirectory, + parameters.minLevel, + parameters.maxLevel); + boxRemovalClient.removeBoxesForZ(z); + return 1; }; final JavaRDD rddLayerCounts = rddZValues.map(generateBoxesFunction); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/CopyStackClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/CopyStackClient.java index 02a101a2e..22b7c4584 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/CopyStackClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/CopyStackClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client.spark; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Serializable; @@ -28,7 +29,9 @@ import org.janelia.alignment.spec.stack.StackStats; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,16 +42,19 @@ */ public class CopyStackClient implements Serializable { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); @Parameter( names = "--stack", description = "Name of source stack", required = true) - private String stack; + public String stack; @Parameter( names = "--targetOwner", @@ -66,29 +72,17 @@ private static class Parameters extends RenderDataClientParameters { names = "--targetStack", description = "Name of target stack", required = true) - private String targetStack; - - @Parameter( - names = "--minZ", - description = "Minimum Z value for sections to be copied", - required = false) - private Double minZ; - - @Parameter( - names = "--maxZ", - description = "Maximum Z value for sections to be copied", - required = false) - private Double maxZ; + public String targetStack; @Parameter( names = "--z", description = "Explicit z values for sections to be processed", required = false, variableArity = true) // e.g. --z 20.0 21.0 22.0 - private List zValues; + public List zValues; public Set getZValues() { - return (zValues == null) ? Collections.emptySet() : new HashSet(zValues); + return (zValues == null) ? Collections.emptySet() : new HashSet<>(zValues); } @Parameter( @@ -96,25 +90,25 @@ public Set getZValues() { description = "If necessary, translate copied stack so that it's minX and minY are near the origin (default is to copy exact location)", required = false, arity = 0) - private boolean moveToOrigin = false; + public boolean moveToOrigin = false; @Parameter( names = "--excludeTileIdsMissingFromStacks", description = "Name(s) of stack(s) that contain ids of tiles to be included in target stack (assumes owner and project are same as source stack).", variableArity = true, required = false) - private List excludeTileIdsMissingFromStacks; + public List excludeTileIdsMissingFromStacks; public String getTargetOwner() { if (targetOwner == null) { - targetOwner = owner; + targetOwner = renderWeb.owner; } return targetOwner; } public String getTargetProject() { if (targetProject == null) { - targetProject = project; + targetProject = renderWeb.project; } return targetProject; } @@ -127,7 +121,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, CopyStackClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -156,14 +150,13 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); - final List sectionDataList = sourceDataClient.getStackSectionData(parameters.stack, - parameters.minZ, - parameters.maxZ, - parameters.getZValues()); + final List sectionDataList = + sourceDataClient.getStackSectionData(parameters.stack, + parameters.layerRange.minZ, + parameters.layerRange.maxZ, + parameters.getZValues()); if (sectionDataList.size() == 0) { throw new IllegalArgumentException("source stack does not contain any matching z values"); } @@ -173,7 +166,7 @@ public void run() final LayerDistributor layerDistributor = new LayerDistributor(numberOfCores); final List> batchedZValues = layerDistributor.distribute(sectionDataList); - final RenderDataClient targetDataClient = new RenderDataClient(parameters.baseDataUrl, + final RenderDataClient targetDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, parameters.getTargetOwner(), parameters.getTargetProject()); @@ -211,11 +204,9 @@ public void run() final Function, Long> copyFunction = (Function, Long>) zBatch -> { - final RenderDataClient localSourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient localSourceDataClient = parameters.renderWeb.getDataClient(); - final RenderDataClient localTargetDataClient = new RenderDataClient(parameters.baseDataUrl, + final RenderDataClient localTargetDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, parameters.getTargetOwner(), parameters.getTargetProject()); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/DMeshPointMatchClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/DMeshPointMatchClient.java index febddc43d..682d12c21 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/DMeshPointMatchClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/DMeshPointMatchClient.java @@ -28,10 +28,10 @@ import org.janelia.alignment.match.OrderedCanvasIdPair; import org.janelia.alignment.match.RenderableCanvasIdPairs; import org.janelia.render.client.ClientRunner; -import org.janelia.render.client.CommandLineParameters; -import org.janelia.render.client.parameters.MatchDataClientParameters; -import org.janelia.render.client.parameters.MatchDerivationParameters; -import org.janelia.render.client.parameters.MatchRenderParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MatchDerivationParameters; +import org.janelia.render.client.parameter.MatchRenderParameters; +import org.janelia.render.client.parameter.MatchWebServiceParameters; import org.janelia.render.client.spark.cache.CanvasDataCache; import org.janelia.render.client.spark.cache.CanvasFileLoader; import org.slf4j.Logger; @@ -48,7 +48,7 @@ public class DMeshPointMatchClient public static class Parameters extends CommandLineParameters { @ParametersDelegate - public MatchDataClientParameters matchClient = new MatchDataClientParameters(); + public MatchWebServiceParameters matchClient = new MatchWebServiceParameters(); @ParametersDelegate public MatchRenderParameters matchRender = new MatchRenderParameters(); @@ -113,7 +113,7 @@ public void runClient(final String[] args) throws Exception { parameters.matchRender.renderWithFilter = false; parameters.match.maxCacheGb = 20; - parameters.parse(args, DMeshPointMatchClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -144,7 +144,7 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); final RenderableCanvasIdPairs renderableCanvasIdPairs = - RenderableCanvasIdPairsUtilities.load(parameters.pairJson); + RenderableCanvasIdPairs.load(parameters.pairJson); final String renderParametersUrlTemplateForRun = RenderableCanvasIdPairsUtilities.getRenderParametersUrlTemplateForRun( diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/FixMipmapUrlClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/FixMipmapUrlClient.java index 6581afad0..c14ea2fb6 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/FixMipmapUrlClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/FixMipmapUrlClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client.spark; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Serializable; @@ -24,7 +25,9 @@ import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,18 +39,21 @@ public class FixMipmapUrlClient implements Serializable { - private enum UrlType { IMAGE, MASK, BOTH } + public enum UrlType { IMAGE, MASK, BOTH } - @SuppressWarnings("unused") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); @Parameter( names = "--stack", description = "Name of source stack", required = true) - private String stack; + public String stack; @Parameter( names = "--targetOwner", @@ -67,40 +73,28 @@ private static class Parameters extends RenderDataClientParameters { required = false) private String targetStack; - @Parameter( - names = "--minZ", - description = "Minimum Z value for sections to be fixed", - required = false) - private Double minZ; - - @Parameter( - names = "--maxZ", - description = "Maximum Z value for sections to be fixed", - required = false) - private Double maxZ; - @Parameter( names = "--replacementDataFile", description = "File containing tab separated patterns and replacement values", required = true) - private String replacementDataFile; + public String replacementDataFile; @Parameter( names = "--urlType", description = "Identifies which URLs to fix", required = false) - private final UrlType urlType = UrlType.BOTH; + public UrlType urlType = UrlType.BOTH; public String getTargetOwner() { if (targetOwner == null) { - targetOwner = owner; + targetOwner = renderWeb.owner; } return targetOwner; } public String getTargetProject() { if (targetProject == null) { - targetProject = project; + targetProject = renderWeb.project; } return targetProject; } @@ -120,7 +114,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, FixMipmapUrlClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -171,19 +165,17 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); final List zValues = sourceDataClient.getStackZValues(parameters.stack, - parameters.minZ, - parameters.maxZ); + parameters.layerRange.minZ, + parameters.layerRange.maxZ); if (zValues.size() == 0) { throw new IllegalArgumentException("source stack does not contain any matching z values"); } - final RenderDataClient targetDataClient = new RenderDataClient(parameters.baseDataUrl, + final RenderDataClient targetDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, parameters.getTargetOwner(), parameters.getTargetProject()); @@ -194,95 +186,88 @@ public void run() final JavaRDD rddZValues = sparkContext.parallelize(zValues); - final Function transformFunction = new Function() { - - final - @Override - public Integer call(final Double z) - throws Exception { - - LogUtilities.setupExecutorLog4j("z " + z); - //get the source client - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); - //get the target client(which can be the same as the source) - final RenderDataClient targetDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.getTargetOwner(), - parameters.getTargetProject()); - - final ResolvedTileSpecCollection sourceCollection = - sourceDataClient.getResolvedTiles(parameters.stack, z); - - final boolean fixImage = UrlType.BOTH.equals(parameters.urlType) || - UrlType.IMAGE.equals(parameters.urlType); - final boolean fixMask = UrlType.BOTH.equals(parameters.urlType) || - UrlType.MASK.equals(parameters.urlType); - - boolean fixedAtLeastOneSpec = false; - ImageAndMask imageAndMask; - ImageAndMask fixedImageAndMask; - String imageUrl; - String maskUrl; - for (final TileSpec tileSpec : sourceCollection.getTileSpecs()) { - - for (final ChannelSpec channelSpec : tileSpec.getAllChannels()) { - - final Map.Entry maxEntry = channelSpec.getFloorMipmapEntry(Integer.MAX_VALUE); - if (maxEntry != null) { - for (int level = maxEntry.getKey(); level >= 0; level--) { - imageAndMask = channelSpec.getMipmap(level); - if (imageAndMask != null) { - - if (fixImage) { - imageUrl = imageAndMask.getImageUrl(); - if ((imageUrl != null) && (imageUrl.length() > 0)) { - for (final Pattern p : replacementData.keySet()) { - imageUrl = fixUrl(p, imageUrl, replacementData.get(p)); - } + final Function transformFunction = (Function) z -> { + + LogUtilities.setupExecutorLog4j("z " + z); + //get the source client + final RenderDataClient sourceDataClient1 = parameters.renderWeb.getDataClient(); + + //get the target client(which can be the same as the source) + final RenderDataClient targetDataClient1 = new RenderDataClient(parameters.renderWeb.baseDataUrl, + parameters.getTargetOwner(), + parameters.getTargetProject()); + + final ResolvedTileSpecCollection sourceCollection = + sourceDataClient1.getResolvedTiles(parameters.stack, z); + + final boolean fixImage = UrlType.BOTH.equals(parameters.urlType) || + UrlType.IMAGE.equals(parameters.urlType); + final boolean fixMask = UrlType.BOTH.equals(parameters.urlType) || + UrlType.MASK.equals(parameters.urlType); + + boolean fixedAtLeastOneSpec = false; + ImageAndMask imageAndMask; + ImageAndMask fixedImageAndMask; + String imageUrl; + String maskUrl; + for (final TileSpec tileSpec : sourceCollection.getTileSpecs()) { + + for (final ChannelSpec channelSpec : tileSpec.getAllChannels()) { + + final Map.Entry maxEntry = channelSpec.getFloorMipmapEntry(Integer.MAX_VALUE); + if (maxEntry != null) { + for (int level = maxEntry.getKey(); level >= 0; level--) { + imageAndMask = channelSpec.getMipmap(level); + if (imageAndMask != null) { + + if (fixImage) { + imageUrl = imageAndMask.getImageUrl(); + if ((imageUrl != null) && (imageUrl.length() > 0)) { + for (final Pattern p : replacementData.keySet()) { + imageUrl = fixUrl(p, imageUrl, replacementData.get(p)); } - } else { - imageUrl = imageAndMask.getImageUrl(); } + } else { + imageUrl = imageAndMask.getImageUrl(); + } - if (fixMask) { - maskUrl = imageAndMask.getMaskUrl(); - if ((maskUrl != null) && (maskUrl.length() > 0)) { - for (final Pattern p : replacementData.keySet()) { - maskUrl = fixUrl(p, maskUrl, replacementData.get(p)); - } + if (fixMask) { + maskUrl = imageAndMask.getMaskUrl(); + if ((maskUrl != null) && (maskUrl.length() > 0)) { + for (final Pattern p : replacementData.keySet()) { + maskUrl = fixUrl(p, maskUrl, replacementData.get(p)); } - } else { - maskUrl = imageAndMask.getMaskUrl(); } + } else { + maskUrl = imageAndMask.getMaskUrl(); + } - fixedImageAndMask = new ImageAndMask(imageUrl, maskUrl); - fixedImageAndMask.validate(); + fixedImageAndMask = new ImageAndMask(imageUrl, maskUrl); + fixedImageAndMask.validate(); - final boolean imagePathChanged = fixImage && (imageUrl != null) && - (! imageUrl.equals(imageAndMask.getImageUrl())); - final boolean maskPathChanged = fixMask && (maskUrl != null) && - (! maskUrl.equals(imageAndMask.getMaskUrl())); - if (imagePathChanged || maskPathChanged) { - fixedAtLeastOneSpec = true; - channelSpec.putMipmap(level, fixedImageAndMask); - } + final boolean imagePathChanged = fixImage && (imageUrl != null) && + (! imageUrl.equals(imageAndMask.getImageUrl())); + final boolean maskPathChanged = fixMask && (maskUrl != null) && + (! maskUrl.equals(imageAndMask.getMaskUrl())); + if (imagePathChanged || maskPathChanged) { + fixedAtLeastOneSpec = true; + channelSpec.putMipmap(level, fixedImageAndMask); } + } - } } } - } - if (fixedAtLeastOneSpec) { - targetDataClient.saveResolvedTiles(sourceCollection, parameters.getTargetStack(), z); - } else { - LOG.info("no changes necessary for z {}", z); - } + } - return sourceCollection.getTileCount(); + if (fixedAtLeastOneSpec) { + targetDataClient1.saveResolvedTiles(sourceCollection, parameters.getTargetStack(), z); + } else { + LOG.info("no changes necessary for z {}", z); } + + return sourceCollection.getTileCount(); }; // assign a transformation to the RDD diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/MipmapClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/MipmapClient.java index 523e0663c..d2c922d57 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/MipmapClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/MipmapClient.java @@ -1,11 +1,10 @@ package org.janelia.render.client.spark; -import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Serializable; import java.net.URISyntaxException; -import java.util.Collections; import java.util.List; import org.apache.spark.SparkConf; @@ -14,6 +13,10 @@ import org.apache.spark.api.java.function.Function; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MipmapParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,22 +29,16 @@ public class MipmapClient implements Serializable { - @SuppressWarnings("ALL") - private static class Parameters extends org.janelia.render.client.MipmapClient.CommonParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: most parameters defined in MipmapClient.CommonParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter( - names = "--minZ", - description = "Minimum Z value for mipmap tiles", - required = false) - private Double minZ; + @ParametersDelegate + public MipmapParameters mipmap = new MipmapParameters(); - @Parameter( - names = "--maxZ", - description = "Maximum Z value for mipmap tiles", - required = false) - private Double maxZ; + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); } @@ -51,7 +48,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, MipmapClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -80,13 +77,11 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); - final List zValues = sourceDataClient.getStackZValues(parameters.stack, - parameters.minZ, - parameters.maxZ); + final List zValues = sourceDataClient.getStackZValues(parameters.mipmap.stack, + parameters.layerRange.minZ, + parameters.layerRange.maxZ); if (zValues.size() == 0) { throw new IllegalArgumentException("source stack does not contain any matching z values"); @@ -94,17 +89,12 @@ public void run() final JavaRDD rddZValues = sparkContext.parallelize(zValues); - final Function mipmapFunction = new Function() { - - final - @Override - public Integer call(final Double z) - throws Exception { - - LogUtilities.setupExecutorLog4j("z " + z); - final org.janelia.render.client.MipmapClient mc = getCoreMipmapClient(parameters, z); - return mc.generateMipmapsForZ(z); - } + final Function mipmapFunction = (Function) z -> { + LogUtilities.setupExecutorLog4j("z " + z); + final org.janelia.render.client.MipmapClient mc = + new org.janelia.render.client.MipmapClient(parameters.renderWeb, + parameters.mipmap); + return mc.generateMipmapsForZ(z); }; final JavaRDD rddTileCounts = rddZValues.map(mipmapFunction); @@ -118,21 +108,13 @@ public Integer call(final Double z) LOG.info("run: collected stats"); LOG.info("run: generated mipmaps for {} tiles", total); - final org.janelia.render.client.MipmapClient mc = getCoreMipmapClient(parameters, null); + final org.janelia.render.client.MipmapClient mc = + new org.janelia.render.client.MipmapClient(parameters.renderWeb, + parameters.mipmap); mc.updateMipmapPathBuilderForStack(); sparkContext.stop(); } - private static org.janelia.render.client.MipmapClient getCoreMipmapClient(final Parameters parameters, - final Double z) - throws IOException { - - final org.janelia.render.client.MipmapClient.Parameters p = - new org.janelia.render.client.MipmapClient.Parameters(parameters, - Collections.singletonList(z)); - return new org.janelia.render.client.MipmapClient(p); - } - private static final Logger LOG = LoggerFactory.getLogger(MipmapClient.class); } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/RenderableCanvasIdPairsUtilities.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/RenderableCanvasIdPairsUtilities.java index e81b30059..a2d7a4067 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/RenderableCanvasIdPairsUtilities.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/RenderableCanvasIdPairsUtilities.java @@ -1,17 +1,12 @@ package org.janelia.render.client.spark; -import java.io.IOException; -import java.io.Reader; import java.net.URISyntaxException; -import java.nio.file.FileSystems; -import java.nio.file.Path; import javax.annotation.Nonnull; import org.apache.http.client.utils.URIBuilder; import org.janelia.alignment.match.CanvasId; import org.janelia.alignment.match.RenderableCanvasIdPairs; -import org.janelia.render.client.FileUtil; import org.janelia.render.client.spark.cache.CachedCanvasData; import org.janelia.render.client.spark.cache.CanvasDataLoader; import org.slf4j.LoggerFactory; @@ -23,28 +18,6 @@ */ public class RenderableCanvasIdPairsUtilities { - /** - * @return pairs object loaded from the specified file. - */ - public static RenderableCanvasIdPairs load(final String dataFile) - throws IOException, IllegalArgumentException { - - final RenderableCanvasIdPairs renderableCanvasIdPairs; - - final Path path = FileSystems.getDefault().getPath(dataFile).toAbsolutePath(); - - LOG.info("load: entry, path={}", path); - - try (final Reader reader = FileUtil.DEFAULT_INSTANCE.getExtensionBasedReader(path.toString())) { - renderableCanvasIdPairs = RenderableCanvasIdPairs.fromJson(reader); - } - - LOG.info("load: exit, loaded {} pairs", renderableCanvasIdPairs.size()); - - - return renderableCanvasIdPairs; - } - /** * Looks the generic template from the specified pairs object and returns * a template containing specifics for a run. diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/SIFTPointMatchClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/SIFTPointMatchClient.java index 4f54cd8fe..ced9bfd25 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/SIFTPointMatchClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/SIFTPointMatchClient.java @@ -27,11 +27,11 @@ import org.janelia.alignment.match.OrderedCanvasIdPair; import org.janelia.alignment.match.RenderableCanvasIdPairs; import org.janelia.render.client.ClientRunner; -import org.janelia.render.client.CommandLineParameters; -import org.janelia.render.client.parameters.MatchClipParameters; -import org.janelia.render.client.parameters.MatchDataClientParameters; -import org.janelia.render.client.parameters.MatchDerivationParameters; -import org.janelia.render.client.parameters.MatchRenderParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MatchClipParameters; +import org.janelia.render.client.parameter.MatchDerivationParameters; +import org.janelia.render.client.parameter.MatchRenderParameters; +import org.janelia.render.client.parameter.MatchWebServiceParameters; import org.janelia.render.client.spark.cache.CanvasDataCache; import org.janelia.render.client.spark.cache.CanvasFeatureListLoader; import org.slf4j.Logger; @@ -48,7 +48,7 @@ public class SIFTPointMatchClient public static class Parameters extends CommandLineParameters { @ParametersDelegate - public MatchDataClientParameters matchClient = new MatchDataClientParameters(); + public MatchWebServiceParameters matchClient = new MatchWebServiceParameters(); @ParametersDelegate public MatchRenderParameters matchRender = new MatchRenderParameters(); @@ -75,7 +75,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, SIFTPointMatchClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -117,8 +117,7 @@ private void generateMatchesForPairFile(final JavaSparkContext sparkContext, LOG.info("generateMatchesForPairFile: pairJsonFileName is {}", pairJsonFileName); - final RenderableCanvasIdPairs renderableCanvasIdPairs = - RenderableCanvasIdPairsUtilities.load(pairJsonFileName); + final RenderableCanvasIdPairs renderableCanvasIdPairs = RenderableCanvasIdPairs.load(pairJsonFileName); generateMatchesForPairs(sparkContext, renderableCanvasIdPairs, parameters.matchClient, @@ -129,7 +128,7 @@ private void generateMatchesForPairFile(final JavaSparkContext sparkContext, public static long generateMatchesForPairs(final JavaSparkContext sparkContext, final RenderableCanvasIdPairs renderableCanvasIdPairs, - final MatchDataClientParameters matchClientParameters, + final MatchWebServiceParameters matchClientParameters, final MatchRenderParameters matchRenderParameters, final MatchDerivationParameters matchParameters, final MatchClipParameters clipParameters) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/ScapeClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/ScapeClient.java index 130a025b5..2729c5c69 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/ScapeClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/ScapeClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client.spark; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import ij.ImagePlus; import ij.ImageStack; @@ -23,6 +24,8 @@ import java.util.Date; import java.util.List; +import mpicbg.ij.util.Util; + import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -34,11 +37,13 @@ import org.janelia.alignment.spec.Bounds; import org.janelia.alignment.spec.SectionData; import org.janelia.alignment.spec.stack.StackMetaData; +import org.janelia.alignment.util.FileUtil; import org.janelia.alignment.util.ImageProcessorCache; import org.janelia.render.client.ClientRunner; -import org.janelia.render.client.FileUtil; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,113 +56,104 @@ public class ScapeClient implements Serializable { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); @Parameter( names = "--stack", description = "Stack name", required = true) - private String stack; + public String stack; @Parameter( names = "--rootDirectory", description = "Root directory for rendered layers (e.g. /groups/flyTEM/flyTEM/rendered_scapes)", required = true) - private String rootDirectory; + public String rootDirectory; @Parameter( names = "--maxImagesPerDirectory", description = "Maximum number of images to render in one directory", required = false) - private Integer maxImagesPerDirectory = 1000; + public Integer maxImagesPerDirectory = 1000; @Parameter( names = "--scale", description = "Scale for each rendered layer", required = false) - private Double scale = 0.02; + public Double scale = 0.02; @Parameter( names = "--zScale", description = "Ratio of z to xy resolution for creating isotropic layer projections (omit to skip projection)", required = false) - private Double zScale; + public Double zScale; @Parameter( names = "--format", description = "Format for rendered boxes", required = false) - private String format = Utils.JPEG_FORMAT; + public String format = Utils.JPEG_FORMAT; @Parameter( names = "--doFilter", description = "Use ad hoc filter to support alignment", required = false) - private boolean doFilter = false; + public boolean doFilter = false; @Parameter( names = "--channels", description = "Specify channel(s) and weights to render (e.g. 'DAPI' or 'DAPI__0.7__TdTomato__0.3')", required = false) - private String channels; + public String channels; @Parameter( names = "--fillWithNoise", description = "Fill image with noise before rendering to improve point match derivation", required = false) - private boolean fillWithNoise = false; + public boolean fillWithNoise = false; @Parameter( names = "--useLayerBounds", description = "Base each scape on layer bounds instead of on stack bounds (e.g. for unaligned data)", required = false, arity = 1) - private boolean useLayerBounds = false; + public boolean useLayerBounds = false; @Parameter( names = "--minX", description = "Left most pixel coordinate in world coordinates. Default is minX of stack (or layer when --useLayerBounds true)", required = false) - private Double minX; + public Double minX; @Parameter( names = "--minY", description = "Top most pixel coordinate in world coordinates. Default is minY of stack (or layer when --useLayerBounds true)", required = false) - private Double minY; + public Double minY; @Parameter( names = "--width", description = "Width in world coordinates. Default is maxX - minX of stack (or layer when --useLayerBounds true)", required = false) - private Double width; + public Double width; @Parameter( names = "--height", description = "Height in world coordinates. Default is maxY - minY of stack (or layer when --useLayerBounds true)", required = false) - private Double height; - - @Parameter( - names = "--minZ", - description = "Minimum Z value for sections to be rendered", - required = false) - private Double minZ; - - @Parameter( - names = "--maxZ", - description = "Maximum Z value for sections to be rendered", - required = false) - private Double maxZ; + public Double height; public File getSectionRootDirectory() { final String scapeDir = "scape_" + new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); final Path sectionRootPath = Paths.get(rootDirectory, - project, + renderWeb.project, stack, scapeDir).toAbsolutePath(); return sectionRootPath.toFile(); @@ -195,7 +191,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, ScapeClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -223,13 +219,11 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); final List sectionDataList = sourceDataClient.getStackSectionData(parameters.stack, - parameters.minZ, - parameters.maxZ); + parameters.layerRange.minZ, + parameters.layerRange.maxZ); // projection process depends upon z ordering, so sort section data results by z ... Collections.sort(sectionDataList, SectionData.Z_COMPARATOR); @@ -250,83 +244,76 @@ public void run() final JavaRDD rddSectionData = sparkContext.parallelize(renderSectionList); - final Function generateScapeFunction = new Function() { + final Function generateScapeFunction = + (Function) renderSection -> { - final - @Override - public Integer call(final RenderSection renderSection) - throws Exception { + final Double z = renderSection.getFirstZ(); + LogUtilities.setupExecutorLog4j("z " + z); - final Double z = renderSection.getFirstZ(); - LogUtilities.setupExecutorLog4j("z " + z); + final RenderDataClient sourceDataClient1 = parameters.renderWeb.getDataClient(); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + // set cache size to 50MB so that masks get cached but most of RAM is left for target image + final int maxCachedPixels = 50 * 1000000; + final ImageProcessorCache imageProcessorCache = + new ImageProcessorCache(maxCachedPixels, false, false); - // set cache size to 50MB so that masks get cached but most of RAM is left for target image - final int maxCachedPixels = 50 * 1000000; - final ImageProcessorCache imageProcessorCache = - new ImageProcessorCache(maxCachedPixels, false, false); + final boolean isProjectionNeeded = renderSection.isProjectionNeeded(); + BufferedImage sectionImage = null; + ImageStack projectedStack = null; - final boolean isProjectionNeeded = renderSection.isProjectionNeeded(); - BufferedImage sectionImage = null; - ImageStack projectedStack = null; + for (final SectionData sectionData : renderSection.getSectionDataList()) { - for (final SectionData sectionData : renderSection.getSectionDataList()) { + final String parametersUrl = + sourceDataClient1.getRenderParametersUrlString(parameters.stack, + sectionData.getMinX(), + sectionData.getMinY(), + sectionData.getZ(), + sectionData.getWidth(), + sectionData.getHeight(), + parameters.scale); - final String parametersUrl = - sourceDataClient.getRenderParametersUrlString(parameters.stack, - sectionData.getMinX(), - sectionData.getMinY(), - sectionData.getZ(), - sectionData.getWidth(), - sectionData.getHeight(), - parameters.scale); + LOG.debug("generateScapeFunction: loading {}", parametersUrl); - LOG.debug("generateScapeFunction: loading {}", parametersUrl); + final RenderParameters renderParameters = RenderParameters.loadFromUrl(parametersUrl); + renderParameters.setDoFilter(parameters.doFilter); + renderParameters.setChannels(parameters.channels); - final RenderParameters renderParameters = RenderParameters.loadFromUrl(parametersUrl); - renderParameters.setDoFilter(parameters.doFilter); - renderParameters.setChannels(parameters.channels); + sectionImage = renderParameters.openTargetImage(); - sectionImage = renderParameters.openTargetImage(); + if (isProjectionNeeded && (projectedStack == null)) { + projectedStack = new ImageStack(sectionImage.getWidth(), sectionImage.getHeight()); + } - if (isProjectionNeeded && (projectedStack == null)) { - projectedStack = new ImageStack(sectionImage.getWidth(), sectionImage.getHeight()); - } - - if (parameters.fillWithNoise) { - final ByteProcessor ip = new ByteProcessor(sectionImage.getWidth(), sectionImage.getHeight()); - mpicbg.ij.util.Util.fillWithNoise(ip); - sectionImage.getGraphics().drawImage(ip.createImage(), 0, 0, null); - } + if (parameters.fillWithNoise) { + final ByteProcessor ip = new ByteProcessor(sectionImage.getWidth(), sectionImage.getHeight()); + Util.fillWithNoise(ip); + sectionImage.getGraphics().drawImage(ip.createImage(), 0, 0, null); + } - ArgbRenderer.render(renderParameters, sectionImage, imageProcessorCache); + ArgbRenderer.render(renderParameters, sectionImage, imageProcessorCache); - if (isProjectionNeeded) { - projectedStack.addSlice(new ColorProcessor(sectionImage).convertToByteProcessor()); + if (isProjectionNeeded) { + projectedStack.addSlice(new ColorProcessor(sectionImage).convertToByteProcessor()); + } } - } - if (projectedStack != null) { + if (projectedStack != null) { - LOG.debug("projecting {} sections", projectedStack.getSize()); + LOG.debug("projecting {} sections", projectedStack.getSize()); - final ZProjector projector = new ZProjector(new ImagePlus("", projectedStack)); - projector.setMethod(ZProjector.AVG_METHOD); - projector.doProjection(); - final ImageProcessor ip = projector.getProjection().getProcessor(); - sectionImage = ip.getBufferedImage(); - } + final ZProjector projector = new ZProjector(new ImagePlus("", projectedStack)); + projector.setMethod(ZProjector.AVG_METHOD); + projector.doProjection(); + final ImageProcessor ip = projector.getProjection().getProcessor(); + sectionImage = ip.getBufferedImage(); + } - final File sectionFile = renderSection.getOutputFile(parameters.format); + final File sectionFile = renderSection.getOutputFile(parameters.format); - Utils.saveImage(sectionImage, sectionFile.getAbsolutePath(), parameters.format, true, 0.85f); + Utils.saveImage(sectionImage, sectionFile.getAbsolutePath(), parameters.format, true, 0.85f); - return 1; - } - }; + return 1; + }; final JavaRDD rddLayerCounts = rddSectionData.map(generateScapeFunction); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/TransformSectionClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/TransformSectionClient.java index 091f926dd..e2ae2e3ee 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/TransformSectionClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/TransformSectionClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client.spark; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Serializable; @@ -16,7 +17,9 @@ import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,10 +30,13 @@ */ public class TransformSectionClient implements Serializable { - @SuppressWarnings("unused") - private static class Parameters extends RenderDataClientParameters { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); @Parameter( names = "--stack", @@ -54,40 +60,28 @@ private static class Parameters extends RenderDataClientParameters { names = "--targetStack", description = "Name of target stack", required = false) - private String targetStack; - - @Parameter( - names = "--minZ", - description = "Minimum Z value for sections to be copied", - required = false) - private Double minZ; - - @Parameter( - names = "--maxZ", - description = "Maximum Z value for sections to be copied", - required = false) - private Double maxZ; + public String targetStack; @Parameter(names = "--transformId", description = "Identifier for tranformation", required = true) - private String transformId; + public String transformId; @Parameter(names = "--transformClass", description = "Name of transformation implementation (java) class", required = true) - private String transformClass; + public String transformClass; // TODO: figure out less hacky way to handle spaces in transform data string @Parameter(names = "--transformData", description = "Data with which transformation implementation should be initialized (expects values to be separated by ',' instead of ' ')", required = true) - private String transformData; + public String transformData; public String getTargetOwner() { if (targetOwner == null) { - targetOwner = owner; + targetOwner = renderWeb.owner; } return targetOwner; } public String getTargetProject() { if (targetProject == null) { - targetProject = project; + targetProject = renderWeb.project; } return targetProject; } @@ -107,7 +101,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, TransformSectionClient.class); + parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); @@ -136,19 +130,17 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); final List zValues = sourceDataClient.getStackZValues(parameters.stack, - parameters.minZ, - parameters.maxZ); + parameters.layerRange.minZ, + parameters.layerRange.maxZ); if (zValues.size() == 0) { throw new IllegalArgumentException("source stack does not contain any matching z values"); } - final RenderDataClient targetDataClient = new RenderDataClient(parameters.baseDataUrl, + final RenderDataClient targetDataClient = new RenderDataClient(parameters.renderWeb.baseDataUrl, parameters.getTargetOwner(), parameters.getTargetProject()); @@ -162,36 +154,29 @@ public void run() // make RDD final JavaRDD rddZValues = sparkContext.parallelize(zValues); - final Function transformFunction = new Function() { + final Function transformFunction = (Function) z -> { - final - @Override - public Integer call(final Double z) - throws Exception { + LogUtilities.setupExecutorLog4j("z " + z); + //get the source client + final RenderDataClient sourceDataClient1 = parameters.renderWeb.getDataClient(); - LogUtilities.setupExecutorLog4j("z " + z); - //get the source client - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); - //get the target client(which can be the same as the source) - final RenderDataClient targetDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.getTargetOwner(), - parameters.getTargetProject()); + //get the target client(which can be the same as the source) + final RenderDataClient targetDataClient1 = new RenderDataClient(parameters.renderWeb.baseDataUrl, + parameters.getTargetOwner(), + parameters.getTargetProject()); - final ResolvedTileSpecCollection sourceCollection = - sourceDataClient.getResolvedTiles(parameters.stack, z); + final ResolvedTileSpecCollection sourceCollection = + sourceDataClient1.getResolvedTiles(parameters.stack, z); - sourceCollection.addTransformSpecToCollection(stackTransform); - sourceCollection.addReferenceTransformToAllTiles(stackTransform.getId(), false); + sourceCollection.addTransformSpecToCollection(stackTransform); + sourceCollection.addReferenceTransformToAllTiles(stackTransform.getId(), false); - //vs tile spec validation? - sourceCollection.removeUnreferencedTransforms(); + //vs tile spec validation? + sourceCollection.removeUnreferencedTransforms(); - targetDataClient.saveResolvedTiles(sourceCollection, parameters.getTargetStack(), z); + targetDataClient1.saveResolvedTiles(sourceCollection, parameters.getTargetStack(), z); - return sourceCollection.getTileCount(); - } + return sourceCollection.getTileCount(); }; // assign a transformation to the RDD diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/WarpTransformClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/WarpTransformClient.java index 586cd00da..f68dcc219 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/WarpTransformClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/WarpTransformClient.java @@ -1,6 +1,7 @@ package org.janelia.render.client.spark; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; import java.io.IOException; import java.io.Serializable; @@ -28,7 +29,11 @@ import org.janelia.alignment.warp.ThinPlateSplineBuilder; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.RenderDataClientParametersWithValidator; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; +import org.janelia.render.client.parameter.WarpStackParameters; +import org.janelia.render.client.parameter.ZRangeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,91 +45,29 @@ public class WarpTransformClient implements Serializable { - @SuppressWarnings("ALL") - private static class Parameters extends RenderDataClientParametersWithValidator { + public static class Parameters extends CommandLineParameters { - // NOTE: --baseDataUrl, --owner, and --project parameters defined in RenderDataClientParameters - // NOTE: --validatorClass and --validatorData parameters defined in RenderDataClientParametersWithValidator + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @Parameter( - names = "--stack", - description = "Montage stack name", - required = true) - private String montageStack; - - @Parameter( - names = "--alignOwner", - description = "Name of align stack owner (default is same as montage stack owner)", - required = false) - private String alignOwner; - - @Parameter( - names = "--alignProject", - description = "Name of align stack project (default is same as montage stack project)", - required = false) - private String alignProject; - - @Parameter( - names = "--alignStack", - description = "Align stack name", - required = true) - private String alignStack; - - @Parameter( - names = "--targetOwner", - description = "Name of target stack owner (default is same as montage stack owner)", - required = false) - private String targetOwner; - - @Parameter( - names = "--targetProject", - description = "Name of target stack project (default is same as montage stack project)", - required = false) - private String targetProject; - - @Parameter( - names = "--targetStack", - description = "Target stack name", - required = true) - private String targetStack; + @ParametersDelegate + public TileSpecValidatorParameters tileSpecValidator = new TileSpecValidatorParameters(); - @Parameter( - names = "--minZ", - description = "Minimum Z value for sections to be processed", - required = false) - private Double minZ; + @ParametersDelegate + public WarpStackParameters warp = new WarpStackParameters(); - @Parameter( - names = "--maxZ", - description = "Maximum Z value for sections to be processed", - required = false) - private Double maxZ; + @ParametersDelegate + public ZRangeParameters layerRange = new ZRangeParameters(); @Parameter( names = "--z", description = "Explicit z values for sections to be processed", required = false, variableArity = true) // e.g. --z 20.0 21.0 22.0 - private List zValues; + public List zValues; public Set getZValues() { - return (zValues == null) ? Collections.emptySet() : new HashSet(zValues); - } - - public String getAlignOwner() { - return alignOwner == null ? owner : alignOwner; - } - - public String getAlignProject() { - return alignProject == null ? project : alignProject; - } - - public String getTargetOwner() { - return targetOwner == null ? owner : targetOwner; - } - - public String getTargetProject() { - return targetProject == null ? project : targetProject; + return (zValues == null) ? Collections.emptySet() : new HashSet<>(zValues); } } @@ -135,7 +78,8 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); - parameters.parse(args, WarpTransformClient.class); + parameters.parse(args); + parameters.warp.initDefaultValues(parameters.renderWeb); LOG.info("runClient: entry, parameters={}", parameters); @@ -164,13 +108,11 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); + final RenderDataClient sourceDataClient = parameters.renderWeb.getDataClient(); - final List sectionDataList = sourceDataClient.getStackSectionData(parameters.montageStack, - parameters.minZ, - parameters.maxZ, + final List sectionDataList = sourceDataClient.getStackSectionData(parameters.warp.montageStack, + parameters.layerRange.minZ, + parameters.layerRange.maxZ, parameters.getZValues()); if (sectionDataList.size() == 0) { throw new IllegalArgumentException("montage stack does not contain any matching z values"); @@ -181,12 +123,10 @@ public void run() final LayerDistributor layerDistributor = new LayerDistributor(numberOfCores); final List> batchedZValues = layerDistributor.distribute(sectionDataList); - final RenderDataClient targetDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.getTargetOwner(), - parameters.getTargetProject()); + final RenderDataClient targetDataClient = parameters.warp.getTargetDataClient(); - final StackMetaData montageStackMetaData = sourceDataClient.getStackMetaData(parameters.montageStack); - targetDataClient.setupDerivedStack(montageStackMetaData, parameters.targetStack); + final StackMetaData montageStackMetaData = sourceDataClient.getStackMetaData(parameters.warp.montageStack); + targetDataClient.setupDerivedStack(montageStackMetaData, parameters.warp.targetStack); final JavaRDD> rddZValues = sparkContext.parallelize(batchedZValues); @@ -205,24 +145,15 @@ public void run() LOG.info("warpFunction: processing layer {} of {}, remaining layer z values are {}", i + 1, zBatch.size(), zBatch.subList(i+1, zBatch.size())); - final TileSpecValidator tileSpecValidator = parameters.getValidatorInstance(); - - final RenderDataClient montageDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.owner, - parameters.project); - - final RenderDataClient alignDataClient = new RenderDataClient(parameters.baseDataUrl, - parameters.getAlignOwner(), - parameters.getAlignProject()); - - final RenderDataClient targetDataClient1 = new RenderDataClient(parameters.baseDataUrl, - parameters.getTargetOwner(), - parameters.getTargetProject()); + final TileSpecValidator tileSpecValidator = parameters.tileSpecValidator.getValidatorInstance(); + final RenderDataClient montageDataClient = parameters.renderWeb.getDataClient(); + final RenderDataClient alignDataClient = parameters.warp.getAlignDataClient(); + final RenderDataClient targetDataClient1 = parameters.warp.getTargetDataClient(); final ResolvedTileSpecCollection montageTiles = - montageDataClient.getResolvedTiles(parameters.montageStack, z); + montageDataClient.getResolvedTiles(parameters.warp.montageStack, z); final ResolvedTileSpecCollection alignTiles = - alignDataClient.getResolvedTiles(parameters.alignStack, z); + alignDataClient.getResolvedTiles(parameters.warp.alignStack, z); final TransformSpec warpTransformSpec = buildTransform(montageTiles.getTileSpecs(), alignTiles.getTileSpecs(), @@ -247,7 +178,7 @@ public void run() throw new IllegalStateException("no tiles left to save after filtering invalid tiles"); } - targetDataClient1.saveResolvedTiles(montageTiles, parameters.targetStack, z); + targetDataClient1.saveResolvedTiles(montageTiles, parameters.warp.targetStack, z); processedTileCount += montageTiles.getTileCount(); } diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/AddMaskUrlClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/AddMaskUrlClientTest.java new file mode 100644 index 000000000..1ec397bc1 --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/AddMaskUrlClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link AddMaskUrlClient} class. + * + * @author Eric Trautman + */ +public class AddMaskUrlClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new AddMaskUrlClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/BoxClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/BoxClientTest.java new file mode 100644 index 000000000..bab24be0e --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/BoxClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link BoxClient} class. + * + * @author Eric Trautman + */ +public class BoxClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new BoxClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/BoxRemovalClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/BoxRemovalClientTest.java new file mode 100644 index 000000000..f9545b647 --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/BoxRemovalClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link BoxRemovalClient} class. + * + * @author Eric Trautman + */ +public class BoxRemovalClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new BoxRemovalClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/CopyStackClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/CopyStackClientTest.java new file mode 100644 index 000000000..e4c336bd7 --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/CopyStackClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link CopyStackClient} class. + * + * @author Eric Trautman + */ +public class CopyStackClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new CopyStackClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/DMeshPointMatchClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/DMeshPointMatchClientTest.java new file mode 100644 index 000000000..46e9ad5ee --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/DMeshPointMatchClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link DMeshPointMatchClient} class. + * + * @author Eric Trautman + */ +public class DMeshPointMatchClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new DMeshPointMatchClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/FixMipmapUrlClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/FixMipmapUrlClientTest.java new file mode 100644 index 000000000..dcad01b98 --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/FixMipmapUrlClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link FixMipmapUrlClient} class. + * + * @author Eric Trautman + */ +public class FixMipmapUrlClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new FixMipmapUrlClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/MipmapClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/MipmapClientTest.java new file mode 100644 index 000000000..d3e8a6889 --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/MipmapClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link MipmapClient} class. + * + * @author Eric Trautman + */ +public class MipmapClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new MipmapClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/RenderableCanvasIdPairsUtilitiesTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/RenderableCanvasIdPairsUtilitiesTest.java index 144ecd821..2347ee6c4 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/RenderableCanvasIdPairsUtilitiesTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/RenderableCanvasIdPairsUtilitiesTest.java @@ -15,7 +15,7 @@ public class RenderableCanvasIdPairsUtilitiesTest { public void testLoad() throws Exception { final RenderableCanvasIdPairs renderableCanvasIdPairs = - RenderableCanvasIdPairsUtilities.load("src/test/resources/tile_pairs_v12_acquire_merged_1_5.json"); + RenderableCanvasIdPairs.load("src/test/resources/tile_pairs_v12_acquire_merged_1_5.json"); Assert.assertNotNull("pairs not loaded", renderableCanvasIdPairs); diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/ScapeClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/ScapeClientTest.java new file mode 100644 index 000000000..e58f1fbe2 --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/ScapeClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link ScapeClient} class. + * + * @author Eric Trautman + */ +public class ScapeClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new ScapeClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/TransformSectionClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/TransformSectionClientTest.java new file mode 100644 index 000000000..2746bf8c1 --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/TransformSectionClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link TransformSectionClient} class. + * + * @author Eric Trautman + */ +public class TransformSectionClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new TransformSectionClient.Parameters()); + } + +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/WarpTransformClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/WarpTransformClientTest.java new file mode 100644 index 000000000..260589f12 --- /dev/null +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/WarpTransformClientTest.java @@ -0,0 +1,18 @@ +package org.janelia.render.client.spark; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link WarpTransformClient} class. + * + * @author Eric Trautman + */ +public class WarpTransformClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new WarpTransformClient.Parameters()); + } + +}