diff --git a/render-app/src/main/java/org/janelia/alignment/spec/SectionData.java b/render-app/src/main/java/org/janelia/alignment/spec/SectionData.java index 56a4b6573..e064f87e7 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/SectionData.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/SectionData.java @@ -1,6 +1,10 @@ package org.janelia.alignment.spec; +import java.beans.Transient; import java.io.Serializable; +import java.util.Comparator; + +import org.janelia.alignment.json.JsonUtils; /** * Maps a section identifier to its z value. @@ -68,4 +72,35 @@ public Double getMinX() { public Double getMinY() { return minY; } + + @Transient + public int getWidth() { + return (int) (maxX - minX + 0.5); + } + + @Transient + public int getHeight() { + return (int) (maxY - minY + 0.5); + } + + public String toJson() { + return JSON_HELPER.toJson(this); + } + + @Override + public String toString() { + return this.toJson(); + } + + public static final Comparator Z_COMPARATOR = new Comparator() { + @Override + public int compare(final SectionData o1, + final SectionData o2) { + return o1.z.compareTo(o2.z); + } + }; + + private static final JsonUtils.Helper JSON_HELPER = + new JsonUtils.Helper<>(SectionData.class); + } diff --git a/render-app/src/main/java/org/janelia/alignment/spec/stack/StackId.java b/render-app/src/main/java/org/janelia/alignment/spec/stack/StackId.java index b6f7fca57..cc84324e7 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/stack/StackId.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/stack/StackId.java @@ -1,5 +1,7 @@ package org.janelia.alignment.spec.stack; +import com.fasterxml.jackson.annotation.JsonIgnore; + import java.io.Serializable; import org.janelia.alignment.util.CollectionNameUtil; @@ -74,14 +76,17 @@ public int compareTo(final StackId that) { return v; } + @JsonIgnore public String getSectionCollectionName() { return getCollectionName(SECTION_COLLECTION_SUFFIX); } + @JsonIgnore public String getTileCollectionName() { return getCollectionName(TILE_COLLECTION_SUFFIX); } + @JsonIgnore public String getTransformCollectionName() { return getCollectionName(TRANSFORM_COLLECTION_SUFFIX); } diff --git a/render-app/src/main/java/org/janelia/alignment/spec/stack/StackMetaData.java b/render-app/src/main/java/org/janelia/alignment/spec/stack/StackMetaData.java index 4599b00ab..d9ce7061e 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/stack/StackMetaData.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/stack/StackMetaData.java @@ -59,6 +59,10 @@ public boolean isLoading() { return LOADING.equals(state); } + public boolean isOffline() { + return OFFLINE.equals(state); + } + public boolean isReadOnly() { return READ_ONLY.equals(state); } @@ -280,6 +284,15 @@ public static StackMetaData fromJson(final String json) { return JSON_HELPER.fromJson(json); } + public static StackMetaData buildDerivedMetaData(final StackId stackId, + final StackMetaData fromStackMetaData) { + final StackMetaData derivedMetaData = new StackMetaData(stackId, fromStackMetaData.getCurrentVersion()); + derivedMetaData.state = fromStackMetaData.state; + derivedMetaData.currentVersionNumber = fromStackMetaData.currentVersionNumber; + derivedMetaData.stats = fromStackMetaData.stats; + return derivedMetaData; + } + private static final JsonUtils.Helper JSON_HELPER = new JsonUtils.Helper<>(StackMetaData.class); } diff --git a/render-app/src/main/java/org/janelia/alignment/util/ImageProcessorCache.java b/render-app/src/main/java/org/janelia/alignment/util/ImageProcessorCache.java index 433c6ab5e..149b65e9b 100644 --- a/render-app/src/main/java/org/janelia/alignment/util/ImageProcessorCache.java +++ b/render-app/src/main/java/org/janelia/alignment/util/ImageProcessorCache.java @@ -306,7 +306,7 @@ public String getUri() { } public boolean isConvertTo16Bit(){ return convertTo16Bit; - } + } public int getDownSampleLevels() { return downSampleLevels; } diff --git a/render-app/src/main/java/org/janelia/alignment/util/LabelImageProcessorCache.java b/render-app/src/main/java/org/janelia/alignment/util/LabelImageProcessorCache.java index 22a82fd6b..69773934e 100644 --- a/render-app/src/main/java/org/janelia/alignment/util/LabelImageProcessorCache.java +++ b/render-app/src/main/java/org/janelia/alignment/util/LabelImageProcessorCache.java @@ -137,7 +137,7 @@ protected ImageProcessor loadImageProcessor(final String url, ImageProcessor imageProcessor; if (isMask) { - imageProcessor = super.loadImageProcessor(url, downSampleLevels, true, false); + imageProcessor = super.loadImageProcessor(url, downSampleLevels, true, convertTo16Bit); } else { final Color labelColor = getColorForUrl(url); diff --git a/render-app/src/test/java/org/janelia/alignment/TransformMeshTest.java b/render-app/src/test/java/org/janelia/alignment/TransformMeshTest.java index bbe0ce0aa..55a11f4c9 100644 --- a/render-app/src/test/java/org/janelia/alignment/TransformMeshTest.java +++ b/render-app/src/test/java/org/janelia/alignment/TransformMeshTest.java @@ -35,6 +35,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +45,7 @@ * * @author Eric Trautman */ +@Ignore public class TransformMeshTest { // increase this to 10 (or more) to see average times diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/FileUtil.java b/render-ws-java-client/src/main/java/org/janelia/render/client/FileUtil.java index 77c73b58e..756418ab4 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/FileUtil.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/FileUtil.java @@ -2,6 +2,7 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -90,6 +91,25 @@ public static void saveJsonFile(final String path, LOG.info("saveJsonFile: exit, wrote data to {}", toPath); } + public static void ensureWritableDirectory(final File directory) { + // try twice to work around concurrent access issues + if (! directory.exists()) { + if (! directory.mkdirs()) { + if (! directory.exists()) { + // last try + if (! directory.mkdirs()) { + if (! directory.exists()) { + throw new IllegalArgumentException("failed to create " + directory); + } + } + } + } + } + if (! directory.canWrite()) { + throw new IllegalArgumentException("not allowed to write to " + directory); + } + } + private static final Logger LOG = LoggerFactory.getLogger(FileUtil.class); private static final int DEFAULT_BUFFER_SIZE = 65536; diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderSectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderSectionClient.java index f9060a2c7..a29cd3205 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderSectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderSectionClient.java @@ -98,7 +98,7 @@ public RenderSectionClient(final Parameters clientParameters) { sectionsAtScaleName).toAbsolutePath(); this.sectionDirectory = sectionPath.toFile(); - ensureWritableDirectory(this.sectionDirectory); + FileUtil.ensureWritableDirectory(this.sectionDirectory); // set cache size to 50MB so that masks get cached but most of RAM is left for target image final int maxCachedPixels = 50 * 1000000; @@ -155,29 +155,11 @@ private File getSectionFile(final Double z) { final int hundreds = (z.intValue() % 1000) / 100; final File hundredsDir = new File(thousandsDir, String.valueOf(hundreds)); - ensureWritableDirectory(hundredsDir); + FileUtil.ensureWritableDirectory(hundredsDir); return new File(hundredsDir, z + "." + clientParameters.format.toLowerCase()); } - private void ensureWritableDirectory(final File directory) { - // try twice to work around concurrent access issues - if (! directory.exists()) { - if (! directory.mkdirs()) { - if (! directory.exists()) { - // last try - if (! directory.mkdirs()) { - if (! directory.exists()) { - throw new IllegalArgumentException("failed to create " + directory); - } - } - } - } - } - if (! directory.canWrite()) { - throw new IllegalArgumentException("not allowed to write to " + directory); - } - } private String getNumericDirectoryName(final int value) { String pad = "00"; diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/ScapeClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/ScapeClient.java index f38273d50..130a025b5 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/ScapeClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/ScapeClient.java @@ -2,7 +2,12 @@ import com.beust.jcommander.Parameter; +import ij.ImagePlus; +import ij.ImageStack; +import ij.plugin.ZProjector; import ij.process.ByteProcessor; +import ij.process.ColorProcessor; +import ij.process.ImageProcessor; import java.awt.image.BufferedImage; import java.io.File; @@ -11,7 +16,11 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; import java.util.List; import org.apache.spark.SparkConf; @@ -21,11 +30,13 @@ import org.janelia.alignment.ArgbRenderer; import org.janelia.alignment.RenderParameters; import org.janelia.alignment.Utils; +import org.janelia.alignment.json.JsonUtils; import org.janelia.alignment.spec.Bounds; import org.janelia.alignment.spec.SectionData; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.util.ImageProcessorCache; import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.FileUtil; import org.janelia.render.client.RenderDataClient; import org.janelia.render.client.RenderDataClientParameters; import org.slf4j.Logger; @@ -35,6 +46,7 @@ * Spark client for rendering montage scapes for a range of layers within a stack. * * @author Eric Trautman + * @author Stephan Saalfeld */ public class ScapeClient implements Serializable { @@ -56,12 +68,24 @@ private static class Parameters extends RenderDataClientParameters { required = true) private String rootDirectory; + @Parameter( + names = "--maxImagesPerDirectory", + description = "Maximum number of images to render in one directory", + required = false) + private Integer maxImagesPerDirectory = 1000; + @Parameter( names = "--scale", description = "Scale for each rendered layer", required = false) private Double scale = 0.02; + @Parameter( + names = "--zScale", + description = "Ratio of z to xy resolution for creating isotropic layer projections (omit to skip projection)", + required = false) + private Double zScale; + @Parameter( names = "--format", description = "Format for rendered boxes", @@ -87,10 +111,35 @@ private static class Parameters extends RenderDataClientParameters { private boolean fillWithNoise = false; @Parameter( - names = "--useStackBounds", - description = "Base each scape on stack bounds instead of on section bounds (e.g. for aligned data)", + names = "--useLayerBounds", + description = "Base each scape on layer bounds instead of on stack bounds (e.g. for unaligned data)", + required = false, + arity = 1) + private boolean useLayerBounds = false; + + @Parameter( + names = "--minX", + description = "Left most pixel coordinate in world coordinates. Default is minX of stack (or layer when --useLayerBounds true)", + required = false) + private Double minX; + + @Parameter( + names = "--minY", + description = "Top most pixel coordinate in world coordinates. Default is minY of stack (or layer when --useLayerBounds true)", required = false) - private boolean useStackBounds = false; + private Double minY; + + @Parameter( + names = "--width", + description = "Width in world coordinates. Default is maxX - minX of stack (or layer when --useLayerBounds true)", + required = false) + private Double width; + + @Parameter( + names = "--height", + description = "Height in world coordinates. Default is maxY - minY of stack (or layer when --useLayerBounds true)", + required = false) + private Double height; @Parameter( names = "--minZ", @@ -104,6 +153,40 @@ private static class Parameters extends RenderDataClientParameters { required = false) private Double maxZ; + public File getSectionRootDirectory() { + + final String scapeDir = "scape_" + new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); + final Path sectionRootPath = Paths.get(rootDirectory, + project, + stack, + scapeDir).toAbsolutePath(); + return sectionRootPath.toFile(); + } + + public double getEffectiveBound(final Double layerValue, + final Double stackValue, + final Double parameterValue) { + final double value; + if (parameterValue == null) { + if (useLayerBounds) { + value = layerValue; + } else { + value = stackValue; + } + } else { + value = parameterValue; + } + return value; + } + + public Double getMaxX(final double effectiveMinX) { + return (width == null) ? null : effectiveMinX + width; + } + + public Double getMaxY(final double effectiveMinY) { + return (height == null) ? null : effectiveMinY + height; + } + } public static void main(final String[] args) { @@ -132,7 +215,7 @@ public ScapeClient(final Parameters parameters) { public void run() throws IOException, URISyntaxException { - final SparkConf conf = new SparkConf().setAppName("BoxClient"); + final SparkConf conf = new SparkConf().setAppName("ScapeClient"); final JavaSparkContext sparkContext = new JavaSparkContext(conf); final String sparkAppId = sparkContext.getConf().getAppId(); @@ -140,7 +223,6 @@ public void run() LOG.info("run: appId is {}, executors data is {}", sparkAppId, executorsJson); - final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, parameters.owner, parameters.project); @@ -149,111 +231,96 @@ public void run() parameters.minZ, parameters.maxZ); + // projection process depends upon z ordering, so sort section data results by z ... + Collections.sort(sectionDataList, SectionData.Z_COMPARATOR); + if (sectionDataList.size() == 0) { throw new IllegalArgumentException("source stack does not contain any matching z values"); } - final List adjustedSectionDataList; - if (parameters.useStackBounds) { - final StackMetaData stackMetaData = sourceDataClient.getStackMetaData(parameters.stack); - final Bounds stackBounds = stackMetaData.getStats().getStackBounds(); - - // TODO: validate size * scale is not too big - - adjustedSectionDataList = new ArrayList<>(sectionDataList.size()); - for (final SectionData sectionData : sectionDataList) { - adjustedSectionDataList.add(new SectionData(sectionData.getSectionId(), - sectionData.getZ(), - sectionData.getTileCount(), - stackBounds.getMinX(), - stackBounds.getMaxX(), - stackBounds.getMinY(), - stackBounds.getMaxY())); - } + final File sectionRootDirectory = parameters.getSectionRootDirectory(); + FileUtil.ensureWritableDirectory(sectionRootDirectory); - } else { - adjustedSectionDataList = sectionDataList; - } + // save run parameters so that we can understand render context later if necessary + final File parametersFile = new File(sectionRootDirectory, "scape_parameters.json"); + JsonUtils.MAPPER.writeValue(parametersFile, parameters); - final Path projectPath = Paths.get(parameters.rootDirectory, - parameters.project).toAbsolutePath(); + final List renderSectionList = + getRenderSections(sourceDataClient, sectionDataList, sectionRootDirectory); - String runContext = "scale_" + parameters.scale; - if (parameters.doFilter) { - runContext = runContext + "_filter"; - } - if (parameters.fillWithNoise) { - runContext = runContext + "_fill"; - } - if (parameters.useStackBounds) { - runContext = runContext + "_align"; - } - - final Path sectionBasePath = Paths.get(projectPath.toString(), - parameters.stack, - runContext).toAbsolutePath(); - - final File sectionBaseDirectory = sectionBasePath.toFile(); - for (final SectionData sectionData : adjustedSectionDataList) { - final File sectionFile = getSectionFile(sectionBaseDirectory, - sectionData.getZ(), - parameters.format.toLowerCase()); - ensureWritableDirectory(sectionFile.getParentFile()); - } + final JavaRDD rddSectionData = sparkContext.parallelize(renderSectionList); - - final JavaRDD rddSectionData = sparkContext.parallelize(adjustedSectionDataList); - - final Function generateScapeFunction = new Function() { + final Function generateScapeFunction = new Function() { final @Override - public Integer call(final SectionData sectionData) + public Integer call(final RenderSection renderSection) throws Exception { - final Double z = sectionData.getZ(); + final Double z = renderSection.getFirstZ(); LogUtilities.setupExecutorLog4j("z " + z); final RenderDataClient sourceDataClient = new RenderDataClient(parameters.baseDataUrl, parameters.owner, parameters.project); - final int width = (int) (sectionData.getMaxX() - sectionData.getMinX() + 0.5); - final int height = (int) (sectionData.getMaxY() - sectionData.getMinY() + 0.5); + // set cache size to 50MB so that masks get cached but most of RAM is left for target image + final int maxCachedPixels = 50 * 1000000; + final ImageProcessorCache imageProcessorCache = + new ImageProcessorCache(maxCachedPixels, false, false); + + final boolean isProjectionNeeded = renderSection.isProjectionNeeded(); + BufferedImage sectionImage = null; + ImageStack projectedStack = null; + + for (final SectionData sectionData : renderSection.getSectionDataList()) { - final String parametersUrl = - sourceDataClient.getRenderParametersUrlString(parameters.stack, - sectionData.getMinX(), - sectionData.getMinY(), - z, - width, - height, - parameters.scale); + final String parametersUrl = + sourceDataClient.getRenderParametersUrlString(parameters.stack, + sectionData.getMinX(), + sectionData.getMinY(), + sectionData.getZ(), + sectionData.getWidth(), + sectionData.getHeight(), + parameters.scale); - LOG.debug("generateScapeFunction: loading {}", parametersUrl); + LOG.debug("generateScapeFunction: loading {}", parametersUrl); - final RenderParameters renderParameters = RenderParameters.loadFromUrl(parametersUrl); - renderParameters.setDoFilter(parameters.doFilter); - renderParameters.setChannels(parameters.channels); + final RenderParameters renderParameters = RenderParameters.loadFromUrl(parametersUrl); + renderParameters.setDoFilter(parameters.doFilter); + renderParameters.setChannels(parameters.channels); - final File sectionFile = getSectionFile(sectionBaseDirectory, - sectionData.getZ(), - parameters.format.toLowerCase()); + sectionImage = renderParameters.openTargetImage(); - final BufferedImage sectionImage = renderParameters.openTargetImage(); + if (isProjectionNeeded && (projectedStack == null)) { + projectedStack = new ImageStack(sectionImage.getWidth(), sectionImage.getHeight()); + } + + if (parameters.fillWithNoise) { + final ByteProcessor ip = new ByteProcessor(sectionImage.getWidth(), sectionImage.getHeight()); + mpicbg.ij.util.Util.fillWithNoise(ip); + sectionImage.getGraphics().drawImage(ip.createImage(), 0, 0, null); + } - if (parameters.fillWithNoise) { - final ByteProcessor ip = new ByteProcessor(sectionImage.getWidth(), sectionImage.getHeight()); - mpicbg.ij.util.Util.fillWithNoise(ip); - sectionImage.getGraphics().drawImage(ip.createImage(), 0, 0, null); + ArgbRenderer.render(renderParameters, sectionImage, imageProcessorCache); + + if (isProjectionNeeded) { + projectedStack.addSlice(new ColorProcessor(sectionImage).convertToByteProcessor()); + } } - // set cache size to 50MB so that masks get cached but most of RAM is left for target image - final int maxCachedPixels = 50 * 1000000; - final ImageProcessorCache imageProcessorCache = - new ImageProcessorCache(maxCachedPixels, false, false); + if (projectedStack != null) { + + LOG.debug("projecting {} sections", projectedStack.getSize()); + + final ZProjector projector = new ZProjector(new ImagePlus("", projectedStack)); + projector.setMethod(ZProjector.AVG_METHOD); + projector.doProjection(); + final ImageProcessor ip = projector.getProjection().getProcessor(); + sectionImage = ip.getBufferedImage(); + } - ArgbRenderer.render(renderParameters, sectionImage, imageProcessorCache); + final File sectionFile = renderSection.getOutputFile(parameters.format); Utils.saveImage(sectionImage, sectionFile.getAbsolutePath(), parameters.format, true, 0.85f); @@ -275,31 +342,122 @@ public Integer call(final SectionData sectionData) sparkContext.stop(); } - public static File getSectionFile(final File sectionBaseDirectory, - final Double z, - final String format) { + private List getRenderSections(final RenderDataClient sourceDataClient, + final List sectionDataList, + final File sectionRootDirectory) + throws IOException { + + final StackMetaData stackMetaData = sourceDataClient.getStackMetaData(parameters.stack); + final Bounds stackBounds = stackMetaData.getStats().getStackBounds(); + + int maxZCharacters = 3; // %3.1d => 1.0 + for (long z = 1; z < stackBounds.getMaxZ().longValue(); z = z * 10) { + maxZCharacters++; + } + final String zFormatSpec = "%0" + maxZCharacters + ".1f"; + + final double zScale = parameters.zScale == null ? 0.0 : parameters.zScale / parameters.scale; + + final List renderSectionList = new ArrayList<>(sectionDataList.size()); + + RenderSection currentRenderSection = null; + double currentZ = -1; + + for (final SectionData sectionData : sectionDataList) { - final String paddedZ = String.format("%08.1f", z); - final File thousandsDir = new File(sectionBaseDirectory, paddedZ.substring(0, 3)); - return new File(thousandsDir, paddedZ.substring(3) + "." + format); + if ((currentRenderSection == null) || (sectionData.getZ() - zScale >= currentZ)) { + currentZ = sectionData.getZ(); + currentRenderSection = new RenderSection(currentZ, + renderSectionList.size(), + zFormatSpec, + parameters.maxImagesPerDirectory, + sectionRootDirectory); + renderSectionList.add(currentRenderSection); + } + + final double minX = parameters.getEffectiveBound(sectionData.getMinX(), stackBounds.getMinX(), parameters.minX); + final double minY = parameters.getEffectiveBound(sectionData.getMinY(), stackBounds.getMinY(), parameters.minY); + + final SectionData boundedSectionData = + new SectionData(sectionData.getSectionId(), + sectionData.getZ(), + sectionData.getTileCount(), + minX, + parameters.getEffectiveBound(sectionData.getMaxX(), + stackBounds.getMaxX(), + parameters.getMaxX(minX)), + minY, + parameters.getEffectiveBound(sectionData.getMaxY(), + stackBounds.getMaxY(), + parameters.getMaxY(minY))); + + final long scaledSectionWidth = (long) (boundedSectionData.getWidth() * parameters.scale + 0.5); + final long scaledSectionHeight = (long) (boundedSectionData.getHeight() * parameters.scale + 0.5); + final long sectionPixelCount = scaledSectionWidth * scaledSectionHeight; + + if (sectionPixelCount >= Integer.MAX_VALUE) { + final DecimalFormat formatter = new DecimalFormat("#,###"); + throw new IllegalArgumentException("section " + boundedSectionData + " has " + + formatter.format(sectionPixelCount) + " pixels at scale " + + parameters.scale + " which is greater than the maximum allowed " + + formatter.format(Integer.MAX_VALUE)); + } + + currentRenderSection.addSection(boundedSectionData); + } + + return renderSectionList; } - public static void ensureWritableDirectory(final File directory) { - // try twice to work around concurrent access issues - if (! directory.exists()) { - if (! directory.mkdirs()) { - if (! directory.exists()) { - // last try - if (! directory.mkdirs()) { - if (! directory.exists()) { - throw new IllegalArgumentException("failed to create " + directory); - } - } - } + public static class RenderSection implements Serializable { + + private final Double firstZ; + private final List sectionDataList; + private final File outputDir; + private final String zFormatSpec; + + public RenderSection(final Double firstZ, + final int stackIndex, + final String zFormatSpec, + final int maxImagesPerDirectory, + final File sectionRootDirectory) { + + this.firstZ = firstZ; + this.sectionDataList = new ArrayList<>(); + this.zFormatSpec = zFormatSpec; + + final int imageDirectoryIndex = stackIndex / maxImagesPerDirectory; + final String imageDirectoryName = String.format("%03d", imageDirectoryIndex); + this.outputDir = new File(sectionRootDirectory, imageDirectoryName); + } + + public List getSectionDataList() { + return sectionDataList; + } + + public Double getFirstZ() { + return firstZ; + } + + public boolean isProjectionNeeded() { + return sectionDataList.size() > 1; + } + + public File getOutputFile(final String fileExtension) { + final String paddedZName; + if (sectionDataList.size() > 1) { + final Double lastZ = sectionDataList.get(sectionDataList.size() - 1).getZ(); + final String formatPattern = "z" + zFormatSpec + "_to_" + zFormatSpec; + paddedZName = String.format(formatPattern, firstZ, lastZ); + } else { + final String formatPattern = "z" + zFormatSpec; + paddedZName = String.format(formatPattern, firstZ); } + return new File(outputDir, paddedZName + "." + fileExtension.toLowerCase()); } - if (! directory.canWrite()) { - throw new IllegalArgumentException("not allowed to write to " + directory); + + public void addSection(final SectionData sectionData) { + this.sectionDataList.add(sectionData); } } diff --git a/render-ws/src/main/java/org/janelia/render/service/StackMetaDataService.java b/render-ws/src/main/java/org/janelia/render/service/StackMetaDataService.java index 93969c3e9..ab8a730cd 100644 --- a/render-ws/src/main/java/org/janelia/render/service/StackMetaDataService.java +++ b/render-ws/src/main/java/org/janelia/render/service/StackMetaDataService.java @@ -153,6 +153,40 @@ public StackMetaData getStackMetaData(@PathParam("owner") final String owner, return stackMetaData; } + @Path("owner/{owner}/project/{fromProject}/stack/{fromStack}/stackId") + @PUT + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation( + tags = {"Stack Data APIs", "Stack Management APIs"}, + value = "Rename a stack") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "stack successfully renamed"), + @ApiResponse(code = 400, message = "stack cannot be renamed"), + @ApiResponse(code = 404, message = "stack not found") + }) + public Response renameStack(@PathParam("owner") final String owner, + @PathParam("fromProject") final String fromProject, + @PathParam("fromStack") final String fromStack, + @Context final UriInfo uriInfo, + final StackId toStackId) { + + LOG.info("renameStack: entry, owner={}, fromProject={}, fromStack={}, toStackId={}", + owner, fromProject, fromStack, toStackId); + + try { + final StackId fromStackId = new StackId(owner, fromProject, fromStack); + + renderDao.renameStack(fromStackId, toStackId); + + LOG.info("renameStack: renamed {} to {}", fromStackId, toStackId); + + } catch (final Throwable t) { + RenderServiceUtil.throwServiceException(t); + } + + return Response.ok().build(); + } + @Path("owner/{owner}/project/{fromProject}/stack/{fromStack}/cloneTo/{toStack}") @PUT @Consumes(MediaType.APPLICATION_JSON) diff --git a/render-ws/src/main/java/org/janelia/render/service/dao/RenderDao.java b/render-ws/src/main/java/org/janelia/render/service/dao/RenderDao.java index 86752d90c..a514b8563 100644 --- a/render-ws/src/main/java/org/janelia/render/service/dao/RenderDao.java +++ b/render-ws/src/main/java/org/janelia/render/service/dao/RenderDao.java @@ -2,6 +2,7 @@ import com.mongodb.BasicDBList; import com.mongodb.MongoClient; +import com.mongodb.MongoNamespace; import com.mongodb.QueryOperators; import com.mongodb.bulk.BulkWriteResult; import com.mongodb.client.MongoCollection; @@ -1438,6 +1439,60 @@ public void cloneStack(final StackId fromStackId, cloneCollection(fromTileCollection, toTileCollection, filterQuery); } + /** + * Renames the specified stack. + * + * @param fromStackId original stack name. + * @param toStackId new stack name. + * + * @throws IllegalArgumentException + * if the new stack already exists or the original stack cannot be renamed for any other reason. + * + * @throws ObjectNotFoundException + * if the original stack does not exist. + */ + public void renameStack(final StackId fromStackId, + final StackId toStackId) + throws IllegalArgumentException, ObjectNotFoundException { + + MongoUtil.validateRequiredParameter("fromStackId", fromStackId); + MongoUtil.validateRequiredParameter("toStackId", toStackId); + + final StackMetaData fromStackMetaData = getStackMetaData(fromStackId); + if (fromStackMetaData == null) { + throw new ObjectNotFoundException(fromStackId + " does not exist"); + } + + if (fromStackMetaData.isReadOnly() || fromStackMetaData.isOffline()) { + throw new IllegalArgumentException(fromStackId + " cannot be modified because it is " + + fromStackMetaData.getState() + "."); + } + + StackMetaData toStackMetaData = getStackMetaData(toStackId); + if (toStackMetaData != null) { + throw new IllegalArgumentException(toStackId + " already exists"); + } + + renameCollection(fromStackId.getSectionCollectionName(), toStackId.getSectionCollectionName()); + renameCollection(fromStackId.getTransformCollectionName(), toStackId.getTransformCollectionName()); + renameCollection(fromStackId.getTileCollectionName(), toStackId.getTileCollectionName()); + + toStackMetaData = StackMetaData.buildDerivedMetaData(toStackId, fromStackMetaData); + + final MongoCollection stackMetaDataCollection = getStackMetaDataCollection(); + final Document query = getStackIdQuery(fromStackId); + final Document stackMetaDataObject = Document.parse(toStackMetaData.toJson()); + final UpdateResult result = stackMetaDataCollection.replaceOne(query, + stackMetaDataObject, + MongoUtil.UPSERT_OPTION); + + LOG.debug("renameStack: ran {}.{},({}), upsertedId is {}", + MongoUtil.fullName(stackMetaDataCollection), + MongoUtil.action(result), + query.toJson(), + result.getUpsertedId()); + } + /** * Writes the layout file data for the specified stack to the specified stream. * @@ -1910,6 +1965,22 @@ private Double getBound(final MongoCollection tileCollection, return bound; } + private void renameCollection(final String fromCollectionName, + final String toCollectionName) { + + if (MongoUtil.exists(renderDatabase, fromCollectionName)) { + + final MongoCollection fromCollection = renderDatabase.getCollection(fromCollectionName); + final MongoNamespace toNamespace = new MongoNamespace(renderDatabase.getName(), toCollectionName); + fromCollection.renameCollection(toNamespace); + + LOG.debug("renameCollection: exit, ran {}.renameCollection({})", + MongoUtil.fullName(fromCollection), + toCollectionName); + } + + } + private void cloneCollection(final MongoCollection fromCollection, final MongoCollection toCollection, final Document filterQuery) diff --git a/render-ws/src/test/java/org/janelia/render/service/dao/RenderDaoTest.java b/render-ws/src/test/java/org/janelia/render/service/dao/RenderDaoTest.java index 2df177cda..33c44449b 100644 --- a/render-ws/src/test/java/org/janelia/render/service/dao/RenderDaoTest.java +++ b/render-ws/src/test/java/org/janelia/render/service/dao/RenderDaoTest.java @@ -76,6 +76,60 @@ public static void after() throws Exception { embeddedMongoDb.stop(); } + @Test + public void testRenameStack() throws Exception { + + StackId fromStackId = stackId; + final List fromZValues = dao.getZValues(fromStackId); + + // ------------------------------------------------------------------------- + // test renaming stack without stats ... + + StackId toStackId = new StackId(fromStackId.getOwner(), fromStackId.getProject(), "renamedStackA"); + + StackMetaData toStackMetaData = dao.getStackMetaData(toStackId); + Assert.assertNull("toStack should not exist before rename", toStackMetaData); + + dao.renameStack(fromStackId, toStackId); + + toStackMetaData = dao.getStackMetaData(toStackId); + Assert.assertNotNull("toStack should exist after rename", toStackMetaData); + + StackMetaData fromStackMetaData = dao.getStackMetaData(fromStackId); + Assert.assertNull("fromStack should not exist after rename", fromStackMetaData); + + List toZValues = dao.getZValues(toStackId); + Assert.assertArrayEquals("z values do not match after rename", fromZValues.toArray(), toZValues.toArray()); + + // ------------------------------------------------------------------------- + // test renaming stack with stats ... + + fromStackId = toStackId; + fromStackMetaData = toStackMetaData; + fromStackMetaData = dao.ensureIndexesAndDeriveStats(fromStackMetaData); + final StackStats fromStats = fromStackMetaData.getStats(); + + toStackId = new StackId(fromStackId.getOwner(), fromStackId.getProject(), "renamedStackB"); + + toStackMetaData = dao.getStackMetaData(toStackId); + Assert.assertNull("toStack should not exist before rename", toStackMetaData); + + dao.renameStack(fromStackId, toStackId); + + toStackMetaData = dao.getStackMetaData(toStackId); + Assert.assertNotNull("toStack should exist after rename", toStackMetaData); + + fromStackMetaData = dao.getStackMetaData(fromStackId); + Assert.assertNull("fromStack should not exist after rename", fromStackMetaData); + + toZValues = dao.getZValues(toStackId); + Assert.assertArrayEquals("z values do not match after rename", fromZValues.toArray(), toZValues.toArray()); + + final StackStats toStats = toStackMetaData.getStats(); + + Assert.assertEquals("incorrect stats after rename", fromStats.toJson(), toStats.toJson()); + } + @Test public void testCloneStack() throws Exception {