diff --git a/doc/axes.md b/doc/axes.md new file mode 100644 index 0000000..15c5b7e --- /dev/null +++ b/doc/axes.md @@ -0,0 +1,72 @@ +# Axes lookup datastructure + +``` +\ +├── ".axes" +│ ├── "some2DVector" +│ │ ├── attributes.json {"dataType":"int32","compression":{"type":"raw"},"blockSize":[2,2],"dimensions":[2,2]} +│ ├── "a3DVector" +│ │ ├── attributes.json {"dataType":"int32","compression":{"type":"raw"},"blockSize":[2,2,2],"dimensions":[2,2,2]} +│ ┊ +│ +├── ... +┊ +``` + +Each "vector" dataset is a 2^n size single block dataset that enumerates the axes indices. The 2D axes lookup for ImgLib2 vectors (F-order) looks like this: + +``` +-1 0 -> -1 0 1 -1 + 1 -1 +``` + +The 3D axes lookup for numpy vectors (C-order) looks like this: + +``` +-1 2 -> -1 2 1 -1 0 -1 -1 -1 + 1 -1 + + 0 -1 +-1 -1 +``` + +Tensor and vector aware API can then load the index lookup by loading the dataset as a tensor and access the first positive coordinate at each axis according to the API's axes indexing scheme. + +This creates a named permutation for vectors. + +The ImgLib2 bindings offer API methods to create and use such lookups. Naturally, the mapping is defined as coming from ImgLib2 F-order: + +```java +createAxes( + String axesName, + int[] axes); + +readDoubleVectorAttribute( + N5Reader n5, + String groupName, + String attributeName, + int[] axes); +writeDoubleVectorAttribute( + double[] vector, + N5Writer n5, + String groupName, + String attributeName, + int[] axes); +readLongVectorAttribute( + N5Reader n5, + String groupName, + String attributeName, + int[] axes); +writeLongVectorAttribute( + double[] vector, + N5Writer n5, + String groupName, + String attributeName, + int[] axes); +``` + + + + + + diff --git a/scripts/open-and-show.bsh b/scripts/open-and-show.bsh index c2b844a..68bf134 100644 --- a/scripts/open-and-show.bsh +++ b/scripts/open-and-show.bsh @@ -2,8 +2,8 @@ import org.janelia.saalfeldlab.n5.*; import org.janelia.saalfeldlab.n5.imglib2.*; import net.imglib2.img.display.imagej.*; n5 = new N5FSReader ( "/nrs/saalfeld/lauritzen/02/workspace.n5" ); -pre = N5Utils.open(new N5FSReader("/nrs/saalfeld/lauritzen/02/workspace.n5"), "/predictions_it150000_pre_and_post-v0.1/pre_dist/s3"); +pre = N5.open(new N5FSReader("/nrs/saalfeld/lauritzen/02/workspace.n5"), "/predictions_it150000_pre_and_post-v0.1/pre_dist/s3"); ImageJFunctions.show(pre); -img = N5Utils.open(new N5FSReader("/nrs/saalfeld/lauritzen/02/workspace.n5"), "/predictions_it150000_pre_and_post-v0.1/post_dist/s3"); +img = N5.open(new N5FSReader("/nrs/saalfeld/lauritzen/02/workspace.n5"), "/predictions_it150000_pre_and_post-v0.1/post_dist/s3"); ImageJFunctions.show(img); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5.java b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5.java new file mode 100644 index 0000000..5380f00 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5.java @@ -0,0 +1,2006 @@ +/** + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5.imglib2; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.IntFunction; + +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.RawCompression; + +import net.imglib2.Interval; +import net.imglib2.IterableInterval; +import net.imglib2.RandomAccess; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.cache.Cache; +import net.imglib2.cache.LoaderCache; +import net.imglib2.cache.img.CachedCellImg; +import net.imglib2.cache.img.DiskCachedCellImgFactory; +import net.imglib2.cache.img.DiskCachedCellImgOptions; +import net.imglib2.cache.img.LoadedCellCacheLoader; +import net.imglib2.cache.ref.BoundedSoftRefLoaderCache; +import net.imglib2.cache.ref.SoftRefLoaderCache; +import net.imglib2.img.array.ArrayImg; +import net.imglib2.img.array.ArrayImgs; +import net.imglib2.img.array.ArrayRandomAccess; +import net.imglib2.img.basictypeaccess.AccessFlags; +import net.imglib2.img.basictypeaccess.ArrayDataAccessFactory; +import net.imglib2.img.basictypeaccess.array.ArrayDataAccess; +import net.imglib2.img.basictypeaccess.array.IntArray; +import net.imglib2.img.basictypeaccess.volatiles.VolatileAccess; +import net.imglib2.img.cell.Cell; +import net.imglib2.img.cell.CellGrid; +import net.imglib2.img.cell.LazyCellImg; +import net.imglib2.type.NativeType; +import net.imglib2.type.Type; +import net.imglib2.type.label.LabelMultisetType; +import net.imglib2.type.numeric.integer.ByteType; +import net.imglib2.type.numeric.integer.IntType; +import net.imglib2.type.numeric.integer.LongType; +import net.imglib2.type.numeric.integer.ShortType; +import net.imglib2.type.numeric.integer.UnsignedByteType; +import net.imglib2.type.numeric.integer.UnsignedIntType; +import net.imglib2.type.numeric.integer.UnsignedLongType; +import net.imglib2.type.numeric.integer.UnsignedShortType; +import net.imglib2.type.numeric.real.DoubleType; +import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.util.Intervals; +import net.imglib2.util.Pair; +import net.imglib2.util.Util; +import net.imglib2.util.ValuePair; +import net.imglib2.view.Views; + +/** + * Static utility methods to open N5 datasets as ImgLib2 + * {@link RandomAccessibleInterval RandomAccessibleIntervals} and to save + * ImgLib2 {@link RandomAccessibleInterval RandomAccessibleIntervals} as + * [sparse] N5 datasets. + * + * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> + * @author Philipp Hanslovsky <hanslovskyp@janelia.hhmi.org> + */ +public class N5 { + + public static final String AXES_PREFIX = "/.axes/"; + + private N5() {} + + public static final > DataType dataType(final T type) { + + if (DoubleType.class.isInstance(type)) + return DataType.FLOAT64; + if (FloatType.class.isInstance(type)) + return DataType.FLOAT32; + if (LongType.class.isInstance(type)) + return DataType.INT64; + if (UnsignedLongType.class.isInstance(type)) + return DataType.UINT64; + if (IntType.class.isInstance(type)) + return DataType.INT32; + if (UnsignedIntType.class.isInstance(type)) + return DataType.UINT32; + if (ShortType.class.isInstance(type)) + return DataType.INT16; + if (UnsignedShortType.class.isInstance(type)) + return DataType.UINT16; + if (ByteType.class.isInstance(type)) + return DataType.INT8; + if (UnsignedByteType.class.isInstance(type)) + return DataType.UINT8; + else + return null; + } + + @SuppressWarnings("unchecked") + public static final > T type(final DataType dataType) { + + switch (dataType) { + case INT8: + return (T) new ByteType(); + case UINT8: + return (T) new UnsignedByteType(); + case INT16: + return (T) new ShortType(); + case UINT16: + return (T) new UnsignedShortType(); + case INT32: + return (T) new IntType(); + case UINT32: + return (T) new UnsignedIntType(); + case INT64: + return (T) new LongType(); + case UINT64: + return (T) new UnsignedLongType(); + case FLOAT32: + return (T) new FloatType(); + case FLOAT64: + return (T) new DoubleType(); + default: + return null; + } + } + + /** + * Creates a {@link DataBlock} of matching type and copies the content of + * source into it. This is a helper method with redundant parameters. + * + * @param source + * @param dataType + * @param intBlockSize + * @param longBlockSize + * @param gridPosition + * @return + */ + @SuppressWarnings("unchecked") + private static final DataBlock createDataBlock( + final RandomAccessibleInterval source, + final DataType dataType, + final int[] intBlockSize, + final long[] longBlockSize, + final long[] gridPosition) { + + final DataBlock dataBlock = dataType.createDataBlock(intBlockSize, gridPosition); + switch (dataType) { + case UINT8: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.unsignedBytes((byte[])dataBlock.getData(), longBlockSize)); + break; + case INT8: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.bytes((byte[])dataBlock.getData(), longBlockSize)); + break; + case UINT16: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.unsignedShorts((short[])dataBlock.getData(), longBlockSize)); + break; + case INT16: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.shorts((short[])dataBlock.getData(), longBlockSize)); + break; + case UINT32: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.unsignedInts((int[])dataBlock.getData(), longBlockSize)); + break; + case INT32: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.ints((int[])dataBlock.getData(), longBlockSize)); + break; + case UINT64: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.unsignedLongs((long[])dataBlock.getData(), longBlockSize)); + break; + case INT64: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.longs((long[])dataBlock.getData(), longBlockSize)); + break; + case FLOAT32: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.floats((float[])dataBlock.getData(), longBlockSize)); + break; + case FLOAT64: + N5CellLoader.burnIn( + (RandomAccessibleInterval)source, + ArrayImgs.doubles((double[])dataBlock.getData(), longBlockSize)); + break; + default: + throw new IllegalArgumentException("Type " + dataType.name() + " not supported!"); + } + + return dataBlock; + } + + /** + * Creates a {@link DataBlock} of matching type and copies the content of + * source into it. This is a helper method with redundant parameters. + * + * @param source + * @param dataType + * @param intBlockSize + * @param longBlockSize + * @param gridPosition + * @param defaultValue + * @return + */ + @SuppressWarnings("unchecked") + private static final > DataBlock createNonEmptyDataBlock( + final RandomAccessibleInterval source, + final DataType dataType, + final int[] intBlockSize, + final long[] longBlockSize, + final long[] gridPosition, + final T defaultValue) { + + final DataBlock dataBlock = dataType.createDataBlock(intBlockSize, gridPosition); + final boolean isEmpty; + switch (dataType) { + case UINT8: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.unsignedBytes((byte[])dataBlock.getData(), longBlockSize), + (UnsignedByteType)defaultValue); + break; + case INT8: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.bytes((byte[])dataBlock.getData(), longBlockSize), + (ByteType)defaultValue); + break; + case UINT16: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.unsignedShorts((short[])dataBlock.getData(), longBlockSize), + (UnsignedShortType)defaultValue); + break; + case INT16: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.shorts((short[])dataBlock.getData(), longBlockSize), + (ShortType)defaultValue); + break; + case UINT32: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.unsignedInts((int[])dataBlock.getData(), longBlockSize), + (UnsignedIntType)defaultValue); + break; + case INT32: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.ints((int[])dataBlock.getData(), longBlockSize), + (IntType)defaultValue); + break; + case UINT64: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.unsignedLongs((long[])dataBlock.getData(), longBlockSize), + (UnsignedLongType)defaultValue); + break; + case INT64: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.longs((long[])dataBlock.getData(), longBlockSize), + (LongType)defaultValue); + break; + case FLOAT32: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.floats((float[])dataBlock.getData(), longBlockSize), + (FloatType)defaultValue); + break; + case FLOAT64: + isEmpty = N5CellLoader.burnInTestAllEqual( + (RandomAccessibleInterval)source, + ArrayImgs.doubles((double[])dataBlock.getData(), longBlockSize), + (DoubleType)defaultValue); + break; + default: + throw new IllegalArgumentException("Type " + dataType.name() + " not supported!"); + } + + return isEmpty ? null : dataBlock; + } + + /** + * Crops the dimensions of a {@link DataBlock} at a given offset to fit into + * and {@link Interval} of given dimensions. Fills long and int version of + * cropped block size. Also calculates the grid raster position assuming + * that the offset divisible by block size without remainder. + * + * @param max + * @param offset + * @param blockDimensions + * @param croppedBlockDimensions + * @param intCroppedBlockDimensions + * @param gridPosition + */ + static final void cropBlockDimensions( + final long[] max, + final long[] offset, + final int[] blockDimensions, + final long[] croppedBlockDimensions, + final int[] intCroppedBlockDimensions, + final long[] gridPosition) { + + for (int d = 0; d < max.length; ++d) { + croppedBlockDimensions[d] = Math.min(blockDimensions[d], max[d] - offset[d] + 1); + intCroppedBlockDimensions[d] = (int)croppedBlockDimensions[d]; + gridPosition[d] = offset[d] / blockDimensions[d]; + } + } + + /** + * Crops the dimensions of a {@link DataBlock} at a given offset to fit into + * and {@link Interval} of given dimensions. Fills long and int version of + * cropped block size. Also calculates the grid raster position plus a grid + * offset assuming that the offset divisible by block size without + * remainder. + * + * @param max + * @param offset + * @param gridOffset + * @param blockDimensions + * @param croppedBlockDimensions + * @param intCroppedBlockDimensions + * @param gridPosition + */ + static final void cropBlockDimensions( + final long[] max, + final long[] offset, + final long[] gridOffset, + final int[] blockDimensions, + final long[] croppedBlockDimensions, + final int[] intCroppedBlockDimensions, + final long[] gridPosition) { + + for (int d = 0; d < max.length; ++d) { + croppedBlockDimensions[d] = Math.min(blockDimensions[d], max[d] - offset[d] + 1); + intCroppedBlockDimensions[d] = (int)croppedBlockDimensions[d]; + gridPosition[d] = offset[d] / blockDimensions[d] + gridOffset[d]; + } + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg}. + * Supports all primitive types and {@link LabelMultisetType}. + * + * @param n5 + * @param dataset + * @return + * @throws IOException + */ + @SuppressWarnings("unchecked") + public static final > CachedCellImg open( + final N5Reader n5, + final String dataset) throws IOException { + + if (N5LabelMultisets.isLabelMultisetType(n5, dataset)) + return (CachedCellImg) N5LabelMultisets.openLabelMultiset(n5, dataset); + else + return open(n5, dataset, (Consumer>)img -> {}); + } + + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg}. + * + * @param n5 + * @param dataset + * @param maxNumCacheEntries + * @return + * @throws IOException + */ + public static final > CachedCellImg openWithBoundedSoftRefCache( + final N5Reader n5, + final String dataset, + final int maxNumCacheEntries) throws IOException + { + return openWithBoundedSoftRefCache(n5, dataset, (Consumer>)img -> {}, maxNumCacheEntries); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg} using + * {@link VolatileAccess}. + * Supports all primitive types and {@link LabelMultisetType}. + * + * @param n5 + * @param dataset + * @return + * @throws IOException + */ + @SuppressWarnings("unchecked") + public static final > CachedCellImg openVolatile( + final N5Reader n5, + final String dataset) throws IOException { + + if (N5LabelMultisets.isLabelMultisetType(n5, dataset)) + return (CachedCellImg) N5LabelMultisets.openLabelMultiset(n5, dataset); + else + return openVolatile(n5, dataset, (Consumer>)img -> {}); + } + + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg} using + * {@link VolatileAccess}. + * + * @param n5 + * @param dataset + * @param maxNumCacheEntries + * @return + * @throws IOException + */ + public static final > CachedCellImg openVolatileWithBoundedSoftRefCache( + final N5Reader n5, + final String dataset, + final int maxNumCacheEntries) throws IOException + { + return openVolatileWithBoundedSoftRefCache(n5, dataset, (Consumer>)img -> {}, maxNumCacheEntries); + } + + /** + * Open an N5 dataset as a disk-cached {@link LazyCellImg}. Note that this + * requires that all parts of the the N5 dataset that will be accessed fit + * into /tmp. + * + * @param n5 + * @param dataset + * @return + * @throws IOException + */ + public static final > CachedCellImg openWithDiskCache( + final N5Reader n5, + final String dataset) throws IOException { + + return openWithDiskCache(n5, dataset, (Consumer>)img -> {}); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg}. + * + * @param n5 + * @param dataset + * @param defaultValue + * @return + * @throws IOException + */ + public static final > CachedCellImg open( + final N5Reader n5, + final String dataset, + final T defaultValue) throws IOException { + + return open(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue)); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg}. + * + * @param n5 + * @param dataset + * @param defaultValue + * @param maxNumCacheEntries + * @return + * @throws IOException + */ + public static final > CachedCellImg openWithBoundedSoftRefCache( + final N5Reader n5, + final String dataset, + final int maxNumCacheEntries, + final T defaultValue) throws IOException { + + return openWithBoundedSoftRefCache(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue), maxNumCacheEntries); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg} using + * {@link VolatileAccess}. + * + * @param n5 + * @param dataset + * @param defaultValue + * @return + * @throws IOException + */ + public static final > CachedCellImg openVolatile( + final N5Reader n5, + final String dataset, + final T defaultValue) throws IOException { + + return openVolatile(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue)); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg} using + * {@link VolatileAccess}. + * + * @param n5 + * @param dataset + * @param defaultValue + * @param maxNumCacheEntries + * @return + * @throws IOException + */ + public static final > CachedCellImg openVolatileWithBoundedSoftRefCache( + final N5Reader n5, + final String dataset, + final int maxNumCacheEntries, + final T defaultValue) throws IOException { + + return openVolatileWithBoundedSoftRefCache(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue), maxNumCacheEntries); + } + + /** + * Open an N5 dataset as a disk-cached {@link LazyCellImg}. Note that this + * requires that all parts of the the N5 dataset that will be accessed fit + * into /tmp. + * + * @param n5 + * @param dataset + * @param defaultValue + * @return + * @throws IOException + */ + public static final > CachedCellImg openWithDiskCache( + final N5Reader n5, + final String dataset, + final T defaultValue) throws IOException { + + return openWithDiskCache(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue)); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg}. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @return + * @throws IOException + */ + public static final > CachedCellImg open( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler) throws IOException { + + return open(n5, dataset, blockNotFoundHandler, AccessFlags.setOf()); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg}. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @return + * @throws IOException + */ + @SuppressWarnings({"rawtypes"}) + public static final > CachedCellImg open( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler, + final Set accessFlags) throws IOException { + + return open(n5, dataset, blockNotFoundHandler, dataType -> new SoftRefLoaderCache(), accessFlags); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg} with a bound on the number of cache entries. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @param maxNumCacheEntries + * @return + * @throws IOException + */ + public static final > CachedCellImg openWithBoundedSoftRefCache( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler, + final int maxNumCacheEntries) throws IOException { + + return openWithBoundedSoftRefCache(n5, dataset, blockNotFoundHandler, maxNumCacheEntries, AccessFlags.setOf()); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg} with a bound on the number of cache entries. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @param maxNumCacheEntries + * @return + * @throws IOException + */ + @SuppressWarnings({"rawtypes"}) + public static final > CachedCellImg openWithBoundedSoftRefCache( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler, + final int maxNumCacheEntries, + final Set accessFlags) throws IOException { + + return open(n5, dataset, blockNotFoundHandler, dataType -> new BoundedSoftRefLoaderCache(maxNumCacheEntries), accessFlags); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg}. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @param loaderCacheFactory + * @return + * @throws IOException + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static final > CachedCellImg open( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler, + final Function loaderCacheFactory, + final Set accessFlags) throws IOException { + + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + final LoaderCache loaderCache = loaderCacheFactory.apply(attributes.getDataType()); + final T type = type(attributes.getDataType()); + return type == null + ? null + : open(n5, dataset, blockNotFoundHandler, loaderCache, accessFlags, type); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg}. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @return + * @throws IOException + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static final , A extends ArrayDataAccess> CachedCellImg open( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler, + final LoaderCache> loaderCache, + final Set accessFlags, + final T type) throws IOException { + + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + final long[] dimensions = attributes.getDimensions(); + final int[] blockSize = attributes.getBlockSize(); + + final N5CellLoader loader = new N5CellLoader<>(n5, dataset, blockSize, blockNotFoundHandler); + + final CellGrid grid = new CellGrid(dimensions, blockSize); + + final Cache> cache = loaderCache.withLoader(LoadedCellCacheLoader.get(grid, loader, type, accessFlags)); + final CachedCellImg img = new CachedCellImg(grid, type, cache, ArrayDataAccessFactory.get(type, accessFlags)); + return img; + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg} using + * {@link VolatileAccess}. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @return + * @throws IOException + */ + public static final > CachedCellImg openVolatile( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler) throws IOException { + + return open(n5, dataset, blockNotFoundHandler, AccessFlags.setOf(AccessFlags.VOLATILE)); + } + + /** + * Open an N5 dataset as a memory cached {@link LazyCellImg} with a bound on the number of cache entries + * using {@link VolatileAccess}. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @param maxNumCacheEntries + * @return + * @throws IOException + */ + public static final > CachedCellImg openVolatileWithBoundedSoftRefCache( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler, + final int maxNumCacheEntries) throws IOException { + return openWithBoundedSoftRefCache(n5, dataset, blockNotFoundHandler, maxNumCacheEntries, AccessFlags.setOf(AccessFlags.VOLATILE)); + } + + /** + * Open an N5 mipmap (multi-scale) group as memory cached + * {@link LazyCellImg}s, optionally backed by {@link VolatileAccess}. + * + * @param n5 + * @param group + * @param useVolatileAccess + * @param blockNotFoundHandlerSupplier + * + * @return + * @throws IOException + */ + public static final > Pair[], double[][]> openMipmapsWithHandler( + final N5Reader n5, + final String group, + final boolean useVolatileAccess, + final IntFunction>> blockNotFoundHandlerSupplier) throws IOException { + + final int numScales = n5.list(group).length; + @SuppressWarnings("unchecked") + final RandomAccessibleInterval[] mipmaps = new RandomAccessibleInterval[numScales]; + final double[][] scales = new double[numScales][]; + + for (int s = 0; s < numScales; ++s) { + final String datasetName = group + "/s" + s; + final long[] dimensions = n5.getAttribute(datasetName, "dimensions", long[].class); + final long[] downsamplingFactors = n5.getAttribute(datasetName, "downsamplingFactors", long[].class); + final double[] scale = new double[dimensions.length]; + if (downsamplingFactors == null) { + final int si = 1 << s; + for (int i = 0; i < scale.length; ++i) + scale[i] = si; + } else { + for (int i = 0; i < scale.length; ++i) + scale[i] = downsamplingFactors[i]; + } + + final RandomAccessibleInterval source; + if (useVolatileAccess) + source = N5.openVolatile(n5, datasetName, blockNotFoundHandlerSupplier.apply(s)); + else + source = N5.open(n5, datasetName, blockNotFoundHandlerSupplier.apply(s)); + + mipmaps[s] = source; + scales[s] = scale; + } + + return new ValuePair<>(mipmaps, scales); + } + + /** + * Open an N5 mipmap (multi-scale) group as memory cached + * {@link LazyCellImg}s, optionally backed by {@link VolatileAccess}. + * + * @param n5 + * @param group + * @param useVolatileAccess + * @param defaultValueSupplier + * + * @return + * @throws IOException + */ + public static final > Pair[], double[][]> openMipmaps( + final N5Reader n5, + final String group, + final boolean useVolatileAccess, + final IntFunction defaultValueSupplier) throws IOException { + + return openMipmapsWithHandler( + n5, + group, + useVolatileAccess, + s -> { + return N5CellLoader.setToDefaultValue(defaultValueSupplier.apply(s)); + }); + } + + /** + * Open an N5 mipmap (multi-scale) group as memory cached + * {@link LazyCellImg}s, optionally backed by {@link VolatileAccess}. + * + * @param n5 + * @param group + * @param useVolatileAccess + * + * @return + * @throws IOException + */ + public static final > Pair[], double[][]> openMipmaps( + final N5Reader n5, + final String group, + final boolean useVolatileAccess) throws IOException { + + return openMipmapsWithHandler( + n5, + group, + useVolatileAccess, + s -> t -> {}); + } + + /** + * Open an N5 dataset as a disk-cached {@link LazyCellImg}. Note that this + * requires that al part of the the N5 dataset that will be accessed fit + * into /tmp. + * + * @param n5 + * @param dataset + * @param blockNotFoundHandler + * @return + * @throws IOException + */ + public static final > CachedCellImg openWithDiskCache( + final N5Reader n5, + final String dataset, + final Consumer> blockNotFoundHandler) throws IOException { + + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + final long[] dimensions = attributes.getDimensions(); + final int[] blockSize = attributes.getBlockSize(); + + final N5CellLoader loader = new N5CellLoader<>(n5, dataset, blockSize, blockNotFoundHandler); + + final DiskCachedCellImgOptions options = DiskCachedCellImgOptions + .options() + .cellDimensions(blockSize) + .dirtyAccesses(true) + .maxCacheSize(100); + + final DiskCachedCellImgFactory factory = new DiskCachedCellImgFactory( + type(attributes.getDataType()), + options); + + return factory.create(dimensions, loader); + } + + /** + * Save a {@link RandomAccessibleInterval} into an N5 dataset at a given + * offset. The offset is given in {@link DataBlock} grid coordinates and the + * source is assumed to align with the {@link DataBlock} grid of the dataset. + * + * @param source + * @param n5 + * @param dataset + * @param attributes + * @param gridOffset + * @throws IOException + */ + public static final > void saveBlock( + RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final DatasetAttributes attributes, + final long[] gridOffset) throws IOException { + + if (N5LabelMultisets.isLabelMultisetType(n5, dataset)) { + @SuppressWarnings("unchecked") + final RandomAccessibleInterval labelMultisetSource = (RandomAccessibleInterval) source; + N5LabelMultisets.saveLabelMultisetBlock(labelMultisetSource, n5, dataset, attributes, gridOffset); + return; + } + + source = Views.zeroMin(source); + final int n = source.numDimensions(); + final long[] max = Intervals.maxAsLongArray(source); + final long[] offset = new long[n]; + final long[] gridPosition = new long[n]; + final int[] blockSize = attributes.getBlockSize(); + final int[] intCroppedBlockSize = new int[n]; + final long[] longCroppedBlockSize = new long[n]; + for (int d = 0; d < n;) { + cropBlockDimensions( + max, + offset, + gridOffset, + blockSize, + longCroppedBlockSize, + intCroppedBlockSize, + gridPosition); + final RandomAccessibleInterval sourceBlock = Views.offsetInterval(source, offset, longCroppedBlockSize); + final DataBlock dataBlock = createDataBlock( + sourceBlock, + attributes.getDataType(), + intCroppedBlockSize, + longCroppedBlockSize, + gridPosition); + + n5.writeBlock(dataset, attributes, dataBlock); + + for (d = 0; d < n; ++d) { + offset[d] += blockSize[d]; + if (offset[d] <= max[d]) + break; + else + offset[d] = 0; + } + } + } + + /** + * Save a {@link RandomAccessibleInterval} into an N5 dataset. + * The block offset is determined by the source position, and the + * source is assumed to align with the {@link DataBlock} grid + * of the dataset. + * + * @param source + * @param n5 + * @param dataset + * @param attributes + * @throws IOException + */ + public static final > void saveBlock( + final RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final DatasetAttributes attributes) throws IOException { + + final int[] blockSize = attributes.getBlockSize(); + final long[] gridOffset = new long[blockSize.length]; + Arrays.setAll(gridOffset, d -> source.min(d) / blockSize[d]); + saveBlock(source, n5, dataset, attributes, gridOffset); + } + + /** + * Save a {@link RandomAccessibleInterval} into an N5 dataset. + * The block offset is determined by the source position, and the + * source is assumed to align with the {@link DataBlock} grid + * of the dataset. + * + * @param source + * @param n5 + * @param dataset + * @throws IOException + */ + public static final > void saveBlock( + final RandomAccessibleInterval source, + final N5Writer n5, + final String dataset) throws IOException { + + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + if (attributes != null) { + saveBlock(source, n5, dataset, attributes); + } else { + throw new IOException("Dataset " + dataset + " does not exist."); + } + } + + /** + * Save a {@link RandomAccessibleInterval} into an N5 dataset at a given + * offset. The offset is given in {@link DataBlock} grid coordinates and the + * source is assumed to align with the {@link DataBlock} grid of the dataset. + * + * @param source + * @param n5 + * @param dataset + * @param gridOffset + * @throws IOException + */ + public static final > void saveBlock( + final RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final long[] gridOffset) throws IOException { + + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + if (attributes != null) { + saveBlock(source, n5, dataset, attributes, gridOffset); + } else { + throw new IOException("Dataset " + dataset + " does not exist."); + } + } + + /** + * Save a {@link RandomAccessibleInterval} as an N5 dataset, multi-threaded. + * + * @param source + * @param n5 + * @param dataset + * @param gridOffset + * @param exec + * @throws IOException + * @throws InterruptedException + * @throws ExecutionException + */ + public static final > void saveBlock( + final RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final long[] gridOffset, + final ExecutorService exec) throws IOException, InterruptedException, ExecutionException { + + if (N5LabelMultisets.isLabelMultisetType(n5, dataset)) { + @SuppressWarnings("unchecked") + final RandomAccessibleInterval labelMultisetSource = (RandomAccessibleInterval) source; + N5LabelMultisets.saveLabelMultisetBlock(labelMultisetSource, n5, dataset, gridOffset, exec); + return; + } + + final RandomAccessibleInterval zeroMinSource = Views.zeroMin(source); + final long[] dimensions = Intervals.dimensionsAsLongArray(zeroMinSource); + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + if (attributes != null) { + final int n = dimensions.length; + final long[] max = Intervals.maxAsLongArray(zeroMinSource); + final long[] offset = new long[n]; + final int[] blockSize = attributes.getBlockSize(); + + final ArrayList> futures = new ArrayList<>(); + for (int d = 0; d < n;) { + final long[] fOffset = offset.clone(); + + futures.add( + exec.submit( + () -> { + + final long[] gridPosition = new long[n]; + final int[] intCroppedBlockSize = new int[n]; + final long[] longCroppedBlockSize = new long[n]; + + cropBlockDimensions( + max, + fOffset, + gridOffset, + blockSize, + longCroppedBlockSize, + intCroppedBlockSize, + gridPosition); + + final RandomAccessibleInterval sourceBlock = Views + .offsetInterval(zeroMinSource, fOffset, longCroppedBlockSize); + final DataBlock dataBlock = createDataBlock( + sourceBlock, + attributes.getDataType(), + intCroppedBlockSize, + longCroppedBlockSize, + gridPosition); + + try { + n5.writeBlock(dataset, attributes, dataBlock); + } catch (final IOException e) { + e.printStackTrace(); + } + })); + + for (d = 0; d < n; ++d) { + offset[d] += blockSize[d]; + if (offset[d] <= max[d]) + break; + else + offset[d] = 0; + } + } + for (final Future f : futures) + f.get(); + } else { + throw new IOException("Dataset " + dataset + " does not exist."); + } + } + + /** + * Save a {@link RandomAccessibleInterval} into an N5 dataset at a given + * offset. The offset is given in {@link DataBlock} grid coordinates and the + * source is assumed to align with the {@link DataBlock} grid of the + * dataset. Only {@link DataBlock DataBlocks} that contain values other than + * a given default value are stored. + * + * @param source + * @param n5 + * @param dataset + * @param attributes + * @param gridOffset + * @param defaultValue + * @throws IOException + */ + public static final > void saveNonEmptyBlock( + RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final DatasetAttributes attributes, + final long[] gridOffset, + final T defaultValue) throws IOException { + + source = Views.zeroMin(source); + final int n = source.numDimensions(); + final long[] max = Intervals.maxAsLongArray(source); + final long[] offset = new long[n]; + final long[] gridPosition = new long[n]; + final int[] blockSize = attributes.getBlockSize(); + final int[] intCroppedBlockSize = new int[n]; + final long[] longCroppedBlockSize = new long[n]; + for (int d = 0; d < n;) { + cropBlockDimensions( + max, + offset, + gridOffset, + blockSize, + longCroppedBlockSize, + intCroppedBlockSize, + gridPosition); + final RandomAccessibleInterval sourceBlock = Views.offsetInterval(source, offset, longCroppedBlockSize); + final DataBlock dataBlock = createNonEmptyDataBlock( + sourceBlock, + attributes.getDataType(), + intCroppedBlockSize, + longCroppedBlockSize, + gridPosition, + defaultValue); + + if (dataBlock != null) + n5.writeBlock(dataset, attributes, dataBlock); + + for (d = 0; d < n; ++d) { + offset[d] += blockSize[d]; + if (offset[d] <= max[d]) + break; + else + offset[d] = 0; + } + } + } + + /** + * Save a {@link RandomAccessibleInterval} into an N5 dataset. + * The block offset is determined by the source position, and the + * source is assumed to align with the {@link DataBlock} grid + * of the dataset. Only {@link DataBlock DataBlocks} that contain + * values other than a given default value are stored. + * + * @param source + * @param n5 + * @param dataset + * @param attributes + * @param defaultValue + * @throws IOException + */ + public static final > void saveNonEmptyBlock( + final RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final DatasetAttributes attributes, + final T defaultValue) throws IOException { + + final int[] blockSize = attributes.getBlockSize(); + final long[] gridOffset = new long[blockSize.length]; + Arrays.setAll(gridOffset, d -> source.min(d) / blockSize[d]); + saveNonEmptyBlock(source, n5, dataset, attributes, gridOffset, defaultValue); + } + + /** + * Save a {@link RandomAccessibleInterval} into an N5 dataset. + * The block offset is determined by the source position, and the + * source is assumed to align with the {@link DataBlock} grid + * of the dataset. Only {@link DataBlock DataBlocks} that contain + * values other than a given default value are stored. + * + * @param source + * @param n5 + * @param dataset + * @param defaultValue + * @throws IOException + */ + public static final > void saveNonEmptyBlock( + final RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final T defaultValue) throws IOException { + + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + if (attributes != null) { + saveNonEmptyBlock(source, n5, dataset, attributes, defaultValue); + } else { + throw new IOException("Dataset " + dataset + " does not exist."); + } + } + + /** + * Save a {@link RandomAccessibleInterval} into an N5 dataset at a given + * offset. The offset is given in {@link DataBlock} grid coordinates and the + * source is assumed to align with the {@link DataBlock} grid of the + * dataset. Only {@link DataBlock DataBlocks} that contain values other than + * a given default value are stored. + * + * @param source + * @param n5 + * @param dataset + * @param gridOffset + * @param defaultValue + * @throws IOException + */ + public static final > void saveNonEmptyBlock( + final RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final long[] gridOffset, + final T defaultValue) throws IOException { + + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + if (attributes != null) { + saveNonEmptyBlock(source, n5, dataset, attributes, gridOffset, defaultValue); + } else { + throw new IOException("Dataset " + dataset + " does not exist."); + } + } + + /** + * Save a {@link RandomAccessibleInterval} as an N5 dataset. + * + * @param source + * @param n5 + * @param dataset + * @param blockSize + * @param compression + * @throws IOException + */ + public static final > void save( + RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final int[] blockSize, + final Compression compression) throws IOException { + + if (Util.getTypeFromInterval(source) instanceof LabelMultisetType) { + @SuppressWarnings("unchecked") + final RandomAccessibleInterval labelMultisetSource = (RandomAccessibleInterval) source; + N5LabelMultisets.saveLabelMultiset(labelMultisetSource, n5, dataset, blockSize, compression); + return; + } + + source = Views.zeroMin(source); + final long[] dimensions = Intervals.dimensionsAsLongArray(source); + final DatasetAttributes attributes = new DatasetAttributes( + dimensions, + blockSize, + dataType(Util.getTypeFromInterval(source)), + compression); + + n5.createDataset(dataset, attributes); + + final int n = dimensions.length; + final long[] max = Intervals.maxAsLongArray(source); + final long[] offset = new long[n]; + final long[] gridPosition = new long[n]; + final int[] intCroppedBlockSize = new int[n]; + final long[] longCroppedBlockSize = new long[n]; + for (int d = 0; d < n;) { + cropBlockDimensions(max, offset, blockSize, longCroppedBlockSize, intCroppedBlockSize, gridPosition); + final RandomAccessibleInterval sourceBlock = Views.offsetInterval(source, offset, longCroppedBlockSize); + final DataBlock dataBlock = createDataBlock( + sourceBlock, + attributes.getDataType(), + intCroppedBlockSize, + longCroppedBlockSize, + gridPosition); + + n5.writeBlock(dataset, attributes, dataBlock); + + for (d = 0; d < n; ++d) { + offset[d] += blockSize[d]; + if (offset[d] <= max[d]) + break; + else + offset[d] = 0; + } + } + } + + /** + * Save a {@link RandomAccessibleInterval} as an N5 dataset, multi-threaded. + * + * @param source + * @param n5 + * @param dataset + * @param blockSize + * @param compression + * @param exec + * @throws IOException + * @throws InterruptedException + * @throws ExecutionException + */ + public static final > void save( + final RandomAccessibleInterval source, + final N5Writer n5, + final String dataset, + final int[] blockSize, + final Compression compression, + final ExecutorService exec) throws IOException, InterruptedException, ExecutionException { + + if (Util.getTypeFromInterval(source) instanceof LabelMultisetType) { + @SuppressWarnings("unchecked") + final RandomAccessibleInterval labelMultisetSource = (RandomAccessibleInterval) source; + N5LabelMultisets.saveLabelMultiset(labelMultisetSource, n5, dataset, blockSize, compression, exec); + return; + } + + final RandomAccessibleInterval zeroMinSource = Views.zeroMin(source); + final long[] dimensions = Intervals.dimensionsAsLongArray(zeroMinSource); + final DatasetAttributes attributes = new DatasetAttributes( + dimensions, + blockSize, + dataType(Util.getTypeFromInterval(zeroMinSource)), + compression); + + n5.createDataset(dataset, attributes); + + final int n = dimensions.length; + final long[] max = Intervals.maxAsLongArray(zeroMinSource); + final long[] offset = new long[n]; + + final ArrayList> futures = new ArrayList<>(); + for (int d = 0; d < n;) { + final long[] fOffset = offset.clone(); + + futures.add( + exec.submit( + () -> { + + final long[] gridPosition = new long[n]; + final int[] intCroppedBlockSize = new int[n]; + final long[] longCroppedBlockSize = new long[n]; + + cropBlockDimensions( + max, + fOffset, + blockSize, + longCroppedBlockSize, + intCroppedBlockSize, + gridPosition); + + final RandomAccessibleInterval sourceBlock = Views + .offsetInterval(zeroMinSource, fOffset, longCroppedBlockSize); + final DataBlock dataBlock = createDataBlock( + sourceBlock, + attributes.getDataType(), + intCroppedBlockSize, + longCroppedBlockSize, + gridPosition); + + try { + n5.writeBlock(dataset, attributes, dataBlock); + } catch (final IOException e) { + e.printStackTrace(); + } + })); + + for (d = 0; d < n; ++d) { + offset[d] += blockSize[d]; + if (offset[d] <= max[d]) + break; + else + offset[d] = 0; + } + } + for (final Future f : futures) + f.get(); + } + + /** + * Create an axes index lookup. + * + * Examples: + * + *

+ * setAxes(n5, "bigcat3d", 2, 1, 0) + * + * creates an axes index lookup for C-order 3D vectors. + *

+ *

+ * setAxes(n5, "imglib4d", 0, 1, 2, 3) + * + * creates an axes index lookup for F-order 4D vectors. + *

+ * + * @param n5 writer + * @param axesName name of the axes index lookup + * @param axes axes index lookup + * @throws IOException + */ + public static final void setAxes(final N5Writer n5, final String axesName, final int... axes) throws IOException { + + if (axes == null) { + removeAxes(n5, axesName); + return; + } + + final int n = axes.length; + final long[] dimensions = new long[n]; + final int[] blockSize = new int[n]; + Arrays.fill(dimensions, 2); + Arrays.fill(blockSize, 2); + + final ArrayImg axesLookup = ArrayImgs.ints(dimensions); + for (final IntType t : axesLookup) + t.set(-1); + + final ArrayRandomAccess ra = axesLookup.randomAccess(axesLookup); + for (int d = 0; d < n; ++d) { + ra.fwd(d); + ra.get().set(axes[d]); + ra.bck(d); + } + + save(axesLookup, n5, AXES_PREFIX + axesName, blockSize, new RawCompression()); + } + + + /** + * Remove an axes index lookup. + * + * @param n5 writer + * @param axesName name of the axes index lookup + * @throws IOException + */ + public static final void removeAxes(final N5Writer n5, final String axesName) throws IOException { + + n5.remove(AXES_PREFIX + axesName); + } + + + /** + * Read an axes index lookup. + * + * @param n5 reader + * @param axesName name of the axes index lookup + * @return axes index lookup + * @throws IOException + */ + public static final int[] getAxes(final N5Reader n5, final String axesName) throws IOException { + + final RandomAccessibleInterval axesLookup = open(n5, AXES_PREFIX + axesName); + + final int n = axesLookup.numDimensions(); + final int[] axes = new int[n]; + + final RandomAccess ra = axesLookup.randomAccess(axesLookup); + for (int d = 0; d < n; ++d) { + ra.fwd(d); + axes[d] = ra.get().get(); + ra.bck(d); + } + + return axes; + } + + + /** + * Read a double vector attribute and permute the axes as specified by an + * axes index lookup. + * + * Examples: + *

+ * getDoubleVector(n5, "/volumes/raw", "resolution", 2, 1, 0) + * + * reads a 3D vector that is stored in C-order as `[3.3, 2.2, 1.1]` as + * `[1.1, 2.2, 3.3]`. + *

+ *

+ * getDoubleVector(n5, "/image", "offset", 0, 1) + * + * reads a 2D vector that is stored in F-order as `[1.1, 2.2]` as + * `[1.1, 2.2]`. + *

+ * + * @param n5 reader + * @param groupName + * @param key + * @param axes axes index lookup + * @return + * @throws IOException + */ + public static final double[] getDoubleVector( + final N5Reader n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final double[] vector = n5.getAttribute(groupName, key, double[].class); + final double[] copy = new double[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[d] = vector[axes[d]]; + return copy; + } + + + /** + * Read a float vector attribute and permute the axes as specified by an + * axes index lookup. + * + * Examples: + *

+ * getFloatVector(n5, "/volumes/raw", "resolution", 2, 1, 0) + * + * reads a 3D vector that is stored in C-order as `[3.3, 2.2, 1.1]` as + * `[1.1, 2.2, 3.3]`. + *

+ *

+ * getFloatVector(n5, "/image", "offset", 0, 1) + * + * reads a 2D vector that is stored in F-order as `[1.1, 2.2]` as + * `[1.1, 2.2]`. + *

+ * + * @param n5 reader + * @param groupName + * @param key + * @param axes axes index lookup + * @return + * @throws IOException + */ + public static final float[] getFloatVector( + final N5Reader n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final float[] vector = n5.getAttribute(groupName, key, float[].class); + final float[] copy = new float[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[d] = vector[axes[d]]; + return copy; + } + + + /** + * Read an long vector attribute and permute the axes as specified by an + * axes index lookup. + * + * Examples: + *

+ * getLongVector(n5, "/volumes/raw", "resolution", 2, 1, 0) + * + * reads a 3D vector that is stored in C-order as `[30, 20, 10]` as + * `[10, 20, 30]`. + *

+ *

+ * getLongVector(n5, "/image", "offset", 0, 1) + * + * reads a 2D vector that is stored in F-order as `[10, 20]` as + * `[10, 20]`. + *

+ * + * @param n5 reader + * @param groupName + * @param key + * @param axes axes index lookup + * @return + * @throws IOException + */ + public static final long[] getLongVector( + final N5Reader n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final long[] vector = n5.getAttribute(groupName, key, long[].class); + final long[] copy = new long[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[d] = vector[axes[d]]; + return copy; + } + + + /** + * Read an int vector attribute and permute the axes as specified by an + * axes index lookup. + * + * Examples: + *

+ * getIntVector(n5, "/volumes/raw", "offset", 2, 1, 0) + * + * reads a 3D vector that is stored in C-order as `[30, 20, 10]` as + * `[10, 20, 30]`. + *

+ *

+ * getIntVector(n5, "/image", "offset", 0, 1) + * + * reads a 2D vector that is stored in F-order as `[10, 20]` as + * `[10, 20]`. + *

+ * + * @param n5 reader + * @param groupName + * @param key + * @param axes axes index lookup + * @return + * @throws IOException + */ + public static final int[] getIntVector( + final N5Reader n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final int[] vector = n5.getAttribute(groupName, key, int[].class); + final int[] copy = new int[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[d] = vector[axes[d]]; + return copy; + } + + /** + * Read a short vector attribute and permute the axes as specified by an + * axes index lookup. + * + * Examples: + *

+ * getShortVector(n5, "/volumes/labels", "background", 2, 1, 0) + * + * reads a 3D vector that is stored in BGR-order as `[30, 20, 10]` as + * RGB-ordered `[10, 20, 30]`. + *

+ * + * @param n5 reader + * @param groupName + * @param key + * @param axes axes index lookup + * @return + * @throws IOException + */ + public static final short[] getShortVector( + final N5Reader n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final short[] vector = n5.getAttribute(groupName, key, short[].class); + final short[] copy = new short[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[d] = vector[axes[d]]; + return copy; + } + + + /** + * Read a byte vector attribute and permute the axes as specified by an + * axes index lookup. + * + * Examples: + *

+ * getByteVector(n5, "/volumes/labels", "background", 2, 1, 0) + * + * reads a 3D vector that is stored in BGR-order as `[30, 20, 10]` as + * RGB-ordered `[10, 20, 30]`. + *

+ * + * @param n5 reader + * @param groupName + * @param key + * @param axes axes index lookup + * @return + * @throws IOException + */ + public static final byte[] getByteVector( + final N5Reader n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final byte[] vector = n5.getAttribute(groupName, key, byte[].class); + final byte[] copy = new byte[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[d] = vector[axes[d]]; + return copy; + } + + + /** + * Read a vector of T attribute and permute the axes as + * specified by an axes index lookup. + * + * Examples: + *

+ * getVector(n5, "/volumes/raw", "axesNames", 2, 1, 0) + * + * reads a 3D vector that is stored in C-order as `["z", "y", "x"]` as + * `["x", "y", "z"]`. + *

+ *

+ * getVector(n5, "/image", "axesNames", 0, 1) + * + * reads a 2D vector that is stored in F-order as `["x", "y"]` as + * `["x", "y"]`. + *

+ * + * @param T scalar type + * @param n5 reader + * @param groupName + * @param key + * @param axes axes index lookup + * @return + * @throws IOException + */ + public static final T[] getVector( + final N5Reader n5, + final String groupName, + final String key, + final Class clazz, + final int... axes) throws IOException { + + @SuppressWarnings("unchecked") + final T[] vector = (T[])n5.getAttribute(groupName, key, Object[].class); + final T[] copy = vector.clone(); + for (int d = 0; d < axes.length; ++d) + copy[d] = vector[axes[d]]; + return copy; + } + + + /** + * Write a double vector attribute with its axes permuted as specified by + * an axes index lookup. + * + * Examples: + *

+ * setVector(new double[]{1.1, 2.2, 3.3}, n5, "/volumes/raw", "resolution", 2, 1, 0) + * + * stores a 3D vector `[1.1, 2.2, 3.3]` in C-order as `[3.3, 2.2, 1.1]`. + *

+ *

+ * setVector(new double[]{1.1, 2.2}, n5, "/image", "offset", 0, 1) + * + * stores a 2D vector `[1.1, 2.2]` in F-order as `[1.1, 2.2]`. + *

+ * + * @param vector + * @param n5 + * @param groupName + * @param key + * @param axes + * @throws IOException + */ + public static final void setVector( + final double[] vector, + final N5Writer n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final double[] copy = new double[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[axes[d]] = vector[d]; + + n5.setAttribute(groupName, key, copy); + } + + + /** + * Write a float vector attribute with its axes permuted as specified by + * an axes index lookup. + * + * Examples: + *

+ * setVector(new float[]{1.1f, 2.2f, 3.3f}, n5, "/volumes/raw", "resolution", 2, 1, 0) + * + * stores a 3D vector `[1.1, 2.2, 3.3]` in C-order as `[3.3, 2.2, 1.1]`. + *

+ *

+ * setVector(new float[]{1.1f, 2.2f}, n5, "/image", "offset", 0, 1) + * + * stores a 2D vector `[1.1, 2.2]` in F-order as `[1.1, 2.2]`. + *

+ * + * @param vector + * @param n5 + * @param groupName + * @param key + * @param axes + * @throws IOException + */ + public static final void setVector( + final float[] vector, + final N5Writer n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final float[] copy = new float[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[axes[d]] = vector[d]; + + n5.setAttribute(groupName, key, copy); + } + + + /** + * Write a long vector attribute with its axes permuted as specified by + * an axes index lookup. + * + * Examples: + *

+ * setVector(new long[]{10, 20, 30}, n5, "/volumes/raw", "offset", 2, 1, 0) + * + * stores a 3D vector `[10, 20, 30]` in C-order as `[30, 20, 10]`. + *

+ *

+ * setVector(new long[]{10, 20}, n5, "/image", "offset", 0, 1) + * + * stores a 2D vector `[10, 20]` in F-order as `[10, 20]`. + *

+ * + * @param vector + * @param n5 + * @param groupName + * @param key + * @param axes + * @throws IOException + */ + public static final void setVector( + final long[] vector, + final N5Writer n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final long[] copy = new long[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[axes[d]] = vector[d]; + + n5.setAttribute(groupName, key, copy); + } + + + /** + * Write an int vector attribute with its axes permuted as specified by + * an axes index lookup. + * + * Examples: + *

+ * setVector(new int[]{10, 20, 30}, n5, "/volumes/raw", "offset", 2, 1, 0) + * + * stores a 3D vector `[10, 20, 30]` in C-order as `[30, 20, 10]`. + *

+ *

+ * setVector(new int[]{10, 20}, n5, "/image", "offset", 0, 1) + * + * stores a 2D vector `[10, 20]` in F-order as `[10, 20]`. + *

+ * + * @param vector + * @param n5 + * @param groupName + * @param key + * @param axes + * @throws IOException + */ + public static final void setVector( + final int[] vector, + final N5Writer n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final int[] copy = new int[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[axes[d]] = vector[d]; + + n5.setAttribute(groupName, key, copy); + } + + + /** + * Write a short vector attribute with its axes permuted as specified by + * an axes index lookup. + * + * Examples: + *

+ * setVector(new short[]{10, 20, 30}, n5, "/volumes/labels", "background", 2, 1, 0) + * + * stores an RGB vector `[10, 20, 30]` in BGR-order as `[30, 20, 10]`. + *

+ * + * @param vector + * @param n5 + * @param groupName + * @param key + * @param axes + * @throws IOException + */ + public static final void setVector( + final short[] vector, + final N5Writer n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final short[] copy = new short[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[axes[d]] = vector[d]; + + n5.setAttribute(groupName, key, copy); + } + + + /** + * Write a byte vector attribute with its axes permuted as specified by + * an axes index lookup. + * + * Examples: + *

+ * setVector(new byte[]{10, 20, 30}, n5, "/volumes/labels", "background", 2, 1, 0) + * + * stores an RGB vector `[10, 20, 30]` in BGR-order as `[30, 20, 10]`. + *

+ * + * @param vector + * @param n5 + * @param groupName + * @param key + * @param axes + * @throws IOException + */ + public static final void setVector( + final byte[] vector, + final N5Writer n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final byte[] copy = new byte[vector.length]; + for (int d = 0; d < axes.length; ++d) + copy[axes[d]] = vector[d]; + + n5.setAttribute(groupName, key, copy); + } + + + /** + * Write a vector of T attribute with its axes permuted as + * specified by an axes index lookup. + * + * Examples: + *

+ * setVector(new String[]{"x", "y", "z"}, n5, "/volumes/raw", "axesNames", 2, 1, 0) + * + * stores a 3D vector `["x", "y", "z"]` in C-order as `["z", "y", "x"]`. + *

+ *

+ * setVector(new String[]{"x", "y"}, n5, "/image", "axesNames", 0, 1) + * + * stores a 2D vector `["x", "y"]` in F-order as `["x", "y"]`. + *

+ * + * @param vector + * @param n5 + * @param groupName + * @param key + * @param axes + * @throws IOException + */ + public static final void setVector( + final T[] vector, + final N5Writer n5, + final String groupName, + final String key, + final int... axes) throws IOException { + + final T[] copy = vector.clone(); + for (int d = 0; d < axes.length; ++d) + copy[axes[d]] = vector[d]; + + n5.setAttribute(groupName, key, copy); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5CellLoader.java b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5CellLoader.java index a54d6af..dc3a18f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5CellLoader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5CellLoader.java @@ -1,5 +1,6 @@ /** - * Copyright (c) 2017-2018, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5DisplacementField.java b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5DisplacementField.java index 010029f..36a1953 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5DisplacementField.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5DisplacementField.java @@ -1,3 +1,30 @@ +/** + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ package org.janelia.saalfeldlab.n5.imglib2; import java.io.IOException; @@ -8,8 +35,6 @@ import org.janelia.saalfeldlab.n5.N5Writer; import net.imglib2.Cursor; -import net.imglib2.FinalInterval; -import net.imglib2.Interval; import net.imglib2.IterableInterval; import net.imglib2.RandomAccessible; import net.imglib2.RandomAccessibleInterval; @@ -20,15 +45,10 @@ import net.imglib2.interpolation.randomaccess.NLinearInterpolatorFactory; import net.imglib2.realtransform.AffineGet; import net.imglib2.realtransform.AffineTransform; -import net.imglib2.realtransform.AffineTransform2D; -import net.imglib2.realtransform.AffineTransform3D; -import net.imglib2.realtransform.ExplicitInvertibleRealTransform; import net.imglib2.realtransform.DeformationFieldTransform; -import net.imglib2.realtransform.InverseRealTransform; +import net.imglib2.realtransform.ExplicitInvertibleRealTransform; import net.imglib2.realtransform.InvertibleRealTransform; -import net.imglib2.realtransform.InvertibleRealTransformSequence; import net.imglib2.realtransform.RealTransform; -import net.imglib2.realtransform.RealTransformRandomAccessible; import net.imglib2.realtransform.RealTransformSequence; import net.imglib2.realtransform.RealViews; import net.imglib2.transform.integer.MixedTransform; @@ -52,7 +72,7 @@ /** * Class with helper methods for saving displacement field transformations as N5 datasets. - * + * * @author John Bogovic * */ @@ -78,9 +98,9 @@ public class N5DisplacementField * @param fwdspacing the pixel spacing (resolution) of the forward deformation field * @param inverseDfield * @param invspacing the pixel spacing (resolution) of the inverse deformation field - * @param blockSize + * @param blockSize * @param compression - */ + */ public static final & RealType> void save( final N5Writer n5Writer, final AffineGet affine, @@ -131,7 +151,7 @@ public static final & RealType> void save( * The deformation field here is saved as an {@link IntegerType} * which could compress better in some cases. The multiplier from * original values to compressed values is chosen as the smallest - * value that keeps the error (L2) between quantized and original vectors. + * value that keeps the error (L2) between quantized and original vectors. * * @param n5Writer * @param dataset @@ -150,11 +170,11 @@ public static final & RealType, Q extends NativeType final RandomAccessibleInterval< T > dfield, final double[] spacing, final int[] blockSize, - final Compression compression, - final Q outputType, + final Compression compression, + final Q outputType, final double maxError ) throws Exception { - + saveQuantized( n5Writer, dataset, dfield, blockSize, compression, outputType, maxError ); saveAffine( affine, n5Writer, dataset ); if( spacing != null ) @@ -184,7 +204,7 @@ public static final void saveAffine( * The deformation field here is saved as an {@link IntegerType} * which could compress better in some cases. The multiplier from * original values to compressed values is chosen as the smallest - * value that keeps the error (L2) between quantized and original vectors. + * value that keeps the error (L2) between quantized and original vectors. * * @param n5Writer * @param dataset @@ -203,24 +223,24 @@ public static final , Q extends NativeType & IntegerTyp final Q outputType, final double maxError ) throws Exception { - /* - * To keep the max vector error below maxError, + /* + * To keep the max vector error below maxError, * the error per coordinate must be below m */ - int nd = ( source.numDimensions() - 1 ); // vector field source has num dims + 1 - double m = 2 * Math.sqrt( maxError * maxError / nd ); + final int nd = ( source.numDimensions() - 1 ); // vector field source has num dims + 1 + final double m = 2 * Math.sqrt( maxError * maxError / nd ); - RandomAccessibleInterval< T > source_permuted = vectorAxisFirst( source ); - RandomAccessibleInterval< Q > source_quant = Converters.convert( - source_permuted, + final RandomAccessibleInterval< T > source_permuted = vectorAxisFirst( source ); + final RandomAccessibleInterval< Q > source_quant = Converters.convert( + source_permuted, new Converter() { @Override - public void convert(T input, Q output) + public void convert(final T input, final Q output) { output.setInteger( Math.round( input.getRealDouble() / m )); } - }, + }, outputType.copy()); N5Utils.save( source_quant, n5Writer, dataset, blockSize, compression); @@ -228,7 +248,7 @@ public void convert(T input, Q output) } /** - * Opens an {@link InvertibleRealTransform} from an n5 object. Uses the provided datasets as the forward and + * Opens an {@link InvertibleRealTransform} from an n5 object. Uses the provided datasets as the forward and * inverse transformations. * * @param n5 @@ -238,7 +258,7 @@ public void convert(T input, Q output) * @param interpolator * @return the invertible transformation */ - public static final & NativeType> ExplicitInvertibleRealTransform openInvertible( + public static final & NativeType> ExplicitInvertibleRealTransform openInvertible( final N5Reader n5, final String forwardDataset, final String inverseDataset, @@ -256,7 +276,7 @@ public static final & NativeType> ExplicitInvertibleRe * @param n5 * @return the invertible transformation */ - public static final & NativeType> ExplicitInvertibleRealTransform openInvertible( + public static final & NativeType> ExplicitInvertibleRealTransform openInvertible( final N5Reader n5 ) throws Exception { return openInvertible( n5, FORWARD_ATTR, INVERSE_ATTR, new DoubleType(), new NLinearInterpolatorFactory<>() ); @@ -267,10 +287,10 @@ public static final & NativeType> ExplicitInvertibleRe * using linear interpolation. * * @param n5 - * @param level + * @param level * @return the invertible transformation */ - public static final & NativeType> ExplicitInvertibleRealTransform openInvertible( + public static final & NativeType> ExplicitInvertibleRealTransform openInvertible( final N5Reader n5, final int level ) throws Exception { @@ -286,7 +306,7 @@ public static final & NativeType> ExplicitInvertibleRe * @param interpolator * @return the invertible transformation */ - public static final & NativeType> ExplicitInvertibleRealTransform openInvertible( + public static final & NativeType> ExplicitInvertibleRealTransform openInvertible( final N5Reader n5, final T defaultType, final InterpolatorFactory< T, RandomAccessible > interpolator ) throws Exception @@ -306,7 +326,7 @@ public static final & NativeType> ExplicitInvertibleRe * @param interpolator * @return the transformation */ - public static final & NativeType> RealTransform open( + public static final & NativeType> RealTransform open( final N5Reader n5, final String dataset, final boolean inverse, @@ -314,14 +334,14 @@ public static final & NativeType> RealTransform open( final InterpolatorFactory< T, RandomAccessible > interpolator ) throws Exception { - AffineGet affine = openAffine( n5, dataset ); + final AffineGet affine = openAffine( n5, dataset ); - DeformationFieldTransform< T > dfield = new DeformationFieldTransform<>( + final DeformationFieldTransform< T > dfield = new DeformationFieldTransform<>( openCalibratedField( n5, dataset, interpolator, defaultType )); if( affine != null ) { - RealTransformSequence xfmSeq = new RealTransformSequence(); + final RealTransformSequence xfmSeq = new RealTransformSequence(); if( inverse ) { xfmSeq.add( affine ); @@ -351,12 +371,12 @@ public static final & NativeType> RealTransform open( */ public static final AffineGet openPixelToPhysical( final N5Reader n5, final String dataset ) throws Exception { - double[] spacing = n5.getAttribute( dataset, SPACING_ATTR, double[].class ); + final double[] spacing = n5.getAttribute( dataset, SPACING_ATTR, double[].class ); if ( spacing == null ) return null; // have to bump the dimension up by one to apply it to the displacement field - int N = spacing.length; + final int N = spacing.length; final AffineTransform affineMtx; if ( N == 1 ) affineMtx = new AffineTransform( 2 ); @@ -382,11 +402,11 @@ else if ( N == 3 ) */ public static final AffineGet openAffine( final N5Reader n5, final String dataset ) throws Exception { - double[] affineMtxRow = n5.getAttribute( dataset, AFFINE_ATTR, double[].class ); + final double[] affineMtxRow = n5.getAttribute( dataset, AFFINE_ATTR, double[].class ); if ( affineMtxRow == null ) return null; - int N = affineMtxRow.length; + final int N = affineMtxRow.length; final AffineTransform affineMtx; if ( N == 2 ) affineMtx = new AffineTransform( 1 ); @@ -414,7 +434,7 @@ else if ( N == 12 ) * @return the deformation field as a RandomAccessibleInterval */ @SuppressWarnings( "unchecked" ) - public static final & RealType, Q extends NativeType & IntegerType> RandomAccessibleInterval< T > openField( + public static final & RealType, Q extends NativeType & IntegerType> RandomAccessibleInterval< T > openField( final N5Reader n5, final String dataset, final T defaultType ) throws Exception @@ -460,7 +480,7 @@ public static final & RealType, Q extends NativeType */ public static < T extends NativeType< T > & RealType< T > > RealRandomAccessible< T > openCalibratedField( final N5Reader n5, final String dataset, - final InterpolatorFactory< T, RandomAccessible< T > > interpolator, + final InterpolatorFactory< T, RandomAccessible< T > > interpolator, final T defaultType ) throws Exception { return openCalibratedField( n5, dataset, interpolator, EXTEND_BORDER, defaultType ); @@ -478,19 +498,19 @@ public static < T extends NativeType< T > & RealType< T > > RealRandomAccessible * * @param n5 the n5 reader * @param dataset the n5 dataset - * @param interpolator the type of interpolation + * @param interpolator the type of interpolation * @param extensionType the type of out-of-bounds extension * @param defaultType the type * @return the deformation field as a RealRandomAccessible */ public static < T extends NativeType< T > & RealType< T > > RealRandomAccessible< T > openCalibratedField( final N5Reader n5, final String dataset, - final InterpolatorFactory< T, RandomAccessible< T > > interpolator, + final InterpolatorFactory< T, RandomAccessible< T > > interpolator, final String extensionType, final T defaultType ) throws Exception { - RandomAccessibleInterval< T > dfieldRai = openField( n5, dataset, defaultType ); - RandomAccessibleInterval< T > dfieldRaiPerm = vectorAxisLast( dfieldRai ); + final RandomAccessibleInterval< T > dfieldRai = openField( n5, dataset, defaultType ); + final RandomAccessibleInterval< T > dfieldRaiPerm = vectorAxisLast( dfieldRai ); if ( dfieldRai == null ) { @@ -524,7 +544,7 @@ else if( extensionType.equals( EXTEND_BORDER )){ * @return the transform */ public static < T extends NativeType< T > & RealType< T > > RealTransform open( - final N5Reader n5, final String dataset, boolean inverse ) throws Exception + final N5Reader n5, final String dataset, final boolean inverse ) throws Exception { return open( n5, dataset, inverse, new FloatType(), new NLinearInterpolatorFactory()); } @@ -543,7 +563,7 @@ public static final < T extends RealType & NativeType > RandomAccessibleIn final String dataset, final T defaultType ) throws Exception { - RandomAccessibleInterval< T > src = N5Utils.open( n5, dataset, defaultType ); + final RandomAccessibleInterval< T > src = N5Utils.open( n5, dataset, defaultType ); return vectorAxisLast( src ); } @@ -563,28 +583,28 @@ public static final & NativeType, T extends RealType src = N5Utils.open( n5, dataset, defaultQuantizedType ); - + final RandomAccessibleInterval< Q > src = N5Utils.open( n5, dataset, defaultQuantizedType ); + // get the factor going from quantized to original values - Double mattr = n5.getAttribute( dataset, MULTIPLIER_ATTR, Double.TYPE ); + final Double mattr = n5.getAttribute( dataset, MULTIPLIER_ATTR, Double.TYPE ); final double m; if( mattr != null ) m = mattr.doubleValue(); else m = 1.0; - RandomAccessibleInterval< Q > src_perm = vectorAxisLast( src ); - RandomAccessibleInterval< T > src_converted = Converters.convert( - src_perm, + final RandomAccessibleInterval< Q > src_perm = vectorAxisLast( src ); + final RandomAccessibleInterval< T > src_converted = Converters.convert( + src_perm, new Converter() { @Override - public void convert(Q input, T output) { + public void convert(final Q input, final T output) { output.setReal( input.getRealDouble() * m ); } - }, + }, defaultType.copy()); - + return src_converted; } @@ -596,7 +616,7 @@ public void convert(Q input, T output) { * @return the possibly permuted deformation field * @throws Exception */ - public static final < T extends RealType< T > > RandomAccessibleInterval< T > vectorAxisLast( RandomAccessibleInterval< T > source ) throws Exception + public static final < T extends RealType< T > > RandomAccessibleInterval< T > vectorAxisLast( final RandomAccessibleInterval< T > source ) throws Exception { final int n = source.numDimensions(); @@ -612,11 +632,11 @@ else if ( source.dimension( 0 ) == (n - 1) ) return permute( source, component ); } - throw new Exception( - String.format( "Displacement fields must store vector components in the first or last dimension. " + + throw new Exception( + String.format( "Displacement fields must store vector components in the first or last dimension. " + "Found a %d-d volume; expect size [%d,...] or [...,%d]", n, ( n - 1 ), ( n - 1 ) ) ); } - + /** * Returns a deformation field as a {@link RandomAccessibleInterval} * with the vector stored in the first dimension. @@ -625,10 +645,10 @@ else if ( source.dimension( 0 ) == (n - 1) ) * @return the possibly permuted deformation field * @throws Exception */ - public static final < T extends RealType< T > > RandomAccessibleInterval< T > vectorAxisFirst( RandomAccessibleInterval< T > source ) throws Exception + public static final < T extends RealType< T > > RandomAccessibleInterval< T > vectorAxisFirst( final RandomAccessibleInterval< T > source ) throws Exception { final int n = source.numDimensions(); - + System.out.println("source2perm: " + Util.printInterval(source)); if ( source.dimension( 0 ) == (n - 1) ) @@ -643,21 +663,21 @@ else if ( source.dimension( n - 1 ) == (n - 1) ) return permute( source, component ); } - throw new Exception( - String.format( "Displacement fields must store vector components in the first or last dimension. " + + throw new Exception( + String.format( "Displacement fields must store vector components in the first or last dimension. " + "Found a %d-d volume; expect size [%d,...] or [...,%d]", n, ( n - 1 ), ( n - 1 ) ) ); } /** * Permutes the dimensions of a {@link RandomAccessibleInterval} * using the given permutation vector, where the ith value in p - * gives destination of the ith input dimension in the output. + * gives destination of the ith input dimension in the output. * * @param source the source data * @param p the permutation * @return the permuted source */ - public static final < T > IntervalView< T > permute( RandomAccessibleInterval< T > source, int[] p ) + public static final < T > IntervalView< T > permute( final RandomAccessibleInterval< T > source, final int[] p ) { final int n = source.numDimensions(); @@ -683,17 +703,17 @@ public static final < T > IntervalView< T > permute( RandomAccessibleInterval< T * @param img the iterable interval * @return the min and max values stored in a double array */ - public static > double[] getMinMax( IterableInterval img ) + public static > double[] getMinMax( final IterableInterval img ) { double min = Double.MAX_VALUE; double max = Double.MIN_VALUE; - Cursor c = img.cursor(); + final Cursor c = img.cursor(); while( c.hasNext() ) { - double v = Math.abs( c.next().getRealDouble()); + final double v = Math.abs( c.next().getRealDouble()); if( v > max ) max = v; - + if( v < min ) min = v; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisetCacheLoader.java b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisetCacheLoader.java index 16783c8..6edeee8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisetCacheLoader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisetCacheLoader.java @@ -1,3 +1,30 @@ +/** + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ package org.janelia.saalfeldlab.n5.imglib2; import java.io.IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisets.java b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisets.java index db4c858..c46224c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisets.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisets.java @@ -1,3 +1,30 @@ +/** + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ package org.janelia.saalfeldlab.n5.imglib2; import java.io.IOException; @@ -169,7 +196,7 @@ public static final void saveLabelMultiset( final int[] intCroppedBlockSize = new int[n]; final long[] longCroppedBlockSize = new long[n]; for (int d = 0; d < n;) { - N5Utils.cropBlockDimensions(max, offset, blockSize, longCroppedBlockSize, intCroppedBlockSize, gridPosition); + N5.cropBlockDimensions(max, offset, blockSize, longCroppedBlockSize, intCroppedBlockSize, gridPosition); final RandomAccessibleInterval sourceBlock = Views.offsetInterval(source, offset, longCroppedBlockSize); final ByteArrayDataBlock dataBlock = createDataBlock(sourceBlock, gridPosition); @@ -233,7 +260,7 @@ public static final void saveLabelMultiset( final int[] intCroppedBlockSize = new int[n]; final long[] longCroppedBlockSize = new long[n]; - N5Utils.cropBlockDimensions( + N5.cropBlockDimensions( max, fOffset, blockSize, @@ -295,7 +322,7 @@ public static final void saveLabelMultisetBlock( final int[] intCroppedBlockSize = new int[n]; final long[] longCroppedBlockSize = new long[n]; for (int d = 0; d < n;) { - N5Utils.cropBlockDimensions( + N5.cropBlockDimensions( max, offset, gridOffset, @@ -432,7 +459,7 @@ public static final void saveLabelMultisetBlock( final int[] intCroppedBlockSize = new int[n]; final long[] longCroppedBlockSize = new long[n]; - N5Utils.cropBlockDimensions( + N5.cropBlockDimensions( max, fOffset, gridOffset, @@ -504,7 +531,7 @@ public static final void saveLabelMultisetNonEmptyBlock( final int[] intCroppedBlockSize = new int[n]; final long[] longCroppedBlockSize = new long[n]; for (int d = 0; d < n;) { - N5Utils.cropBlockDimensions( + N5.cropBlockDimensions( max, offset, gridOffset, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5Utils.java b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5Utils.java index becd34d..7181bea 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5Utils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5Utils.java @@ -1,5 +1,6 @@ /** - * Copyright (c) 2017-2018, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -27,12 +28,9 @@ package org.janelia.saalfeldlab.n5.imglib2; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.IntFunction; @@ -45,43 +43,18 @@ import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; -import net.imglib2.Interval; import net.imglib2.IterableInterval; import net.imglib2.RandomAccessibleInterval; -import net.imglib2.cache.Cache; import net.imglib2.cache.LoaderCache; import net.imglib2.cache.img.CachedCellImg; -import net.imglib2.cache.img.DiskCachedCellImgFactory; -import net.imglib2.cache.img.DiskCachedCellImgOptions; -import net.imglib2.cache.img.LoadedCellCacheLoader; -import net.imglib2.cache.ref.BoundedSoftRefLoaderCache; -import net.imglib2.cache.ref.SoftRefLoaderCache; -import net.imglib2.img.array.ArrayImgs; import net.imglib2.img.basictypeaccess.AccessFlags; -import net.imglib2.img.basictypeaccess.ArrayDataAccessFactory; import net.imglib2.img.basictypeaccess.array.ArrayDataAccess; import net.imglib2.img.basictypeaccess.volatiles.VolatileAccess; import net.imglib2.img.cell.Cell; -import net.imglib2.img.cell.CellGrid; import net.imglib2.img.cell.LazyCellImg; import net.imglib2.type.NativeType; -import net.imglib2.type.Type; import net.imglib2.type.label.LabelMultisetType; -import net.imglib2.type.numeric.integer.ByteType; -import net.imglib2.type.numeric.integer.IntType; -import net.imglib2.type.numeric.integer.LongType; -import net.imglib2.type.numeric.integer.ShortType; -import net.imglib2.type.numeric.integer.UnsignedByteType; -import net.imglib2.type.numeric.integer.UnsignedIntType; -import net.imglib2.type.numeric.integer.UnsignedLongType; -import net.imglib2.type.numeric.integer.UnsignedShortType; -import net.imglib2.type.numeric.real.DoubleType; -import net.imglib2.type.numeric.real.FloatType; -import net.imglib2.util.Intervals; import net.imglib2.util.Pair; -import net.imglib2.util.Util; -import net.imglib2.util.ValuePair; -import net.imglib2.view.Views; /** * Static utility methods to open N5 datasets as ImgLib2 @@ -91,293 +64,23 @@ * * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> * @author Philipp Hanslovsky <hanslovskyp@janelia.hhmi.org> + * + * @deprecated use {@link #N5} instead */ -public class N5Utils { - - private N5Utils() {} - - public static final > DataType dataType(final T type) { - - if (DoubleType.class.isInstance(type)) - return DataType.FLOAT64; - if (FloatType.class.isInstance(type)) - return DataType.FLOAT32; - if (LongType.class.isInstance(type)) - return DataType.INT64; - if (UnsignedLongType.class.isInstance(type)) - return DataType.UINT64; - if (IntType.class.isInstance(type)) - return DataType.INT32; - if (UnsignedIntType.class.isInstance(type)) - return DataType.UINT32; - if (ShortType.class.isInstance(type)) - return DataType.INT16; - if (UnsignedShortType.class.isInstance(type)) - return DataType.UINT16; - if (ByteType.class.isInstance(type)) - return DataType.INT8; - if (UnsignedByteType.class.isInstance(type)) - return DataType.UINT8; - else - return null; - } - - @SuppressWarnings("unchecked") - public static final > T type(final DataType dataType) { - - switch (dataType) { - case INT8: - return (T) new ByteType(); - case UINT8: - return (T) new UnsignedByteType(); - case INT16: - return (T) new ShortType(); - case UINT16: - return (T) new UnsignedShortType(); - case INT32: - return (T) new IntType(); - case UINT32: - return (T) new UnsignedIntType(); - case INT64: - return (T) new LongType(); - case UINT64: - return (T) new UnsignedLongType(); - case FLOAT32: - return (T) new FloatType(); - case FLOAT64: - return (T) new DoubleType(); - default: - return null; - } - } +@Deprecated +public interface N5Utils { - /** - * Creates a {@link DataBlock} of matching type and copies the content of - * source into it. This is a helper method with redundant parameters. - * - * @param source - * @param dataType - * @param intBlockSize - * @param longBlockSize - * @param gridPosition - * @return - */ - @SuppressWarnings("unchecked") - private static final DataBlock createDataBlock( - final RandomAccessibleInterval source, - final DataType dataType, - final int[] intBlockSize, - final long[] longBlockSize, - final long[] gridPosition) { - - final DataBlock dataBlock = dataType.createDataBlock(intBlockSize, gridPosition); - switch (dataType) { - case UINT8: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.unsignedBytes((byte[])dataBlock.getData(), longBlockSize)); - break; - case INT8: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.bytes((byte[])dataBlock.getData(), longBlockSize)); - break; - case UINT16: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.unsignedShorts((short[])dataBlock.getData(), longBlockSize)); - break; - case INT16: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.shorts((short[])dataBlock.getData(), longBlockSize)); - break; - case UINT32: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.unsignedInts((int[])dataBlock.getData(), longBlockSize)); - break; - case INT32: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.ints((int[])dataBlock.getData(), longBlockSize)); - break; - case UINT64: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.unsignedLongs((long[])dataBlock.getData(), longBlockSize)); - break; - case INT64: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.longs((long[])dataBlock.getData(), longBlockSize)); - break; - case FLOAT32: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.floats((float[])dataBlock.getData(), longBlockSize)); - break; - case FLOAT64: - N5CellLoader.burnIn( - (RandomAccessibleInterval)source, - ArrayImgs.doubles((double[])dataBlock.getData(), longBlockSize)); - break; - default: - throw new IllegalArgumentException("Type " + dataType.name() + " not supported!"); - } + public static > DataType dataType(final T type) { - return dataBlock; + return N5.dataType(type); } - /** - * Creates a {@link DataBlock} of matching type and copies the content of - * source into it. This is a helper method with redundant parameters. - * - * @param source - * @param dataType - * @param intBlockSize - * @param longBlockSize - * @param gridPosition - * @param defaultValue - * @return - */ @SuppressWarnings("unchecked") - private static final > DataBlock createNonEmptyDataBlock( - final RandomAccessibleInterval source, - final DataType dataType, - final int[] intBlockSize, - final long[] longBlockSize, - final long[] gridPosition, - final T defaultValue) { - - final DataBlock dataBlock = dataType.createDataBlock(intBlockSize, gridPosition); - final boolean isEmpty; - switch (dataType) { - case UINT8: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.unsignedBytes((byte[])dataBlock.getData(), longBlockSize), - (UnsignedByteType)defaultValue); - break; - case INT8: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.bytes((byte[])dataBlock.getData(), longBlockSize), - (ByteType)defaultValue); - break; - case UINT16: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.unsignedShorts((short[])dataBlock.getData(), longBlockSize), - (UnsignedShortType)defaultValue); - break; - case INT16: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.shorts((short[])dataBlock.getData(), longBlockSize), - (ShortType)defaultValue); - break; - case UINT32: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.unsignedInts((int[])dataBlock.getData(), longBlockSize), - (UnsignedIntType)defaultValue); - break; - case INT32: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.ints((int[])dataBlock.getData(), longBlockSize), - (IntType)defaultValue); - break; - case UINT64: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.unsignedLongs((long[])dataBlock.getData(), longBlockSize), - (UnsignedLongType)defaultValue); - break; - case INT64: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.longs((long[])dataBlock.getData(), longBlockSize), - (LongType)defaultValue); - break; - case FLOAT32: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.floats((float[])dataBlock.getData(), longBlockSize), - (FloatType)defaultValue); - break; - case FLOAT64: - isEmpty = N5CellLoader.burnInTestAllEqual( - (RandomAccessibleInterval)source, - ArrayImgs.doubles((double[])dataBlock.getData(), longBlockSize), - (DoubleType)defaultValue); - break; - default: - throw new IllegalArgumentException("Type " + dataType.name() + " not supported!"); - } + public static > T type(final DataType dataType) { - return isEmpty ? null : dataBlock; + return N5.type(dataType); } - /** - * Crops the dimensions of a {@link DataBlock} at a given offset to fit into - * and {@link Interval} of given dimensions. Fills long and int version of - * cropped block size. Also calculates the grid raster position assuming - * that the offset divisible by block size without remainder. - * - * @param max - * @param offset - * @param blockDimensions - * @param croppedBlockDimensions - * @param intCroppedBlockDimensions - * @param gridPosition - */ - static void cropBlockDimensions( - final long[] max, - final long[] offset, - final int[] blockDimensions, - final long[] croppedBlockDimensions, - final int[] intCroppedBlockDimensions, - final long[] gridPosition) { - - for (int d = 0; d < max.length; ++d) { - croppedBlockDimensions[d] = Math.min(blockDimensions[d], max[d] - offset[d] + 1); - intCroppedBlockDimensions[d] = (int)croppedBlockDimensions[d]; - gridPosition[d] = offset[d] / blockDimensions[d]; - } - } - - /** - * Crops the dimensions of a {@link DataBlock} at a given offset to fit into - * and {@link Interval} of given dimensions. Fills long and int version of - * cropped block size. Also calculates the grid raster position plus a grid - * offset assuming that the offset divisible by block size without - * remainder. - * - * @param max - * @param offset - * @param gridOffset - * @param blockDimensions - * @param croppedBlockDimensions - * @param intCroppedBlockDimensions - * @param gridPosition - */ - static void cropBlockDimensions( - final long[] max, - final long[] offset, - final long[] gridOffset, - final int[] blockDimensions, - final long[] croppedBlockDimensions, - final int[] intCroppedBlockDimensions, - final long[] gridPosition) { - - for (int d = 0; d < max.length; ++d) { - croppedBlockDimensions[d] = Math.min(blockDimensions[d], max[d] - offset[d] + 1); - intCroppedBlockDimensions[d] = (int)croppedBlockDimensions[d]; - gridPosition[d] = offset[d] / blockDimensions[d] + gridOffset[d]; - } - } /** * Open an N5 dataset as a memory cached {@link LazyCellImg}. @@ -389,14 +92,11 @@ static void cropBlockDimensions( * @throws IOException */ @SuppressWarnings("unchecked") - public static final > CachedCellImg open( + public static > CachedCellImg open( final N5Reader n5, final String dataset) throws IOException { - if (N5LabelMultisets.isLabelMultisetType(n5, dataset)) - return (CachedCellImg) N5LabelMultisets.openLabelMultiset(n5, dataset); - else - return open(n5, dataset, (Consumer>)img -> {}); + return N5.open(n5, dataset); } @@ -409,12 +109,12 @@ static void cropBlockDimensions( * @return * @throws IOException */ - public static final > CachedCellImg openWithBoundedSoftRefCache( + public static > CachedCellImg openWithBoundedSoftRefCache( final N5Reader n5, final String dataset, final int maxNumCacheEntries) throws IOException { - return openWithBoundedSoftRefCache(n5, dataset, (Consumer>)img -> {}, maxNumCacheEntries); + return N5.openWithBoundedSoftRefCache(n5, dataset, maxNumCacheEntries); } /** @@ -428,14 +128,11 @@ static void cropBlockDimensions( * @throws IOException */ @SuppressWarnings("unchecked") - public static final > CachedCellImg openVolatile( + public static > CachedCellImg openVolatile( final N5Reader n5, final String dataset) throws IOException { - if (N5LabelMultisets.isLabelMultisetType(n5, dataset)) - return (CachedCellImg) N5LabelMultisets.openLabelMultiset(n5, dataset); - else - return openVolatile(n5, dataset, (Consumer>)img -> {}); + return N5.openVolatile(n5, dataset); } @@ -449,12 +146,12 @@ static void cropBlockDimensions( * @return * @throws IOException */ - public static final > CachedCellImg openVolatileWithBoundedSoftRefCache( + public static > CachedCellImg openVolatileWithBoundedSoftRefCache( final N5Reader n5, final String dataset, final int maxNumCacheEntries) throws IOException { - return openVolatileWithBoundedSoftRefCache(n5, dataset, (Consumer>)img -> {}, maxNumCacheEntries); + return N5.openVolatileWithBoundedSoftRefCache(n5, dataset, maxNumCacheEntries); } /** @@ -467,11 +164,11 @@ static void cropBlockDimensions( * @return * @throws IOException */ - public static final > CachedCellImg openWithDiskCache( + public static > CachedCellImg openWithDiskCache( final N5Reader n5, final String dataset) throws IOException { - return openWithDiskCache(n5, dataset, (Consumer>)img -> {}); + return N5.openWithDiskCache(n5, dataset); } /** @@ -483,12 +180,12 @@ static void cropBlockDimensions( * @return * @throws IOException */ - public static final > CachedCellImg open( + public static > CachedCellImg open( final N5Reader n5, final String dataset, final T defaultValue) throws IOException { - return open(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue)); + return N5.open(n5, dataset, defaultValue); } /** @@ -501,13 +198,13 @@ static void cropBlockDimensions( * @return * @throws IOException */ - public static final > CachedCellImg openWithBoundedSoftRefCache( + public static > CachedCellImg openWithBoundedSoftRefCache( final N5Reader n5, final String dataset, final int maxNumCacheEntries, final T defaultValue) throws IOException { - return openWithBoundedSoftRefCache(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue), maxNumCacheEntries); + return N5.openWithBoundedSoftRefCache(n5, dataset, maxNumCacheEntries, defaultValue); } /** @@ -520,12 +217,12 @@ static void cropBlockDimensions( * @return * @throws IOException */ - public static final > CachedCellImg openVolatile( + public static > CachedCellImg openVolatile( final N5Reader n5, final String dataset, final T defaultValue) throws IOException { - return openVolatile(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue)); + return N5.openVolatile(n5, dataset, defaultValue); } /** @@ -539,13 +236,13 @@ static void cropBlockDimensions( * @return * @throws IOException */ - public static final > CachedCellImg openVolatileWithBoundedSoftRefCache( + public static > CachedCellImg openVolatileWithBoundedSoftRefCache( final N5Reader n5, final String dataset, final int maxNumCacheEntries, final T defaultValue) throws IOException { - return openVolatileWithBoundedSoftRefCache(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue), maxNumCacheEntries); + return N5.openVolatileWithBoundedSoftRefCache(n5, dataset, maxNumCacheEntries, defaultValue); } /** @@ -559,12 +256,12 @@ static void cropBlockDimensions( * @return * @throws IOException */ - public static final > CachedCellImg openWithDiskCache( + public static > CachedCellImg openWithDiskCache( final N5Reader n5, final String dataset, final T defaultValue) throws IOException { - return openWithDiskCache(n5, dataset, N5CellLoader.setToDefaultValue(defaultValue)); + return N5.openWithDiskCache(n5, dataset, defaultValue); } /** @@ -577,11 +274,12 @@ static void cropBlockDimensions( * @throws IOException */ @SuppressWarnings({"unchecked", "rawtypes"}) - public static final > CachedCellImg open( + public static > CachedCellImg open( final N5Reader n5, final String dataset, final Consumer> blockNotFoundHandler) throws IOException { - return open(n5, dataset, blockNotFoundHandler, AccessFlags.setOf()); + + return N5.open(n5, dataset, blockNotFoundHandler); } /** @@ -594,12 +292,13 @@ static void cropBlockDimensions( * @throws IOException */ @SuppressWarnings({"unchecked", "rawtypes"}) - public static final > CachedCellImg open( + public static > CachedCellImg open( final N5Reader n5, final String dataset, final Consumer> blockNotFoundHandler, final Set accessFlags) throws IOException { - return open(n5, dataset, blockNotFoundHandler, dataType -> new SoftRefLoaderCache(), accessFlags); + + return N5.open(n5, dataset, blockNotFoundHandler, accessFlags); } /** @@ -613,12 +312,13 @@ static void cropBlockDimensions( * @throws IOException */ @SuppressWarnings({"unchecked", "rawtypes"}) - public static final > CachedCellImg openWithBoundedSoftRefCache( + public static > CachedCellImg openWithBoundedSoftRefCache( final N5Reader n5, final String dataset, final Consumer> blockNotFoundHandler, final int maxNumCacheEntries) throws IOException { - return openWithBoundedSoftRefCache(n5, dataset, blockNotFoundHandler, maxNumCacheEntries, AccessFlags.setOf()); + + return N5.openWithBoundedSoftRefCache(n5, dataset, blockNotFoundHandler, maxNumCacheEntries); } /** @@ -632,13 +332,14 @@ static void cropBlockDimensions( * @throws IOException */ @SuppressWarnings({"unchecked", "rawtypes"}) - public static final > CachedCellImg openWithBoundedSoftRefCache( + public static > CachedCellImg openWithBoundedSoftRefCache( final N5Reader n5, final String dataset, final Consumer> blockNotFoundHandler, final int maxNumCacheEntries, final Set accessFlags) throws IOException { - return open(n5, dataset, blockNotFoundHandler, dataType -> new BoundedSoftRefLoaderCache(maxNumCacheEntries), accessFlags); + + return N5.openWithBoundedSoftRefCache(n5, dataset, blockNotFoundHandler, maxNumCacheEntries, accessFlags); } /** @@ -652,19 +353,14 @@ static void cropBlockDimensions( * @throws IOException */ @SuppressWarnings({"unchecked", "rawtypes"}) - public static final > CachedCellImg open( + public static > CachedCellImg open( final N5Reader n5, final String dataset, final Consumer> blockNotFoundHandler, final Function loaderCacheFactory, final Set accessFlags) throws IOException { - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final LoaderCache loaderCache = loaderCacheFactory.apply(attributes.getDataType()); - final T type = type(attributes.getDataType()); - return type == null - ? null - : open(n5, dataset, blockNotFoundHandler, loaderCache, accessFlags, type); + return N5.open(n5, dataset, blockNotFoundHandler, loaderCacheFactory, accessFlags); } /** @@ -677,7 +373,7 @@ static void cropBlockDimensions( * @throws IOException */ @SuppressWarnings({"unchecked", "rawtypes"}) - public static final , A extends ArrayDataAccess> CachedCellImg open( + public static , A extends ArrayDataAccess> CachedCellImg open( final N5Reader n5, final String dataset, final Consumer> blockNotFoundHandler, @@ -685,17 +381,7 @@ public static final , A extends ArrayDataAccess> Cach final Set accessFlags, final T type) throws IOException { - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final long[] dimensions = attributes.getDimensions(); - final int[] blockSize = attributes.getBlockSize(); - - final N5CellLoader loader = new N5CellLoader<>(n5, dataset, blockSize, blockNotFoundHandler); - - final CellGrid grid = new CellGrid(dimensions, blockSize); - - final Cache> cache = loaderCache.withLoader(LoadedCellCacheLoader.get(grid, loader, type, accessFlags)); - final CachedCellImg img = new CachedCellImg(grid, type, cache, ArrayDataAccessFactory.get(type, accessFlags)); - return img; + return N5.open(n5, dataset, blockNotFoundHandler, loaderCache, accessFlags, type); } /** @@ -709,11 +395,12 @@ public static final , A extends ArrayDataAccess> Cach * @throws IOException */ @SuppressWarnings({"unchecked", "rawtypes"}) - public static final > CachedCellImg openVolatile( + public static > CachedCellImg openVolatile( final N5Reader n5, final String dataset, final Consumer> blockNotFoundHandler) throws IOException { - return open(n5, dataset, blockNotFoundHandler, AccessFlags.setOf(AccessFlags.VOLATILE)); + + return N5.openVolatile(n5, dataset, blockNotFoundHandler); } /** @@ -732,7 +419,8 @@ public static final , A extends ArrayDataAccess> Cach final String dataset, final Consumer> blockNotFoundHandler, final int maxNumCacheEntries) throws IOException { - return openWithBoundedSoftRefCache(n5, dataset, blockNotFoundHandler, maxNumCacheEntries, AccessFlags.setOf(AccessFlags.VOLATILE)); + + return N5.openVolatileWithBoundedSoftRefCache(n5, dataset, blockNotFoundHandler, maxNumCacheEntries); } /** @@ -747,42 +435,13 @@ public static final , A extends ArrayDataAccess> Cach * @return * @throws IOException */ - public static final > Pair[], double[][]> openMipmapsWithHandler( + public static > Pair[], double[][]> openMipmapsWithHandler( final N5Reader n5, final String group, final boolean useVolatileAccess, final IntFunction>> blockNotFoundHandlerSupplier) throws IOException { - final int numScales = n5.list(group).length; - @SuppressWarnings("unchecked") - final RandomAccessibleInterval[] mipmaps = new RandomAccessibleInterval[numScales]; - final double[][] scales = new double[numScales][]; - - for (int s = 0; s < numScales; ++s) { - final String datasetName = group + "/s" + s; - final long[] dimensions = n5.getAttribute(datasetName, "dimensions", long[].class); - final long[] downsamplingFactors = n5.getAttribute(datasetName, "downsamplingFactors", long[].class); - final double[] scale = new double[dimensions.length]; - if (downsamplingFactors == null) { - final int si = 1 << s; - for (int i = 0; i < scale.length; ++i) - scale[i] = si; - } else { - for (int i = 0; i < scale.length; ++i) - scale[i] = downsamplingFactors[i]; - } - - final RandomAccessibleInterval source; - if (useVolatileAccess) - source = N5Utils.openVolatile(n5, datasetName, blockNotFoundHandlerSupplier.apply(s)); - else - source = N5Utils.open(n5, datasetName, blockNotFoundHandlerSupplier.apply(s)); - - mipmaps[s] = source; - scales[s] = scale; - } - - return new ValuePair<>(mipmaps, scales); + return N5.openMipmapsWithHandler(n5, group, useVolatileAccess, blockNotFoundHandlerSupplier); } /** @@ -797,19 +456,13 @@ public static final > Pair[] * @return * @throws IOException */ - public static final > Pair[], double[][]> openMipmaps( + public static > Pair[], double[][]> openMipmaps( final N5Reader n5, final String group, final boolean useVolatileAccess, final IntFunction defaultValueSupplier) throws IOException { - return openMipmapsWithHandler( - n5, - group, - useVolatileAccess, - s -> { - return N5CellLoader.setToDefaultValue(defaultValueSupplier.apply(s)); - }); + return N5.openMipmaps(n5, group, useVolatileAccess, defaultValueSupplier); } /** @@ -823,16 +476,12 @@ public static final > Pair[] * @return * @throws IOException */ - public static final > Pair[], double[][]> openMipmaps( + public static > Pair[], double[][]> openMipmaps( final N5Reader n5, final String group, final boolean useVolatileAccess) throws IOException { - return openMipmapsWithHandler( - n5, - group, - useVolatileAccess, - s -> t -> {}); + return N5.openMipmaps(n5, group, useVolatileAccess); } /** @@ -846,28 +495,12 @@ public static final > Pair[] * @return * @throws IOException */ - public static final > CachedCellImg openWithDiskCache( + public static > CachedCellImg openWithDiskCache( final N5Reader n5, final String dataset, final Consumer> blockNotFoundHandler) throws IOException { - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final long[] dimensions = attributes.getDimensions(); - final int[] blockSize = attributes.getBlockSize(); - - final N5CellLoader loader = new N5CellLoader<>(n5, dataset, blockSize, blockNotFoundHandler); - - final DiskCachedCellImgOptions options = DiskCachedCellImgOptions - .options() - .cellDimensions(blockSize) - .dirtyAccesses(true) - .maxCacheSize(100); - - final DiskCachedCellImgFactory factory = new DiskCachedCellImgFactory( - type(attributes.getDataType()), - options); - - return factory.create(dimensions, loader); + return N5.openWithDiskCache(n5, dataset, blockNotFoundHandler); } /** @@ -882,55 +515,14 @@ public static final > Pair[] * @param gridOffset * @throws IOException */ - public static final > void saveBlock( - RandomAccessibleInterval source, + public static > void saveBlock( + final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final DatasetAttributes attributes, final long[] gridOffset) throws IOException { - if (N5LabelMultisets.isLabelMultisetType(n5, dataset)) { - @SuppressWarnings("unchecked") - final RandomAccessibleInterval labelMultisetSource = (RandomAccessibleInterval) source; - N5LabelMultisets.saveLabelMultisetBlock(labelMultisetSource, n5, dataset, attributes, gridOffset); - return; - } - - source = Views.zeroMin(source); - final int n = source.numDimensions(); - final long[] max = Intervals.maxAsLongArray(source); - final long[] offset = new long[n]; - final long[] gridPosition = new long[n]; - final int[] blockSize = attributes.getBlockSize(); - final int[] intCroppedBlockSize = new int[n]; - final long[] longCroppedBlockSize = new long[n]; - for (int d = 0; d < n;) { - cropBlockDimensions( - max, - offset, - gridOffset, - blockSize, - longCroppedBlockSize, - intCroppedBlockSize, - gridPosition); - final RandomAccessibleInterval sourceBlock = Views.offsetInterval(source, offset, longCroppedBlockSize); - final DataBlock dataBlock = createDataBlock( - sourceBlock, - attributes.getDataType(), - intCroppedBlockSize, - longCroppedBlockSize, - gridPosition); - - n5.writeBlock(dataset, attributes, dataBlock); - - for (d = 0; d < n; ++d) { - offset[d] += blockSize[d]; - if (offset[d] <= max[d]) - break; - else - offset[d] = 0; - } - } + N5.saveBlock(source, n5, dataset, attributes, gridOffset); } /** @@ -945,16 +537,13 @@ public static final > void saveBlock( * @param attributes * @throws IOException */ - public static final > void saveBlock( + public static > void saveBlock( final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final DatasetAttributes attributes) throws IOException { - final int[] blockSize = attributes.getBlockSize(); - final long[] gridOffset = new long[blockSize.length]; - Arrays.setAll(gridOffset, d -> source.min(d) / blockSize[d]); - saveBlock(source, n5, dataset, attributes, gridOffset); + N5.saveBlock(source, n5, dataset, attributes); } /** @@ -968,17 +557,12 @@ public static final > void saveBlock( * @param dataset * @throws IOException */ - public static final > void saveBlock( + public static > void saveBlock( final RandomAccessibleInterval source, final N5Writer n5, final String dataset) throws IOException { - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - if (attributes != null) { - saveBlock(source, n5, dataset, attributes); - } else { - throw new IOException("Dataset " + dataset + " does not exist."); - } + N5.saveBlock(source, n5, dataset); } /** @@ -992,18 +576,13 @@ public static final > void saveBlock( * @param gridOffset * @throws IOException */ - public static final > void saveBlock( + public static > void saveBlock( final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final long[] gridOffset) throws IOException { - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - if (attributes != null) { - saveBlock(source, n5, dataset, attributes, gridOffset); - } else { - throw new IOException("Dataset " + dataset + " does not exist."); - } + N5.saveBlock(source, n5, dataset, gridOffset); } /** @@ -1018,79 +597,14 @@ public static final > void saveBlock( * @throws InterruptedException * @throws ExecutionException */ - public static final > void saveBlock( + public static > void saveBlock( final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final long[] gridOffset, final ExecutorService exec) throws IOException, InterruptedException, ExecutionException { - if (N5LabelMultisets.isLabelMultisetType(n5, dataset)) { - @SuppressWarnings("unchecked") - final RandomAccessibleInterval labelMultisetSource = (RandomAccessibleInterval) source; - N5LabelMultisets.saveLabelMultisetBlock(labelMultisetSource, n5, dataset, gridOffset, exec); - return; - } - - final RandomAccessibleInterval zeroMinSource = Views.zeroMin(source); - final long[] dimensions = Intervals.dimensionsAsLongArray(zeroMinSource); - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - if (attributes != null) { - final int n = dimensions.length; - final long[] max = Intervals.maxAsLongArray(zeroMinSource); - final long[] offset = new long[n]; - final int[] blockSize = attributes.getBlockSize(); - - final ArrayList> futures = new ArrayList<>(); - for (int d = 0; d < n;) { - final long[] fOffset = offset.clone(); - - futures.add( - exec.submit( - () -> { - - final long[] gridPosition = new long[n]; - final int[] intCroppedBlockSize = new int[n]; - final long[] longCroppedBlockSize = new long[n]; - - cropBlockDimensions( - max, - fOffset, - gridOffset, - blockSize, - longCroppedBlockSize, - intCroppedBlockSize, - gridPosition); - - final RandomAccessibleInterval sourceBlock = Views - .offsetInterval(zeroMinSource, fOffset, longCroppedBlockSize); - final DataBlock dataBlock = createDataBlock( - sourceBlock, - attributes.getDataType(), - intCroppedBlockSize, - longCroppedBlockSize, - gridPosition); - - try { - n5.writeBlock(dataset, attributes, dataBlock); - } catch (final IOException e) { - e.printStackTrace(); - } - })); - - for (d = 0; d < n; ++d) { - offset[d] += blockSize[d]; - if (offset[d] <= max[d]) - break; - else - offset[d] = 0; - } - } - for (final Future f : futures) - f.get(); - } else { - throw new IOException("Dataset " + dataset + " does not exist."); - } + N5.saveBlock(source, n5, dataset, gridOffset, exec); } /** @@ -1108,51 +622,15 @@ public static final > void saveBlock( * @param defaultValue * @throws IOException */ - public static final > void saveNonEmptyBlock( - RandomAccessibleInterval source, + public static > void saveNonEmptyBlock( + final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final DatasetAttributes attributes, final long[] gridOffset, final T defaultValue) throws IOException { - source = Views.zeroMin(source); - final int n = source.numDimensions(); - final long[] max = Intervals.maxAsLongArray(source); - final long[] offset = new long[n]; - final long[] gridPosition = new long[n]; - final int[] blockSize = attributes.getBlockSize(); - final int[] intCroppedBlockSize = new int[n]; - final long[] longCroppedBlockSize = new long[n]; - for (int d = 0; d < n;) { - cropBlockDimensions( - max, - offset, - gridOffset, - blockSize, - longCroppedBlockSize, - intCroppedBlockSize, - gridPosition); - final RandomAccessibleInterval sourceBlock = Views.offsetInterval(source, offset, longCroppedBlockSize); - final DataBlock dataBlock = createNonEmptyDataBlock( - sourceBlock, - attributes.getDataType(), - intCroppedBlockSize, - longCroppedBlockSize, - gridPosition, - defaultValue); - - if (dataBlock != null) - n5.writeBlock(dataset, attributes, dataBlock); - - for (d = 0; d < n; ++d) { - offset[d] += blockSize[d]; - if (offset[d] <= max[d]) - break; - else - offset[d] = 0; - } - } + N5.saveNonEmptyBlock(source, n5, dataset, attributes, gridOffset, defaultValue); } /** @@ -1169,17 +647,14 @@ public static final > void saveNonEmptyBlock( * @param defaultValue * @throws IOException */ - public static final > void saveNonEmptyBlock( + public static > void saveNonEmptyBlock( final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final DatasetAttributes attributes, final T defaultValue) throws IOException { - final int[] blockSize = attributes.getBlockSize(); - final long[] gridOffset = new long[blockSize.length]; - Arrays.setAll(gridOffset, d -> source.min(d) / blockSize[d]); - saveNonEmptyBlock(source, n5, dataset, attributes, gridOffset, defaultValue); + N5.saveNonEmptyBlock(source, n5, dataset, attributes, defaultValue); } /** @@ -1195,18 +670,13 @@ public static final > void saveNonEmptyBlock( * @param defaultValue * @throws IOException */ - public static final > void saveNonEmptyBlock( + public static > void saveNonEmptyBlock( final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final T defaultValue) throws IOException { - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - if (attributes != null) { - saveNonEmptyBlock(source, n5, dataset, attributes, defaultValue); - } else { - throw new IOException("Dataset " + dataset + " does not exist."); - } + N5.saveNonEmptyBlock(source, n5, dataset, defaultValue); } /** @@ -1223,19 +693,14 @@ public static final > void saveNonEmptyBlock( * @param defaultValue * @throws IOException */ - public static final > void saveNonEmptyBlock( + public static > void saveNonEmptyBlock( final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final long[] gridOffset, final T defaultValue) throws IOException { - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - if (attributes != null) { - saveNonEmptyBlock(source, n5, dataset, attributes, gridOffset, defaultValue); - } else { - throw new IOException("Dataset " + dataset + " does not exist."); - } + N5.saveNonEmptyBlock(source, n5, dataset, gridOffset, defaultValue); } /** @@ -1248,56 +713,14 @@ public static final > void saveNonEmptyBlock( * @param compression * @throws IOException */ - public static final > void save( - RandomAccessibleInterval source, + public static > void save( + final RandomAccessibleInterval source, final N5Writer n5, final String dataset, final int[] blockSize, final Compression compression) throws IOException { - if (Util.getTypeFromInterval(source) instanceof LabelMultisetType) { - @SuppressWarnings("unchecked") - final RandomAccessibleInterval labelMultisetSource = (RandomAccessibleInterval) source; - N5LabelMultisets.saveLabelMultiset(labelMultisetSource, n5, dataset, blockSize, compression); - return; - } - - source = Views.zeroMin(source); - final long[] dimensions = Intervals.dimensionsAsLongArray(source); - final DatasetAttributes attributes = new DatasetAttributes( - dimensions, - blockSize, - dataType(Util.getTypeFromInterval(source)), - compression); - - n5.createDataset(dataset, attributes); - - final int n = dimensions.length; - final long[] max = Intervals.maxAsLongArray(source); - final long[] offset = new long[n]; - final long[] gridPosition = new long[n]; - final int[] intCroppedBlockSize = new int[n]; - final long[] longCroppedBlockSize = new long[n]; - for (int d = 0; d < n;) { - cropBlockDimensions(max, offset, blockSize, longCroppedBlockSize, intCroppedBlockSize, gridPosition); - final RandomAccessibleInterval sourceBlock = Views.offsetInterval(source, offset, longCroppedBlockSize); - final DataBlock dataBlock = createDataBlock( - sourceBlock, - attributes.getDataType(), - intCroppedBlockSize, - longCroppedBlockSize, - gridPosition); - - n5.writeBlock(dataset, attributes, dataBlock); - - for (d = 0; d < n; ++d) { - offset[d] += blockSize[d]; - if (offset[d] <= max[d]) - break; - else - offset[d] = 0; - } - } + N5.save(source, n5, dataset, blockSize, compression); } /** @@ -1313,7 +736,7 @@ public static final > void save( * @throws InterruptedException * @throws ExecutionException */ - public static final > void save( + public static > void save( final RandomAccessibleInterval source, final N5Writer n5, final String dataset, @@ -1321,73 +744,7 @@ public static final > void save( final Compression compression, final ExecutorService exec) throws IOException, InterruptedException, ExecutionException { - if (Util.getTypeFromInterval(source) instanceof LabelMultisetType) { - @SuppressWarnings("unchecked") - final RandomAccessibleInterval labelMultisetSource = (RandomAccessibleInterval) source; - N5LabelMultisets.saveLabelMultiset(labelMultisetSource, n5, dataset, blockSize, compression, exec); - return; - } - - final RandomAccessibleInterval zeroMinSource = Views.zeroMin(source); - final long[] dimensions = Intervals.dimensionsAsLongArray(zeroMinSource); - final DatasetAttributes attributes = new DatasetAttributes( - dimensions, - blockSize, - dataType(Util.getTypeFromInterval(zeroMinSource)), - compression); - - n5.createDataset(dataset, attributes); - - final int n = dimensions.length; - final long[] max = Intervals.maxAsLongArray(zeroMinSource); - final long[] offset = new long[n]; - - final ArrayList> futures = new ArrayList<>(); - for (int d = 0; d < n;) { - final long[] fOffset = offset.clone(); - - futures.add( - exec.submit( - () -> { - - final long[] gridPosition = new long[n]; - final int[] intCroppedBlockSize = new int[n]; - final long[] longCroppedBlockSize = new long[n]; - - cropBlockDimensions( - max, - fOffset, - blockSize, - longCroppedBlockSize, - intCroppedBlockSize, - gridPosition); - - final RandomAccessibleInterval sourceBlock = Views - .offsetInterval(zeroMinSource, fOffset, longCroppedBlockSize); - final DataBlock dataBlock = createDataBlock( - sourceBlock, - attributes.getDataType(), - intCroppedBlockSize, - longCroppedBlockSize, - gridPosition); - - try { - n5.writeBlock(dataset, attributes, dataBlock); - } catch (final IOException e) { - e.printStackTrace(); - } - })); - - for (d = 0; d < n; ++d) { - offset[d] += blockSize[d]; - if (offset[d] <= max[d]) - break; - else - offset[d] = 0; - } - } - for (final Future f : futures) - f.get(); + N5.save(source, n5, dataset, blockSize, compression, exec); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/RandomAccessibleLoader.java b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/RandomAccessibleLoader.java index 066db61..6c51e07 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/imglib2/RandomAccessibleLoader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/imglib2/RandomAccessibleLoader.java @@ -1,5 +1,6 @@ /** - * Copyright (c) 2017-2018, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5DatasetViewerExample.java b/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5DatasetViewerExample.java index 2cc0236..35cbfa7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5DatasetViewerExample.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5DatasetViewerExample.java @@ -1,5 +1,6 @@ /** - * Copyright (c) 2017-2018, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisetsTest.java b/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisetsTest.java index 84a1a89..766147f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisetsTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5LabelMultisetsTest.java @@ -1,5 +1,6 @@ /** - * Copyright (c) 2017-2018, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5Test.java new file mode 100644 index 0000000..899a77e --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5Test.java @@ -0,0 +1,351 @@ +/** + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5.imglib2; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.N5FSWriter; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import net.imglib2.Cursor; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.cache.img.CachedCellImg; +import net.imglib2.img.array.ArrayImg; +import net.imglib2.img.array.ArrayImgs; +import net.imglib2.img.basictypeaccess.ShortAccess; +import net.imglib2.img.basictypeaccess.array.ShortArray; +import net.imglib2.img.basictypeaccess.volatiles.VolatileAccess; +import net.imglib2.type.numeric.integer.IntType; +import net.imglib2.type.numeric.integer.UnsignedShortType; +import net.imglib2.util.Pair; +import net.imglib2.util.Util; +import net.imglib2.view.IntervalView; +import net.imglib2.view.Views; + +public class N5Test { + + static private String testDirPath = System.getProperty("user.home") + "/tmp/n5-imglib2-test"; + + static private String datasetName = "/test/group/dataset"; + + static private long[] dimensions = new long[]{11, 22, 33}; + + static private int[] blockSize = new int[]{5, 7, 9}; + + static short[] data; + + static short[] excessData; + + static private N5Writer n5; + + private static final int MAX_NUM_CACHE_ENTRIES = 10; + + private static final String EMPTY_DATASET = "/test/group/empty-dataset"; + + private static final int EMPTY_BLOCK_VALUE = 123; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + final File testDir = new File(testDirPath); + testDir.mkdirs(); + if (!(testDir.exists() && testDir.isDirectory())) + throw new IOException("Could not create test directory for HDF5Utils test."); + + n5 = new N5FSWriter(testDirPath); + + final Random rnd = new Random(); + + data = new short[(int)(dimensions[0] * dimensions[1] * dimensions[2])]; + for (int i = 0; i < data.length; ++i) + data[i] = (short)rnd.nextInt(); + + excessData = new short[(int)((dimensions[0] + 2) * (dimensions[1] + 3) * (dimensions[2] + 4))]; + for (int i = 0; i < excessData.length; ++i) + excessData[i] = (short)rnd.nextInt(); + + n5.createDataset(EMPTY_DATASET, dimensions, blockSize, N5.dataType(new UnsignedShortType()), new GzipCompression()); + } + + @AfterClass + public static void rampDownAfterClass() throws Exception { + + n5.remove(""); + } + + @Before + public void setUp() throws Exception {} + + @After + public void tearDown() throws Exception {} + + @Test + public void testSaveAndOpen() throws InterruptedException, ExecutionException { + + final ArrayImg img = ArrayImgs.unsignedShorts(data, dimensions); + try { + N5.save(img, n5, datasetName, blockSize, new RawCompression()); + RandomAccessibleInterval loaded = N5.open(n5, datasetName); + for (final Pair pair : Views + .flatIterable(Views.interval(Views.pair(img, loaded), img))) + Assert.assertEquals(pair.getA().get(), pair.getB().get()); + + final ExecutorService exec = Executors.newFixedThreadPool(4); + N5.save(img, n5, datasetName, blockSize, new RawCompression(), exec); + loaded = N5.open(n5, datasetName); + for (final Pair pair : Views + .flatIterable(Views.interval(Views.pair(img, loaded), img))) + Assert.assertEquals(pair.getA().get(), pair.getB().get()); + exec.shutdown(); + + } catch (final IOException e) { + fail("Failed by I/O exception."); + e.printStackTrace(); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testOpenWithBoundedSoftRefCache() throws IOException { + + // existing dataset + { + final ArrayImg img = ArrayImgs.unsignedShorts(data, dimensions); + N5.save(img, n5, datasetName, blockSize, new RawCompression()); + final RandomAccessibleInterval loaded = + N5.openWithBoundedSoftRefCache(n5, datasetName, MAX_NUM_CACHE_ENTRIES); + for (final Pair pair : Views + .flatIterable(Views.interval(Views.pair(img, loaded), img))) + Assert.assertEquals(pair.getA().get(), pair.getB().get()); + MatcherAssert.assertThat(((CachedCellImg)loaded).getAccessType(), CoreMatchers.instanceOf(ShortAccess.class)); + } + + // empty dataset with default value + { + final RandomAccessibleInterval loaded = + N5.openWithBoundedSoftRefCache(n5, EMPTY_DATASET, MAX_NUM_CACHE_ENTRIES, new UnsignedShortType(EMPTY_BLOCK_VALUE)); + Views.iterable(loaded).forEach(val -> Assert.assertEquals(EMPTY_BLOCK_VALUE, val.get())); + MatcherAssert.assertThat(((CachedCellImg)loaded).getAccessType(), CoreMatchers.instanceOf(ShortAccess.class)); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testVolatileOpenWithBoundedSoftRefCache() throws IOException { + + // existing dataset + { + final ArrayImg img = ArrayImgs.unsignedShorts(data, dimensions); + N5.save(img, n5, datasetName, blockSize, new RawCompression()); + final RandomAccessibleInterval loaded = + N5.openVolatileWithBoundedSoftRefCache(n5, datasetName, MAX_NUM_CACHE_ENTRIES); + for (final Pair pair : Views + .flatIterable(Views.interval(Views.pair(img, loaded), img))) + Assert.assertEquals(pair.getA().get(), pair.getB().get()); + Assert.assertEquals(UnsignedShortType.class, Util.getTypeFromInterval(loaded).getClass()); + MatcherAssert.assertThat(((CachedCellImg)loaded).getAccessType(), CoreMatchers.instanceOf(VolatileAccess.class)); + MatcherAssert.assertThat(((CachedCellImg)loaded).getAccessType(), CoreMatchers.instanceOf(ShortAccess.class)); + } + + // empty dataset with default value + { + final RandomAccessibleInterval loaded = + N5.openVolatileWithBoundedSoftRefCache(n5, EMPTY_DATASET, MAX_NUM_CACHE_ENTRIES, new UnsignedShortType(EMPTY_BLOCK_VALUE)); + Views.iterable(loaded).forEach(val -> Assert.assertEquals(EMPTY_BLOCK_VALUE, val.get())); + Assert.assertEquals(UnsignedShortType.class, Util.getTypeFromInterval(loaded).getClass()); + MatcherAssert.assertThat(((CachedCellImg)loaded).getAccessType(), CoreMatchers.instanceOf(VolatileAccess.class)); + MatcherAssert.assertThat(((CachedCellImg)loaded).getAccessType(), CoreMatchers.instanceOf(ShortAccess.class)); + } + } + + private short[] fillData(final int[] size) { + + return Arrays.copyOf(excessData, Arrays.stream(size).reduce(1, (a, b) -> a * b)); + } + + @Test + public void testBlockSize() throws IOException { + + n5.remove(datasetName); + final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT16, new GzipCompression()); + n5.createDataset(datasetName, datasetAttributes); + + final int[] blockSize000 = new int[blockSize.length]; + Arrays.setAll(blockSize000, i -> blockSize[i] - 2); + final ShortArrayDataBlock block000 = new ShortArrayDataBlock(blockSize000, new long[]{0, 0, 0}, fillData(blockSize000)); + n5.writeBlock(datasetName, datasetAttributes, block000); + + final int[] blockSize001 = new int[blockSize.length]; + Arrays.setAll(blockSize001, i -> blockSize[i] + 2); + final ShortArrayDataBlock block001 = new ShortArrayDataBlock(blockSize001, new long[]{0, 0, 1}, fillData(blockSize001)); + n5.writeBlock(datasetName, datasetAttributes, block001); + + final RandomAccessibleInterval img = N5.open(n5, datasetName); + + final IntervalView interval000 = Views.interval( + img, + new long[] {0, 0, 0}, + new long[] {blockSize000[0] - 1, blockSize000[1] - 1, blockSize000[2] - 1}); + + int i = 0; + for (final UnsignedShortType t : interval000) + assertTrue(t.getShort() == excessData[i++]); + + final IntervalView interval001 = Views.interval( + img, + new long[] {0, 0, blockSize[2]}, + new long[] {blockSize[0] - 1, blockSize[1] - 1, blockSize[2] + blockSize[2] - 1}); + + i = 0; + final ArrayImg referenceDataImg = + ArrayImgs.unsignedShorts( + excessData, + blockSize001[0], + blockSize001[1], + blockSize001[2]); + final Cursor c = Views.interval( + referenceDataImg, + new long[] {0, 0, 0}, + new long[] {blockSize[0] - 1, blockSize[1] - 1, blockSize[2] - 1}).cursor(); + final Cursor d = interval001.cursor(); + while (c.hasNext()) + assertTrue(c.next().valueEquals(d.next())); + } + + @Test + public void testAxes() throws IOException { + + final double[][] doubleVectors = new double[][] { + {5, 6}, + {5, 6, 7}, + {5, 6, 7, 8} + }; + final float[][] floatVectors = new float[][] { + {5, 6}, + {5, 6, 7}, + {5, 6, 7, 8} + }; + final long[][] longVectors = new long[][] { + {5, 6}, + {5, 6, 7}, + {5, 6, 7, 8} + }; + final int[][] intVectors = new int[][] { + {5, 6}, + {5, 6, 7}, + {5, 6, 7, 8} + }; + final short[][] shortVectors = new short[][] { + {5, 6}, + {5, 6, 7}, + {5, 6, 7, 8} + }; + final byte[][] byteVectors = new byte[][] { + {5, 6}, + {5, 6, 7}, + {5, 6, 7, 8} + }; + final String[][] stringVectors = new String[][] { + {"x", "y"}, + {"z", "y", "x"}, + {"y", "t", "z", "x"} + }; + + final int[][] axes = new int[][] { + {0, 1}, + {2, 1, 0}, + {1, 3, 2, 0} + }; + final int[][] arrays = new int[][] { + {-1, 0, 1, -1}, + {-1, 2, 1, -1, 0, -1, -1, -1}, + {-1, 1, 3, -1, 2, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, -1} + }; + + for (int i = 0; i < axes.length; ++i) { + + final String axesName = "axes-test-" + i; + N5.setAxes(n5, axesName, axes[i]); + + /* array */ + final RandomAccessibleInterval axesImg = N5.open(n5, N5.AXES_PREFIX + axesName); + int d = 0; + for (final IntType t : Views.flatIterable(axesImg)) { + if (t.get() != arrays[i][d]) + throw new AssertionError("values not equal: expected " + arrays[i][d] + ", actual " + t.get()); + ++d; + } + + /* get the axes permutation */ + final int[] axes_test = N5.getAxes(n5, axesName); + assertArrayEquals(axes[i], axes_test); + + /* write a vector */ + n5.createGroup("vector-test"); + + N5.setVector(doubleVectors[i], n5, "vector-test", axesName + "-double", axes[i]); + N5.setVector(floatVectors[i], n5, "vector-test", axesName + "-float", axes[i]); + N5.setVector(longVectors[i], n5, "vector-test", axesName + "-long", axes[i]); + N5.setVector(intVectors[i], n5, "vector-test", axesName + "-int", axes[i]); + N5.setVector(shortVectors[i], n5, "vector-test", axesName + "-short", axes[i]); + N5.setVector(byteVectors[i], n5, "vector-test", axesName + "-byte", axes[i]); + N5.setVector(stringVectors[i], n5, "vector-test", axesName + "-string", axes[i]); + + assertArrayEquals(doubleVectors[i], N5.getDoubleVector(n5, "vector-test", axesName + "-double", axes[i]), 0.01); + assertArrayEquals(floatVectors[i], N5.getFloatVector(n5, "vector-test", axesName + "-float", axes[i]), 0.01f); + assertArrayEquals(longVectors[i], N5.getLongVector(n5, "vector-test", axesName + "-long", axes[i])); + assertArrayEquals(intVectors[i], N5.getIntVector(n5, "vector-test", axesName + "-int", axes[i])); + assertArrayEquals(shortVectors[i], N5.getShortVector(n5, "vector-test", axesName + "-short", axes[i])); + assertArrayEquals(byteVectors[i], N5.getByteVector(n5, "vector-test", axesName + "-byte", axes[i])); + assertArrayEquals(stringVectors[i], N5.getVector(n5, "vector-test", axesName + "-string", String.class, axes[i])); + } + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5UtilsTest.java b/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5UtilsTest.java index 75af6c0..ac5d8eb 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5UtilsTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/imglib2/N5UtilsTest.java @@ -1,5 +1,6 @@ /** - * Copyright (c) 2017-2018, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * Copyright (c) 2017-2019, Stephan Saalfeld, Philipp Hanslovsky, Igor Pisarev + * John Bogovic * All rights reserved. * * Redistribution and use in source and binary forms, with or without