add spark client module with SIFT and DMesh point match generation pi…

…pelines

pom.xml

@@ -13,6 +13,7 @@
         <module>render-app</module>
         <module>render-ws</module>
         <module>render-ws-java-client</module>
+        <module>render-ws-spark-client</module>
         <module>trakem2-scripts</module>
         <module>docs</module>
     </modules>

render-app/src/main/java/org/janelia/alignment/match/CanvasFeatureExtractor.java

@@ -1,10 +1,10 @@
 package org.janelia.alignment.match;
 
 import ij.ImagePlus;
-import ij.process.ByteProcessor;
 
 import java.awt.image.BufferedImage;
 import java.io.File;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -16,7 +16,6 @@
 import org.janelia.alignment.Render;
 import org.janelia.alignment.RenderParameters;
 import org.janelia.alignment.Utils;
-import org.janelia.alignment.util.ImageProcessorCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -26,7 +25,7 @@
  *
  * @author Eric Trautman
  */
-public class CanvasFeatureExtractor {
+public class CanvasFeatureExtractor implements Serializable {
 
     private final FloatArray2DSIFT.Param coreSiftParameters;
     private final double minScale;
@@ -76,7 +75,18 @@ public List<Feature> extractFeatures(final RenderParameters renderParameters,
 
         renderParameters.validate();
 
-        final BufferedImage bufferedImage = renderImage(renderParameters, renderFile);
+        final BufferedImage bufferedImage = Render.renderWithNoise(renderParameters, fillWithNoise);
+
+        if (renderFile != null) {
+            try {
+                Utils.saveImage(bufferedImage,
+                                renderFile,
+                                renderParameters.isConvertToGray(),
+                                renderParameters.getQuality());
+            } catch (final Throwable t) {
+                LOG.warn("extractFeatures: failed to save " + renderFile.getAbsolutePath(), t);
+            }
+        }
 
         return extractFeaturesFromImage(bufferedImage);
     }
@@ -156,42 +166,5 @@ public List<Feature> extractFeaturesFromImage(final BufferedImage bufferedImage)
         return featureList;
     }
 
-    private BufferedImage renderImage(final RenderParameters renderParameters,
-                                      final File renderFile) {
-
-        final Timer timer = new Timer();
-        timer.start();
-
-        LOG.info("renderImage: entry");
-
-        final BufferedImage bufferedImage = renderParameters.openTargetImage();
-        final ByteProcessor ip = new ByteProcessor(bufferedImage.getWidth(), bufferedImage.getHeight());
-
-        if (fillWithNoise) {
-            mpicbg.ij.util.Util.fillWithNoise(ip);
-            bufferedImage.getGraphics().drawImage(ip.createImage(), 0, 0, null);
-        }
-
-        Render.render(renderParameters, bufferedImage, ImageProcessorCache.DISABLED_CACHE);
-
-        if (renderFile != null) {
-            final String renderAbsolutePath = renderFile.getAbsolutePath();
-            try {
-                final String outputFormat = renderAbsolutePath.substring(renderAbsolutePath.lastIndexOf('.') + 1);
-                Utils.saveImage(bufferedImage,
-                                renderAbsolutePath,
-                                outputFormat,
-                                renderParameters.isConvertToGray(),
-                                renderParameters.getQuality());
-            } catch (final Throwable t) {
-                LOG.warn("renderImage: failed to save " + renderAbsolutePath, t);
-            }
-        }
-
-        LOG.info("renderImage: exit, elapsedTime=" + timer.stop() + "ms");
-
-        return bufferedImage;
-    }
-
     private static final Logger LOG = LoggerFactory.getLogger(CanvasFeatureExtractor.class);
 }

render-app/src/main/java/org/janelia/alignment/match/CanvasFeatureMatchResult.java

@@ -1,5 +1,6 @@
 package org.janelia.alignment.match;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import mpicbg.models.Point;
@@ -125,4 +126,38 @@ public static Matches convertPointMatchListToMatches(final List<PointMatch> poin
         return matches;
     }
 
+    /**
+     * @param  matches  point match list in {@link Matches} form.
+     *
+     * @return the corresponding list of {@link PointMatch} objects.
+     */
+    public static List<PointMatch> convertMatchesToPointMatchList(final Matches matches) {
+
+        final double w[] = matches.getWs();
+
+        final int pointMatchCount = w.length;
+        final List<PointMatch> pointMatchList = new ArrayList<>(pointMatchCount);
+
+        if (pointMatchCount > 0) {
+            final double p[][] = matches.getPs();
+            final double q[][] = matches.getQs();
+
+            final int dimensionCount = p.length;
+
+            for (int matchIndex = 0; matchIndex < pointMatchCount; matchIndex++) {
+
+                final double pLocal[] = new double[dimensionCount];
+                final double qLocal[] = new double[dimensionCount];
+
+                for (int dimensionIndex = 0; dimensionIndex < dimensionCount; dimensionIndex++) {
+                    pLocal[dimensionIndex] = p[dimensionIndex][matchIndex];
+                    qLocal[dimensionIndex] = q[dimensionIndex][matchIndex];
+                }
+
+                pointMatchList.add(new PointMatch(new Point(pLocal), new Point(qLocal), w[matchIndex]));
+            }
+        }
+
+        return pointMatchList;
+    }
 }

render-app/src/main/java/org/janelia/alignment/match/CanvasFeatureMatcher.java

@@ -1,5 +1,6 @@
 package org.janelia.alignment.match;
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -20,12 +21,13 @@
  *
  * @author Eric Trautman
  */
-public class CanvasFeatureMatcher {
+public class CanvasFeatureMatcher implements Serializable {
 
     private final float rod;
     private final float maxEpsilon;
     private final float minInlierRatio;
     private final int minNumInliers;
+    private final boolean filterMatches;
 
     /**
      * Sets up everything that is needed to derive point matches from the feature lists of two canvases.
@@ -34,15 +36,22 @@ public class CanvasFeatureMatcher {
      * @param  maxEpsilon      minimal allowed transfer error (e.g. 20.0f).
      * @param  minInlierRatio  minimal ratio of inliers to candidates (e.g. 0.0f).
      * @param  minNumInliers   minimal absolute number of inliers for matches (e.g. 10).
+     * @param  filterMatches   indicates whether matches should be filtered.
      */
     public CanvasFeatureMatcher(final float rod,
                                 final float maxEpsilon,
                                 final float minInlierRatio,
-                                final int minNumInliers) {
+                                final int minNumInliers,
+                                final boolean filterMatches) {
         this.rod = rod;
         this.maxEpsilon = maxEpsilon;
         this.minInlierRatio = minInlierRatio;
         this.minNumInliers = minNumInliers;
+        this.filterMatches = filterMatches;
+    }
+
+    public boolean isFilterMatches() {
+        return filterMatches;
     }
 
     /**
@@ -65,7 +74,12 @@ public CanvasFeatureMatchResult deriveMatchResult(final List<Feature> canvas1Fea
 
         FeatureTransform.matchFeatures(canvas1Features, canvas2Features, candidates, rod);
 
-        final List<PointMatch> inliers = filterMatches(candidates, model);
+        final List<PointMatch> inliers;
+        if (filterMatches) {
+            inliers = filterMatches(candidates, model);
+        } else {
+            inliers = candidates;
+        }
 
         final Double inlierRatio;
         if (candidates.size() > 0) {
@@ -109,5 +123,14 @@ public List<PointMatch> filterMatches(final List<PointMatch> candidates,
         return inliers;
     }
 
+    public Matches filterMatches(final Matches candidates,
+                                 final Model model) {
+
+        final List<PointMatch> candidatesList =
+                CanvasFeatureMatchResult.convertMatchesToPointMatchList(candidates);
+        final List<PointMatch> inliersList = filterMatches(candidatesList, model);
+        return CanvasFeatureMatchResult.convertPointMatchListToMatches(inliersList);
+    }
+
     private static final Logger LOG = LoggerFactory.getLogger(CanvasFeatureMatcher.class);
 }

render-app/src/main/java/org/janelia/alignment/match/CanvasMatches.java

@@ -228,6 +228,10 @@ public static CanvasMatches fromJson(final String json) {
         return JSON_HELPER.fromJson(json);
     }
 
+    public static CanvasMatches fromJson(final Reader json) {
+        return JSON_HELPER.fromJson(json);
+    }
+
     public static List<CanvasMatches> fromJsonArray(final String json) {
         // TODO: verify using Arrays.asList optimization is actually faster
         // return JSON_HELPER.fromJsonArray(json);

render-app/src/test/java/org/janelia/alignment/match/CanvasFeatureMatchResultTest.java

@@ -0,0 +1,82 @@
+/**
+ * License: GPL
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License 2
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+package org.janelia.alignment.match;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import mpicbg.models.Point;
+import mpicbg.models.PointMatch;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests the {@link CanvasFeatureMatchResult} class.
+ *
+ * @author Eric Trautman
+ */
+public class CanvasFeatureMatchResultTest {
+
+    @Test
+    public void testConvertMethods() throws Exception {
+
+
+        final List<PointMatch> originalList = new ArrayList<>();
+        for (int i = 0; i < 10; i++) {
+            originalList.add(new PointMatch(new Point(new double[]{i,i*2}), new Point(new double[]{i*10,i*20}), i*0.1));
+        }
+
+        final Matches matches = CanvasFeatureMatchResult.convertPointMatchListToMatches(originalList);
+
+        Assert.assertEquals("incorrect number of matches weights", originalList.size(), matches.getWs().length);
+
+        final List<PointMatch> convertedList = CanvasFeatureMatchResult.convertMatchesToPointMatchList(matches);
+
+        Assert.assertEquals("incorrect number of point matches", originalList.size(), convertedList.size());
+
+        for (int i = 0; i < originalList.size(); i++) {
+            verifyEquality("match " + i, originalList.get(i), convertedList.get(i));
+        }
+
+    }
+
+    private void verifyEquality(final String context,
+                                final PointMatch expected,
+                                final PointMatch actual) {
+
+        verifyEquality(context + " p1", expected.getP1(), actual.getP1());
+        verifyEquality(context + " p2", expected.getP2(), actual.getP2());
+
+    }
+
+    private void verifyEquality(final String context,
+                                final Point expected,
+                                final Point actual) {
+
+        final double[] expectedLocal = expected.getL();
+        final double[] actualLocal = actual.getL();
+
+        Assert.assertEquals("incorrect dimension size for " + context, expectedLocal.length, actualLocal.length);
+
+        for (int i = 0; i < expectedLocal.length; i++) {
+            Assert.assertEquals("incorrect value at index " + i + " of " + context,
+                                expectedLocal[i], actualLocal[i], 0.0001);
+        }
+
+    }
+}

render-ws-java-client/src/main/java/org/janelia/render/client/CommandLineParameters.java

@@ -5,6 +5,8 @@
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.core.JsonProcessingException;
 
+import java.io.Serializable;
+
 import org.janelia.alignment.json.JsonUtils;
 
 /**
@@ -13,7 +15,7 @@
  * @author Eric Trautman
  */
 @Parameters
-public class CommandLineParameters {
+public class CommandLineParameters implements Serializable {
 
     @Parameter(names = "--help", description = "Display this note", help = true)
     protected transient boolean help;

render-ws-java-client/src/main/java/org/janelia/render/client/MatchDataClientParameters.java

@@ -16,19 +16,19 @@ public class MatchDataClientParameters
             names = "--baseDataUrl",
             description = "Base web service URL for data (e.g. http://host[:port]/render-ws/v1)",
             required = true)
-    protected String baseDataUrl;
+    public String baseDataUrl;
 
     @Parameter(
             names = "--owner",
             description = "Match collection owner",
             required = true)
-    protected String owner;
+    public String owner;
 
     @Parameter(
             names = "--collection",
             description = "Match collection name",
             required = true)
-    protected String collection;
+    public String collection;
 
     public MatchDataClientParameters() {
         this.baseDataUrl = null;

render-ws-java-client/src/main/java/org/janelia/render/client/PointMatchClient.java

@@ -325,7 +325,8 @@ public List<CanvasMatches> deriveMatches() throws Exception {
         final CanvasFeatureMatcher matcher = new CanvasFeatureMatcher(parameters.matchRod,
                                                                       parameters.matchMaxEpsilon,
                                                                       parameters.matchMinInlierRatio,
-                                                                      parameters.matchMinNumInliers);
+                                                                      parameters.matchMinNumInliers,
+                                                                      true);
 
         String pUrlString;
         String qUrlString;