Add MASGITOFF (Mastodon Git Optimized File Format) Support

pom.xml

@@ -38,11 +38,6 @@
 			<artifactId>mastodon-tomancak</artifactId>
 			<version>${mastodon-tomancak.version}</version>
 		</dependency>
-		<dependency>
-			<groupId>org.mastodon</groupId>
-			<artifactId>mastodon-collection</artifactId>
-			<version>${mastodon-collection.version}</version>
-		</dependency>
 
 		<dependency>
 			<groupId>org.eclipse.jgit</groupId>
@@ -81,7 +76,6 @@
 			<artifactId>imagej-legacy</artifactId>
 			<scope>test</scope>
 		</dependency>
-
 	</dependencies>
 
 	<mailingLists>
@@ -113,9 +107,8 @@
 		<license.organizationName>Mastodon authors</license.organizationName>
 		<license.copyrightOwners>Matthias Arzt</license.copyrightOwners>
 
-		<mastodon.version>1.0.0-beta-30</mastodon.version>
+		<mastodon.version>1.0.0-beta-31-SNAPSHOT</mastodon.version>
 		<mastodon-tomancak.version>0.4.2</mastodon-tomancak.version>
-		<mastodon-collection.version>1.0.0-beta-26</mastodon-collection.version>
 
 		<!-- NB: Deploy releases to the SciJava Maven repository. -->
 		<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>

src/main/java/org/mastodon/mamut/collaboration/MastodonGitRepository.java

@@ -6,13 +6,13 @@
  * %%
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
- * 
+ *
  * 1. Redistributions of source code must retain the above copyright notice,
  *    this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright notice,
  *    this list of conditions and the following disclaimer in the documentation
  *    and/or other materials provided with the distribution.
- * 
+ *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -36,6 +36,7 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
+import org.apache.commons.io.FileUtils;
 import org.eclipse.jgit.api.CheckoutCommand;
 import org.eclipse.jgit.api.CommitCommand;
 import org.eclipse.jgit.api.CreateBranchCommand;
@@ -50,26 +51,22 @@
 import org.eclipse.jgit.lib.StoredConfig;
 import org.eclipse.jgit.transport.PushResult;
 import org.eclipse.jgit.transport.RemoteRefUpdate;
-import org.mastodon.graph.io.RawGraphIO;
 import org.mastodon.mamut.MainWindow;
 import org.mastodon.mamut.ProjectModel;
+import org.mastodon.mamut.collaboration.credentials.PersistentCredentials;
+import org.mastodon.mamut.collaboration.exceptions.GraphMergeConflictException;
+import org.mastodon.mamut.collaboration.exceptions.GraphMergeException;
 import org.mastodon.mamut.collaboration.exceptions.MastodonGitException;
+import org.mastodon.mamut.collaboration.io.MasgitoffIo;
+import org.mastodon.mamut.collaboration.io.MasgitoffProjectLoader;
+import org.mastodon.mamut.collaboration.io.MasgitoffProjectSaver;
 import org.mastodon.mamut.collaboration.settings.MastodonGitSettingsService;
 import org.mastodon.mamut.collaboration.utils.ConflictUtils;
 import org.mastodon.mamut.collaboration.utils.ReloadFromDiskUtils;
-import org.mastodon.mamut.feature.MamutRawFeatureModelIO;
-import org.mastodon.mamut.io.ProjectLoader;
-import org.mastodon.mamut.io.ProjectSaver;
 import org.mastodon.mamut.io.project.MamutProject;
-import org.mastodon.mamut.io.project.MamutProjectIO;
-import org.mastodon.mamut.model.Link;
 import org.mastodon.mamut.model.Model;
-import org.mastodon.mamut.model.Spot;
-import org.mastodon.mamut.collaboration.credentials.PersistentCredentials;
-import org.mastodon.mamut.collaboration.exceptions.GraphMergeConflictException;
-import org.mastodon.mamut.collaboration.exceptions.GraphMergeException;
-import org.mastodon.mamut.tomancak.merging.Dataset;
 import org.mastodon.mamut.tomancak.merging.MergeDatasets;
+import org.mastodon.mamut.tomancak.merging.MergeModels;
 import org.scijava.Context;
 
 // make it one synchronized class per repository
@@ -133,7 +130,7 @@ public static MastodonGitRepository shareProject(
 
 			addGitIgnoreFile( git, directory );
 
-			ProjectSaver.saveProject( mastodonProjectPath.toFile(), projectModel );
+			MasgitoffProjectSaver.saveProject( mastodonProjectPath.toFile(), projectModel );
 			copyXmlsFromTo( mastodonProjectPath, initialStateFolder );
 			git.add().addFilepattern( INITIAL_STATE_FOLDER ).addFilepattern( MASTODON_PROJECT_FOLDER ).call();
 			git.commit().setMessage( "Share mastodon project" ).call();
@@ -200,7 +197,7 @@ public static void openProjectInRepository( final Context context, final File di
 		final String mastodonFile = directory.toPath().resolve( MASTODON_PROJECT_FOLDER ).toString();
 		final boolean restoreGUIState = true;
 		final boolean authorizeSubstituteDummyData = true;
-		final ProjectModel newProject = ProjectLoader.open( mastodonFile, context, restoreGUIState, authorizeSubstituteDummyData );
+		final ProjectModel newProject = MasgitoffProjectLoader.open( mastodonFile, context, restoreGUIState, authorizeSubstituteDummyData );
 		new MainWindow( newProject ).setVisible( true );
 	}
 
@@ -211,14 +208,19 @@ public synchronized void commitWithoutSave( final String message ) throws Except
 	{
 		try (final Git git = initGit())
 		{
-			git.add().addFilepattern( MASTODON_PROJECT_FOLDER ).call();
-			final CommitCommand commit = git.commit();
-			commit.setMessage( message );
-			commit.setAuthor( settingsService.getPersonIdent() );
-			commit.call();
+			commitWithoutSave( git, message );
 		}
 	}
 
+	private void commitWithoutSave( Git git, String message ) throws GitAPIException
+	{
+		git.add().addFilepattern( MASTODON_PROJECT_FOLDER ).call();
+		final CommitCommand commit = git.commit().setAll( true );
+		commit.setMessage( message );
+		commit.setAuthor( settingsService.getPersonIdent() );
+		commit.call();
+	}
+
 	/**
 	 * Save the project and add a commit.
 	 * <p>
@@ -231,15 +233,17 @@ public synchronized void commitWithoutSave( final String message ) throws Except
 	 */
 	public synchronized void commit( final String message ) throws Exception
 	{
-		ProjectSaver.saveProject( projectRoot, projectModel );
+		final boolean clean = isClean();
+		if ( clean )
+			throw new MastodonGitException( "There are no changes to commit." );
 		commitWithoutSave( message );
 	}
 
 	/**
 	 * This method performs an operation similar to {@code "git push origin --set-upstream <current-branch>"}.
 	 *
 	 * @throws MastodonGitException if the push fails because the remote server has changes that the local
-	 * repository does not have. Or if the push fails for any other reason.
+	 *                              repository does not have. Or if the push fails for any other reason.
 	 */
 	public synchronized void push() throws Exception
 	{
@@ -383,19 +387,19 @@ public synchronized String getCurrentBranch() throws Exception
 	 */
 	public synchronized void mergeBranch( final String selectedBranch ) throws Exception
 	{
-		final Context context = projectModel.getContext();
 		try (final Git git = initGit())
 		{
 			ensureClean( git, "merging" );
 			final String currentBranch = getCurrentBranch();
-			final Dataset dsA = new Dataset( projectRoot.getAbsolutePath() );
+			final Model dsA = loadModel( projectRoot );
 			git.checkout().setName( selectedBranch ).call();
-			final Dataset dsB = new Dataset( projectRoot.getAbsolutePath() );
+			final Model dsB = loadModel( projectRoot );
 			git.checkout().setName( currentBranch ).call();
 			git.merge().setCommit( false ).include( git.getRepository().exactRef( selectedBranch ) ).call(); // TODO selected branch, should not be a string but a ref instead
 			final MamutProject project = projectModel.getProject();
 			project.setProjectRoot( projectRoot );
-			mergeAndCommit( context, project, dsA, dsB, "Merge commit generated with Mastodon" );
+			merge( project, dsA, dsB );
+			commitWithoutSave( git, "Merge commit generated with Mastodon" );
 			reloadFromDisk();
 		}
 	}
@@ -407,7 +411,6 @@ public synchronized void mergeBranch( final String selectedBranch ) throws Excep
 	 */
 	public synchronized void pull() throws Exception
 	{
-		final Context context = projectModel.getContext();
 		try (final Git git = initGit())
 		{
 			ensureClean( git, "pulling" );
@@ -422,7 +425,7 @@ public synchronized void pull() throws Exception
 				{
 					final MamutProject project = projectModel.getProject();
 					project.setProjectRoot( projectRoot );
-					automaticMerge( context, project, projectRoot, git );
+					automaticMerge( project, projectRoot, git );
 				}
 			}
 			finally
@@ -433,17 +436,18 @@ public synchronized void pull() throws Exception
 		}
 	}
 
-	private void automaticMerge( final Context context, final MamutProject project, final File projectRoot, final Git git )
+	private void automaticMerge( final MamutProject project, final File projectRoot, final Git git )
 	{
 		try
 		{
 			git.checkout().setAllPaths( true ).setStage( CheckoutCommand.Stage.OURS ).call();
-			final Dataset dsA = new Dataset( projectRoot.getAbsolutePath() );
+			final Model dsA = loadModel( projectRoot );
 			git.checkout().setAllPaths( true ).setStage( CheckoutCommand.Stage.THEIRS ).call();
-			final Dataset dsB = new Dataset( projectRoot.getAbsolutePath() );
+			final Model dsB = loadModel( projectRoot );
 			git.checkout().setAllPaths( true ).setStage( CheckoutCommand.Stage.OURS ).call();
 			final String commitMessage = "Automatic merge by Mastodon during pull";
-			mergeAndCommit( context, project, dsA, dsB, commitMessage );
+			merge( project, dsA, dsB );
+			commitWithoutSave( git, commitMessage );
 		}
 		catch ( final GraphMergeException e )
 		{
@@ -455,33 +459,35 @@ private void automaticMerge( final Context context, final MamutProject project,
 		}
 	}
 
-	private void mergeAndCommit( final Context context, final MamutProject project, final Dataset datasetA, final Dataset datasetB, final String commitMessage ) throws Exception
+	private Model loadModel( File projectRoot ) throws IOException
 	{
-		final Model mergedModel = merge( datasetA, datasetB );
+		return MasgitoffIo.readMasgitoff( new File( projectRoot, "model" ) );
+	}
+
+	private static void merge( MamutProject project, Model modelA, Model modelB ) throws IOException
+	{
+		final Model mergedModel = merge( modelA, modelB );
 		if ( ConflictUtils.hasConflict( mergedModel ) )
 			throw new GraphMergeConflictException();
 		ConflictUtils.removeMergeConflictTagSets( mergedModel );
-		saveModel( context, mergedModel, project );
-		commitWithoutSave( commitMessage );
+		saveModel( mergedModel, project );
 	}
 
-	private static void saveModel( final Context context, final Model model, final MamutProject project ) throws IOException
+	private static void saveModel( final Model model, final MamutProject project ) throws IOException
 	{
-		try (final MamutProject.ProjectWriter writer = project.openForWriting())
-		{
-			MamutProjectIO.save( project, writer );
-			final RawGraphIO.GraphToFileIdMap< Spot, Link > idmap = model.saveRaw( writer );
-			MamutRawFeatureModelIO.serialize( context, model, idmap, writer );
-		}
+		File modelFolder = new File( project.getProjectRoot(), "model" );
+		if ( modelFolder.isDirectory() )
+			FileUtils.deleteDirectory( modelFolder );
+		MasgitoffIo.writeMasgitoff( model, modelFolder );
 	}
 
-	private static Model merge( final Dataset dsA, final Dataset dsB )
+	private static Model merge( final Model modelA, final Model modelB )
 	{
 		final MergeDatasets.OutputDataSet output = new MergeDatasets.OutputDataSet( new Model() );
 		final double distCutoff = 1000;
 		final double mahalanobisDistCutoff = 1;
 		final double ratioThreshold = 2;
-		MergeDatasets.merge( dsA, dsB, output, distCutoff, mahalanobisDistCutoff, ratioThreshold );
+		MergeModels.merge( modelA, modelB, output, distCutoff, mahalanobisDistCutoff, ratioThreshold );
 		return output.getModel();
 	}
 
@@ -568,7 +574,7 @@ public synchronized boolean isClean() throws Exception
 
 	private boolean isClean( final Git git ) throws Exception
 	{
-		ProjectSaver.saveProject( projectRoot, projectModel );
+		MasgitoffProjectSaver.saveProject( projectRoot, projectModel );
 		return git.status().call().isClean();
 	}
 }

src/main/java/org/mastodon/mamut/collaboration/io/Index.java

@@ -0,0 +1,122 @@
+package org.mastodon.mamut.collaboration.io;
+
+import org.mastodon.Ref;
+import org.mastodon.RefPool;
+
+import gnu.trove.list.TIntList;
+import gnu.trove.list.array.TIntArrayList;
+import gnu.trove.map.TObjectIntMap;
+import gnu.trove.procedure.TObjectIntProcedure;
+
+public class Index< T extends Ref< T > >
+{
+	private final TIntList idToPoolId = new TIntArrayList();
+
+	private final TIntList poolIdToId = new TIntArrayList();
+
+	private final RefPool< T > pool;
+
+	public Index( final RefPool< T > pool )
+	{
+		this.pool = pool;
+	}
+
+	public void put( final T obj, final int id )
+	{
+		if ( id < 0 )
+			throw new IllegalArgumentException( "id must be >= 0" );
+		if ( containsId( id ) )
+			throw new IllegalArgumentException( "id already in use." );
+		final int poolId = obj.getInternalPoolIndex();
+		if ( poolId < 0 )
+			throw new IllegalArgumentException( "Object is not valid." );
+		if ( containsObject( obj ) )
+			throw new IllegalArgumentException( "There is already an id for the given object." );
+		setAndGrow( idToPoolId, id, poolId );
+		setAndGrow( poolIdToId, poolId, id );
+	}
+
+	private static void setAndGrow( final TIntList list, final int index, final int value )
+	{
+		if ( index >= list.size() )
+			list.fill( list.size(), index + 1, -1 );
+		list.set( index, value );
+	}
+
+	public int getId( final T object )
+	{
+		final int poolId = object.getInternalPoolIndex();
+		if ( poolId < 0 || poolId >= poolIdToId.size() )
+			return -1;
+		return poolIdToId.get( poolId );
+	}
+
+	public int getOrCreateId( final T object )
+	{
+		final int id = getId( object );
+		if ( id >= 0 )
+			return id;
+		final int newId = idToPoolId.size();
+		final int poolId = object.getInternalPoolIndex();
+		idToPoolId.add( poolId );
+		setAndGrow( poolIdToId, poolId, newId );
+		return newId;
+	}
+
+	public T getObject( final int id, final T ref )
+	{
+		if ( id < 0 || id >= idToPoolId.size() )
+			return null;
+		final int poolId = idToPoolId.get( id );
+		if ( poolId < 0 )
+			return null;
+		return pool.getObject( poolId, ref );
+	}
+
+	public int getMaxId()
+	{
+		return idToPoolId.size() - 1;
+	}
+
+	public boolean containsId( final int id )
+	{
+		return id >= 0 && id < idToPoolId.size() && idToPoolId.get( id ) >= 0;
+	}
+
+	public boolean containsObject( final T object )
+	{
+		final int poolId = object.getInternalPoolIndex();
+		return poolId >= 0 && poolId < poolIdToId.size() && poolIdToId.get( poolId ) >= 0;
+	}
+
+	public T createRef() {
+		return pool.createRef();
+	}
+
+	public void forEach( IndexAction< T > action )
+	{
+		T ref = pool.createRef();
+		try
+		{
+			for ( int i = 0; i < idToPoolId.size(); i++ )
+			{
+				final int poolId = idToPoolId.get( i );
+				if ( poolId >= 0 )
+				{
+					final T obj = pool.getObject( poolId, ref );
+					if ( obj != null )
+						action.run( obj, i );
+				}
+			}
+		}
+		finally
+		{
+			pool.releaseRef( ref );
+		}
+	}
+
+	public interface IndexAction< T >
+	{
+		void run( T object, int index );
+	}
+}