Merge pull request #1985 from Clinical-Genomics/release/11.0.1

Release/11.0.1
Clinical-Genomics · May 17, 2022 · 82cebd1 · 82cebd1
2 parents 9758991 + e451449
commit 82cebd1
Show file tree

Hide file tree

Showing 12 changed files with 140 additions and 42 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,11 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [11.0.1]
+
+- When running Deepvariant, set tmpdir to analysis folder and use `intermediate_results_dir`.
+- When running DeepVariant via singularity, use the options `--no-home` and `--cleanenv`.
+
 ## [11.0.0]
 
 - HmtNote: annotate mitochondrial variants in VCF file

diff --git a/definitions/rd_dna_parameters.yaml b/definitions/rd_dna_parameters.yaml
@@ -87,6 +87,16 @@ gpu_capable_executables:
     - run_deeptrio
     - call_variants
   type: mip
+##  Isolate singularity containers (--no-home, --cleanenv)
+isolate_singularity_containers:
+  associated_recipe:
+    - mip
+  data_type: ARRAY
+  default:
+    - call_variants
+    - run_deeptrio
+    - run_deepvariant
+  type: mip
 ## HumanGenomeReference
 human_genome_reference:
   associated_recipe:

diff --git a/lib/MIP/Cli/Mip/Analyse/Rd_dna.pm b/lib/MIP/Cli/Mip/Analyse/Rd_dna.pm
@@ -1961,6 +1961,16 @@ q{Regular expression file containing the regular expression to be used for each
         )
     );
 
+    option(
+        q{isolate_singularity_containers} => (
+            cmd_tags      => [q{Contain these containers}],
+            documentation =>
+              q{Run these singularity executables with --no-home and --cleanenv option},
+            is  => qw{rw},
+            isa => ArrayRef,
+        )
+    );
+
     return;
 }
 

diff --git a/lib/MIP/Constants.pm b/lib/MIP/Constants.pm
@@ -81,7 +81,7 @@ Readonly our %ANALYSIS => (
 );
 
 ## Set MIP version
-Readonly our $MIP_VERSION => q{11.0.0};
+Readonly our $MIP_VERSION => q{11.0.1};
 
 ## Cli
 Readonly our $MOOSEX_APP_SCEEN_WIDTH => 160;

diff --git a/lib/MIP/Environment/Container.pm b/lib/MIP/Environment/Container.pm
@@ -128,13 +128,25 @@ sub build_container_cmd {
                 $gpu_switch = $container_href->{$container_name}{gpu_support} ? 1 : 0;
             }
 
+            ## Isolate singularity container
+            my $no_home   = 0;
+            my $clean_env = 0;
+            if ( any { $_ eq $executable_name }
+                @{ $active_parameter_href->{isolate_singularity_containers} } )
+            {
+                $no_home   = 1;
+                $clean_env = 1;
+            }
+
             my @cmds = run_container(
                 {
                     bind_paths_ref    => \@bind_paths,
                     executable_name   => $executable_name,
                     container_manager => $container_manager,
                     container_path    => $container_href->{$container_name}{uri},
                     gpu_switch        => $gpu_switch,
+                    no_home           => $no_home,
+                    clean_env         => $clean_env,
                 }
             );
 
@@ -665,6 +677,7 @@ sub run_container {
 ## Function : Run a docker container or exec a singularity image
 ## Returns  : @commands
 ## Arguments: $bind_paths_ref         => Bind host directory to container {REF}
+##          : $clean_env              => Start with clean environment
 ##          : $container_cmds_ref     => Cmds to be executed in container {REF}
 ##          : $container_manager      => Container manager
 ##          : $container_path         => Path to container
@@ -673,6 +686,7 @@ sub run_container {
 ##          : $filehandle             => Filehandle to write to
 ##          : $gpu_switch             => Use nvidia experimental support
 ##          : $image                  => Image to run
+##          : $no_home                => Don't mount home if it isn't the current working dir
 ##          : $remove                 => Remove stopped container
 ##          : $stderrfile_path        => Stderrfile path
 ##          : $stderrfile_path_append => Append stderr info to file path
@@ -683,12 +697,14 @@ sub run_container {
 
     ## Flatten argument(s)
     my $bind_paths_ref;
+    my $clean_env;
     my $container_cmds_ref;
     my $container_manager;
     my $container_path;
     my $executable_name;
     my $filehandle;
     my $gpu_switch;
+    my $no_home;
     my $stderrfile_path;
     my $stderrfile_path_append;
     my $stdinfile_path;
@@ -703,6 +719,11 @@ sub run_container {
             store       => \$bind_paths_ref,
             strict_type => 1,
         },
+        clean_env => {
+            allow       => [ undef, 0, 1 ],
+            store       => \$clean_env,
+            strict_type => 1,
+        },
         container_cmds_ref => {
             default     => [],
             store       => \$container_cmds_ref,
@@ -732,6 +753,11 @@ sub run_container {
             store       => \$gpu_switch,
             strict_type => 1,
         },
+        no_home => {
+            allow       => [ undef, 0, 1 ],
+            store       => \$no_home,
+            strict_type => 1,
+        },
         remove => {
             allow       => [ undef, 0, 1 ],
             default     => 1,
@@ -787,10 +813,12 @@ sub run_container {
         singularity => {
             arg_href => {
                 bind_paths_ref         => $bind_paths_ref,
+                clean_env              => $clean_env,
                 container_cmds_ref     => $container_cmds_ref,
                 filehandle             => $filehandle,
                 image                  => $container_path,
                 gpu_switch             => $gpu_switch,
+                no_home                => $no_home,
                 stderrfile_path        => $stderrfile_path,
                 stderrfile_path_append => $stderrfile_path_append,
                 stdoutfile_path        => $stdoutfile_path,

diff --git a/lib/MIP/Program/Deepvariant.pm b/lib/MIP/Program/Deepvariant.pm
@@ -31,25 +31,27 @@ sub deepvariant {
 
 ## Function : Perl wrapper for generic commands module
 ## Returns  : @commands
-## Arguments: $bedfile                => Bed file containing the list of regions we want to process
-##          : $filehandle             => Filehandle to write to
-##          : $infile_path            => Aligned, sorted, indexed bam file containing the reads we want to call
-##          : $model_type             => Type of model to use for variant calling. Allowed values WES, WGS, or PACBIO
-##          : $num_shards             => Number of files the input is split into for the make examples step
-##          : $outfile_path           => Path to the output gvcf file
-##          : $outfile_path_vcf       => Path to the output vcf file
-##          : $referencefile_path     => Path to genome reference
-##          : $stderrfile_path        => Stderrfile path
-##          : $stderrfile_path_append => Append stderr info to file path
-##          : $stdinfile_path         => Stdinfile path
-##          : $stdoutfile_path        => Stdoutfile path
+## Arguments: $bedfile                  => Bed file containing the list of regions we want to process
+##          : $filehandle               => Filehandle to write to
+##          : $infile_path              => Aligned, sorted, indexed bam file containing the reads we want to call
+##          : $intermediate_results_dir => Store intermediate results here
+##          : $model_type               => Type of model to use for variant calling. Allowed values WES, WGS, or PACBIO
+##          : $num_shards               => Number of files the input is split into for the make examples step
+##          : $outfile_path             => Path to the output gvcf file
+##          : $outfile_path_vcf         => Path to the output vcf file
+##          : $referencefile_path       => Path to genome reference
+##          : $stderrfile_path          => Stderrfile path
+##          : $stderrfile_path_append   => Append stderr info to file path
+##          : $stdinfile_path           => Stdinfile path
+##          : $stdoutfile_path          => Stdoutfile path
 
     my ($arg_href) = @_;
 
     ## Flatten argument(s)
     my $bedfile;
     my $filehandle;
     my $infile_path;
+    my $intermediate_results_dir;
     my $model_type;
     my $num_shards;
     my $outfile_path;
@@ -76,6 +78,10 @@ sub deepvariant {
             store       => \$infile_path,
             strict_type => 1,
         },
+        intermediate_results_dir => {
+            store       => \$intermediate_results_dir,
+            strict_type => 1
+        },
         model_type => {
             allow       => [qw{ WES WGS PACBIO }],
             defined     => 1,
@@ -139,6 +145,9 @@ sub deepvariant {
     if ($bedfile) {
         push @commands, q{--regions} . $EQUALS . $bedfile;
     }
+    if ($intermediate_results_dir) {
+        push @commands, q{--intermediate_results_dir} . $SPACE . $intermediate_results_dir;
+    }
 
     push @commands,
       unix_standard_streams(

diff --git a/lib/MIP/Program/Singularity.pm b/lib/MIP/Program/Singularity.pm
@@ -31,23 +31,27 @@ sub singularity_exec {
 
 ## Function : Perl wrapper for writing singularity execute command. Based on singularity v3.1.
 ## Returns  : @commands
-## Arguments: $bind_paths_ref                 => Array with paths to bind {REF}
-##          : $filehandle                     => Filehandle to write to
-##          : $image                          => Singularity container name
-##          : $gpu_switch                     => Add nvidia experimental support
-##          : $container_cmds_ref             => Array with commands to be executed inside container {REF}
-##          : $stderrfile_path                => Stderrfile path
-##          : $stderrfile_path_append         => Append stderr info to file path
-##          : $stdoutfile_path                => Stdoutfile path
+## Arguments: $bind_paths_ref          => Array with paths to bind {REF}
+##          : $container_cmds_ref      => Array with commands to be executed inside container {REF}
+##          : $clean_env               => Start with clean environment
+##          : $filehandle              => Filehandle to write to
+##          : $gpu_switch              => Add nvidia experimental support
+##          : $image                   => Singularity container name
+##          : $no_home                 => Don't mount home if it isn't the current working dir
+##          : $stderrfile_path         => Stderrfile path
+##          : $stderrfile_path_append  => Append stderr info to file path
+##          : $stdoutfile_path         => Stdoutfile path
 
     my ($arg_href) = @_;
 
     ## Flatten argument(s)
     my $bind_paths_ref;
+    my $clean_env;
     my $container_cmds_ref;
     my $filehandle;
     my $gpu_switch;
     my $image;
+    my $no_home;
     my $stderrfile_path;
     my $stderrfile_path_append;
     my $stdoutfile_path;
@@ -58,6 +62,11 @@ sub singularity_exec {
             store       => \$bind_paths_ref,
             strict_type => 1,
         },
+        clean_env => {
+            allow       => [ undef, 0, 1 ],
+            store       => \$clean_env,
+            strict_type => 1,
+        },
         container_cmds_ref => {
             default     => [],
             store       => \$container_cmds_ref,
@@ -77,6 +86,11 @@ sub singularity_exec {
             store       => \$gpu_switch,
             strict_type => 1,
         },
+        no_home => {
+            allow       => [ undef, 0, 1 ],
+            store       => \$no_home,
+            strict_type => 1,
+        },
         stderrfile_path => {
             store       => \$stderrfile_path,
             strict_type => 1,
@@ -104,6 +118,14 @@ sub singularity_exec {
         push @commands, q{--bind} . $SPACE . join $COMMA, @{$bind_paths_ref};
     }
 
+    if ($clean_env) {
+        push @commands, q{--cleanenv};
+    }
+
+    if ($no_home) {
+        push @commands, q{--no-home};
+    }
+
     ## Add container
     push @commands, $image;
 

diff --git a/lib/MIP/Recipes/Analysis/Deepvariant.pm b/lib/MIP/Recipes/Analysis/Deepvariant.pm
@@ -16,7 +16,7 @@ use autodie qw{ :all };
 use Readonly;
 
 ## MIPs lib/
-use MIP::Constants qw{ $LOG_NAME $NEWLINE $UNDERSCORE };
+use MIP::Constants qw{ $CONTAINER_MANAGER $EQUALS $LOG_NAME $NEWLINE $SPACE $UNDERSCORE };
 
 BEGIN {
 
@@ -121,6 +121,7 @@ sub analysis_deepvariant {
 
     use MIP::File_info qw{ get_io_files parse_io_outfiles };
     use MIP::Processmanagement::Processes qw{ submit_recipe };
+    use MIP::Program::Gnu::Bash qw{ gnu_export };
     use MIP::Program::Deepvariant qw{ deepvariant };
     use MIP::Recipe qw{ parse_recipe_prerequisites };
     use MIP::Sample_info qw{ set_recipe_metafile_in_sample_info set_recipe_outfile_in_sample_info };
@@ -201,15 +202,24 @@ sub analysis_deepvariant {
 
     say {$filehandle} q{## } . $recipe_name;
 
+    if ( $CONTAINER_MANAGER eq q{singularity} ) {
+
+        print {$filehandle} q{SINGULARITYENV_TMPDIR}
+          . $EQUALS
+          . $active_parameter_href->{temp_directory}
+          . $SPACE;
+    }
+
     deepvariant(
         {
-            filehandle         => $filehandle,
-            infile_path        => $infile_path,
-            model_type         => uc $analysis_type,
-            num_shards         => $recipe{core_number},
-            outfile_path       => $outfile_path,
-            outfile_path_vcf   => $outfile_path_vcf,
-            referencefile_path => $active_parameter_href->{human_genome_reference},
+            filehandle               => $filehandle,
+            infile_path              => $infile_path,
+            intermediate_results_dir => $active_parameter_href->{temp_directory},
+            model_type               => uc $analysis_type,
+            num_shards               => $recipe{core_number},
+            outfile_path             => $outfile_path,
+            outfile_path_vcf         => $outfile_path_vcf,
+            referencefile_path       => $active_parameter_href->{human_genome_reference},
         }
     );
 

diff --git a/t/deepvariant.t b/t/deepvariant.t
@@ -22,16 +22,13 @@ use lib catdir( dirname($Bin), q{lib} );
 use MIP::Constants qw{ $COMMA $SPACE };
 use MIP::Test::Commands qw{ test_function };
 
-
 BEGIN {
 
     use MIP::Test::Fixtures qw{ test_import };
 
 ### Check all internal dependency modules and imports
 ## Modules with import
-    my %perl_module = (
-        q{MIP::Program::Deepvariant} => [qw{ deepvariant }],
-);
+    my %perl_module = ( q{MIP::Program::Deepvariant} => [qw{ deepvariant }], );
 
     test_import( { perl_module_href => \%perl_module, } );
 }
@@ -77,8 +74,7 @@ my %required_argument = (
     },
     referencefile_path => {
         input           => catfile(qw{ reference_dir human_genome_build.fasta }),
-        expected_output => q{--ref=}
-          . catfile(qw{ reference_dir human_genome_build.fasta}),
+        expected_output => q{--ref=} . catfile(qw{ reference_dir human_genome_build.fasta}),
     },
     model_type => {
         input           => q{WES},
@@ -103,7 +99,10 @@ my %specific_argument = (
         input           => catfile(qw{ dir infile.bed }),
         expected_output => q{--regions=} . catfile(qw{ dir infile.bed }),
     },
-
+    intermediate_results_dir => {
+        input           => catdir(qw{ temp dir}),
+        expected_output => q{--intermediate_results_dir} . $SPACE . catdir(qw{ temp dir }),
+    },
 );
 
 ## Coderef - enables generalized use of generate call

diff --git a/t/mip_analyse_rd_dna.test b/t/mip_analyse_rd_dna.test
@@ -163,7 +163,7 @@ while ( my ( $pedigree, $pedigree_path ) = each %pedigree ) {
 }
 
 ## Clean-up
-remove_tree( catfile( $cluster_constant_path, qw{ 643594-miptest analysis } ) );
+#remove_tree( catfile( $cluster_constant_path, qw{ 643594-miptest analysis } ) );
 
 GENOME_BUILD:
 foreach my $genome_build (@genome_builds) {