Merge pull request #79 from bioinfo-chru-strasbourg/fix_logger

fix #78 logger #4 docs

.gitignore

@@ -4,8 +4,10 @@ howard.egg-info/
 .cache
 .vscode
 .coverage
-tests/data/annotations/annovar/
-tests/data/annotations/snpeff/
-tests/data/annotations/hg19.fa
-tests/data/annotations/hg19.fa.fai
-tests/data/annotations/.DS_Store
+tests/databases/annovar/
+tests/databases/snpeff/
+tests/databases/genomes/
+tests/databases/refseq/
+tests/databases/hg19.fa
+tests/databases/hg19.fa.fai
+tests/databases/.DS_Store

README.md

@@ -113,12 +113,12 @@ howard query --input=tests/data/example.vcf.gz --explode_infos --query='SELECT "
 
 - Query a Parquet file with specific columns (e.g. from VCF convertion to Parquet)
 ```
-howard query --query="SELECT * FROM 'tests/data/annotations/dbnsfp42a.parquet' WHERE \"INFO/Interpro_domain\" NOT NULL ORDER BY \"INFO/SiPhy_29way_logOdds_rankscore\" DESC"
+howard query --query="SELECT * FROM 'tests/databases/annotations/hg19/dbnsfp42a.parquet' WHERE \"INFO/Interpro_domain\" NOT NULL ORDER BY \"INFO/SiPhy_29way_logOdds_rankscore\" DESC"
 ```
 
 - Query multiple Parquet files, merge INFO columns, and extract as TSV (in VCF format)
 ```
-howard query --query="SELECT \"#CHROM\" AS \"#CHROM\", POS AS POS, '' AS ID, REF AS REF, ALT AS ALT, '' AS QUAL, '' AS FILTER, STRING_AGG(INFO, ';') AS INFO FROM 'tests/data/annotations/*.parquet' GROUP BY \"#CHROM\", POS, REF, ALT" --output=/tmp/full_annotation.tsv
+howard query --query="SELECT \"#CHROM\" AS \"#CHROM\", POS AS POS, '' AS ID, REF AS REF, ALT AS ALT, '' AS QUAL, '' AS FILTER, STRING_AGG(INFO, ';') AS INFO FROM 'tests/databases/annotations/hg19/*.parquet' GROUP BY \"#CHROM\", POS, REF, ALT" --output=/tmp/full_annotation.tsv
 ```
 
 
@@ -128,12 +128,12 @@ Annotation is mainly based on a build-in Parquet annotation method, and tools su
 
 - VCF annotation with Parquet and VCF databases, output as VCF format
 ```
-howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.vcf.gz --annotations='tests/data/annotations/dbnsfp42a.parquet,tests/data/annotations/gnomad211_genome.parquet,tests/data/annotations/cosmic70.vcf.gz'
+howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.vcf.gz --annotations='tests/databases/annotations/hg19/dbnsfp42a.parquet,tests/databases/annotations/hg19/gnomad211_genome.parquet,tests/databases/annotations/hg19/cosmic70.vcf.gz'
 ```
 
 - VCF annotation with Clinvar Parquet, Annovar refGene and snpEff databases, output as TSV format
 ```
-howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.tsv --annotations='annovar:refGene,snpeff,tests/data/annotations/clinvar_20210123.parquet'
+howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.tsv --annotations='annovar:refGene,snpeff,tests/databases/annotations/hg19/clinvar_20210123.parquet'
 ```
 
 ## Calculation
@@ -226,20 +226,20 @@ howard process --config=config/config.json --param=config/param.json --input=tes
     },
     "parquet": {
       "annotations": {
-        "tests/data/annotations/avsnp150.parquet": {
+        "tests/databases/annotations/hg19/avsnp150.parquet": {
           "INFO": null
         },
-        "tests/data/annotations/dbnsfp42a.parquet": {
+        "tests/databases/annotations/hg19/dbnsfp42a.parquet": {
           "INFO": null
         },
-        "tests/data/annotations/gnomad211_genome.parquet": {
+        "tests/databases/annotations/hg19/gnomad211_genome.parquet": {
           "INFO": null
         }
       }
     },
     "bcftools": {
       "annotations": {
-        "tests/data/annotations/cosmic70.vcf.gz": {
+        "tests/databases/annotations/hg19/cosmic70.vcf.gz": {
           "INFO": null
         }
       }

config/param.json

@@ -16,20 +16,20 @@
     },
     "parquet": {
       "annotations": {
-        "tests/data/annotations/avsnp150.parquet": {
+        "tests/databases/annotations/hg19/avsnp150.parquet": {
           "INFO": null
         },
-        "tests/data/annotations/dbnsfp42a.parquet": {
+        "tests/databases/annotations/hg19/dbnsfp42a.parquet": {
           "INFO": null
         },
-        "tests/data/annotations/gnomad211_genome.parquet": {
+        "tests/databases/annotations/hg19/gnomad211_genome.parquet": {
           "INFO": null
         }
       }
     },
     "bcftools": {
       "annotations": {
-        "tests/data/annotations/cosmic70.vcf.gz": {
+        "tests/databases/annotations/hg19/cosmic70.vcf.gz": {
           "INFO": null
         }
       }

docs/howard.md

@@ -252,8 +252,8 @@ Shared options:
                         Default: INFO
 
 Usage examples:
-   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.vcf.gz --annotations='tests/data/annotations/avsnp150.parquet,tests/data/annotations/dbnsfp42a.parquet,tests/data/annotations/gnomad211_genome.parquet' 
-   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.tsv --annotations='annovar:refGene,snpeff,tests/data/annotations/clinvar_20210123.parquet'
+   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.vcf.gz --annotations='tests/databases/annotations/hg19/avsnp150.parquet,tests/databases/annotations/hg19/dbnsfp42a.parquet,tests/databases/annotations/hg19/gnomad211_genome.parquet' 
+   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.tsv --annotations='annovar:refGene,snpeff,tests/databases/annotations/hg19/clinvar_20210123.parquet'
 ```
 
 # Calculation
@@ -397,8 +397,8 @@ Shared options:
 Usage examples:
    howard query --input=tests/data/example.vcf.gz --query="SELECT * FROM variants WHERE REF = 'A' AND POS < 100000" 
    howard query --input=tests/data/example.vcf.gz --explode_infos --query='SELECT "#CHROM", POS, REF, ALT, "INFO/DP", "INFO/CLNSIG", sample2, sample3 FROM variants WHERE "INFO/DP" >= 50 OR "INFO/CLNSIG" NOT NULL ORDER BY "INFO/DP" DESC' 
-   howard query --query="SELECT * FROM 'tests/data/annotations/dbnsfp42a.parquet' WHERE \"INFO/Interpro_domain\" NOT NULL ORDER BY \"INFO/SiPhy_29way_logOdds_rankscore\" DESC" 
-   howard query --query="SELECT \"#CHROM\" AS \"#CHROM\", POS AS POS, '' AS ID, REF AS REF, ALT AS ALT, '' AS QUAL, '' AS FILTER, STRING_AGG(INFO, ';') AS INFO FROM 'tests/data/annotations/*.parquet' GROUP BY \"#CHROM\", POS, REF, ALT" --output=/tmp/full_annotation.tsv 
+   howard query --query="SELECT * FROM 'tests/databases/annotations/hg19/dbnsfp42a.parquet' WHERE \"INFO/Interpro_domain\" NOT NULL ORDER BY \"INFO/SiPhy_29way_logOdds_rankscore\" DESC" 
+   howard query --query="SELECT \"#CHROM\" AS \"#CHROM\", POS AS POS, '' AS ID, REF AS REF, ALT AS ALT, '' AS QUAL, '' AS FILTER, STRING_AGG(INFO, ';') AS INFO FROM 'tests/databases/annotations/hg19/*.parquet' GROUP BY \"#CHROM\", POS, REF, ALT" --output=/tmp/full_annotation.tsv 
 ```
 
 # Stats

howard/commons.py

@@ -22,7 +22,6 @@
 import zipfile
 import gzip
 import requests
-import genomepy
 
 
 file_folder = os.path.dirname(__file__)
@@ -90,6 +89,8 @@
     "arm64": "arm64"
 }
 
+LOG_FORMAT = "#[%(asctime)s] [%(levelname)s] %(message)s"
+
 def remove_if_exists(filepaths: list) -> None:
     """
     The function removes a file if it exists at the specified filepath(s).
@@ -104,7 +105,7 @@ def remove_if_exists(filepaths: list) -> None:
             os.remove(filepath)
 
 
-def set_log_level(verbosity: str) -> str:
+def set_log_level(verbosity: str, log_file:str = None) -> str:
     """
     It sets the log level of the Python logging module
 
@@ -120,8 +121,11 @@ def set_log_level(verbosity: str) -> str:
     }
     if verbosity not in configs.keys():
         raise ValueError("Unknown verbosity level:" + verbosity)
+
     log.basicConfig(
-        format="#[%(asctime)s] [%(levelname)s] %(message)s",
+        filename=log_file,
+        encoding='utf-8', 
+        format=LOG_FORMAT,
         datefmt="%Y-%m-%d %H:%M:%S",
         level=configs[verbosity],
     )

howard/main.py

@@ -42,7 +42,7 @@ def main() -> None:
         #usage="howard [<shared-args>]",
         epilog="Usage examples:\n"
             """   howard process --input=tests/data/example.vcf.gz --output=/tmp/example.annotated.vcf.gz --param=config/param.json \n"""
-            """   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.vcf.gz --annotations='tests/data/annotations/dbnsfp42a.parquet,tests/data/annotations/gnomad211_genome.parquet' \n"""
+            """   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.vcf.gz --annotations='tests/databases/annotations/hg19/dbnsfp42a.parquet,tests/databases/annotations/hg19/gnomad211_genome.parquet' \n"""
             """   howard calculation --input=tests/data/example.full.vcf --output=/tmp/example.calculation.tsv --calculations='vartype' \n"""
             """   howard prioritization --input=tests/data/example.vcf.gz --output=/tmp/example.prioritized.vcf.gz --prioritizations=config/prioritization_profiles.json --profiles='default,GERMLINE' \n"""
             """   howard query --input=tests/data/example.vcf.gz --explode_infos --query='SELECT "#CHROM", POS, REF, ALT, "INFO/DP", "INFO/CLNSIG", sample2, sample3 FROM variants WHERE "INFO/DP" >= 50 OR "INFO/CLNSIG" NOT NULL ORDER BY "INFO/DP" DESC' \n"""
@@ -105,7 +105,7 @@ def main() -> None:
         args.verbosity = "info"
 
     # Logging
-    set_log_level(args.verbosity)
+    set_log_level(args.verbosity, args.log)
 
     # Threads
     if "threads" in args and args.threads:

howard/tools/databases.py

@@ -378,6 +378,8 @@ def databases_download_genomes(assemblies: list, genome_folder: str = None, prov
 
     log.info(f"Download Genomes {assemblies}")
 
+    import genomepy
+
     if not genome_folder:
         genome_folder = DEFAULT_GENOME_FOLDER
 

howard/tools/tools.py

@@ -32,7 +32,6 @@
 from howard.tools.from_annovar import *
 
 
-
 # Arguments dict
 arguments = {
 
@@ -428,6 +427,14 @@
             "required": False,
             "default": "info"
         },
+        "log": {
+            "metavar": "FILE",
+            "help": """Logs file\n"""
+                    """Format: LOG\n"""
+                    """Example: 'my.log'\n"""
+                    """Default: None""",
+            "default": None
+        },
         "quiet": {
             "help": argparse.SUPPRESS,
             "action": "store_true"
@@ -445,7 +452,7 @@
 
 
 # Shared arguments
-shared_arguments = ["config", "threads", "memory", "verbosity", "quiet", "verbose", "debug"]
+shared_arguments = ["config", "threads", "memory", "verbosity", "log", "quiet", "verbose", "debug"]
 
 # Command dict
 commands_arguments = {
@@ -482,8 +489,8 @@
         "description":  """Annotation is mainly based on a build-in Parquet annotation method, and tools such as BCFTOOLS, Annovar and snpEff. It uses available databases (see Annovar and snpEff) and homemade databases. Format of databases are: parquet, duckdb, vcf, bed, Annovar and snpEff (Annovar and snpEff databases are automatically downloaded, see howard databases tool). """,
         "help":         """Annotation of genetic variations using databases/files and tools.""",
         "epilog":       """Usage examples:\n"""
-                        """   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.vcf.gz --annotations='tests/data/annotations/avsnp150.parquet,tests/data/annotations/dbnsfp42a.parquet,tests/data/annotations/gnomad211_genome.parquet' \n"""
-                        """   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.tsv --annotations='annovar:refGene,snpeff,tests/data/annotations/clinvar_20210123.parquet' \n""", 
+                        """   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.vcf.gz --annotations='tests/databases/annotations/hg19/avsnp150.parquet,tests/databases/annotations/hg19/dbnsfp42a.parquet,tests/databases/annotations/hg19/gnomad211_genome.parquet' \n"""
+                        """   howard annotation --input=tests/data/example.vcf.gz --output=/tmp/example.howard.tsv --annotations='annovar:refGene,snpeff,tests/databases/annotations/hg19/clinvar_20210123.parquet' \n""", 
         "groups": {
             "main": {
                 "input": True,
@@ -571,8 +578,8 @@
         "epilog": """Usage examples:\n"""
                         """   howard query --input=tests/data/example.vcf.gz --query="SELECT * FROM variants WHERE REF = 'A' AND POS < 100000" \n"""
                         """   howard query --input=tests/data/example.vcf.gz --explode_infos --query='SELECT "#CHROM", POS, REF, ALT, "INFO/DP", "INFO/CLNSIG", sample2, sample3 FROM variants WHERE "INFO/DP" >= 50 OR "INFO/CLNSIG" NOT NULL ORDER BY "INFO/DP" DESC' \n"""
-                        """   howard query --query="SELECT * FROM 'tests/data/annotations/dbnsfp42a.parquet' WHERE \\"INFO/Interpro_domain\\" NOT NULL ORDER BY \\"INFO/SiPhy_29way_logOdds_rankscore\\" DESC" \n"""
-                        """   howard query --query="SELECT \\"#CHROM\\" AS \\"#CHROM\\", POS AS POS, '' AS ID, REF AS REF, ALT AS ALT, '' AS QUAL, '' AS FILTER, STRING_AGG(INFO, ';') AS INFO FROM 'tests/data/annotations/*.parquet' GROUP BY \\"#CHROM\\", POS, REF, ALT" --output=/tmp/full_annotation.tsv \n"""
+                        """   howard query --query="SELECT * FROM 'tests/databases/annotations/hg19/dbnsfp42a.parquet' WHERE \\"INFO/Interpro_domain\\" NOT NULL ORDER BY \\"INFO/SiPhy_29way_logOdds_rankscore\\" DESC" \n"""
+                        """   howard query --query="SELECT \\"#CHROM\\" AS \\"#CHROM\\", POS AS POS, '' AS ID, REF AS REF, ALT AS ALT, '' AS QUAL, '' AS FILTER, STRING_AGG(INFO, ';') AS INFO FROM 'tests/databases/annotations/hg19/*.parquet' GROUP BY \\"#CHROM\\", POS, REF, ALT" --output=/tmp/full_annotation.tsv \n"""
                         , 
         "groups": {
             "main": {

tests/test_tools_databases.py

@@ -437,6 +437,8 @@ def test_databases_download_genomes():
     different assemblies and contig filters.
     """
 
+    import genomepy
+
     # Init
     assemblies_config = {
             "sacCer3": {