From 340e7a1fbf7e3ef06b6e216aa148e5b5cf97d31b Mon Sep 17 00:00:00 2001 From: Aledj2 Date: Mon, 9 Dec 2019 11:09:46 +0000 Subject: [PATCH 1/5] Add wscleaner command for automating cronjob --- wscleaner/wscleaner_command.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 wscleaner/wscleaner_command.sh diff --git a/wscleaner/wscleaner_command.sh b/wscleaner/wscleaner_command.sh new file mode 100755 index 0000000..b4cb8c2 --- /dev/null +++ b/wscleaner/wscleaner_command.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Activate wscleaner environment +eval "$(conda shell.bash hook)" # Add conda environment to system path +conda activate wscleaner + +# Set varianbles +logfile="/home/mokaguys/Documents/automate_demultiplexing_logfiles/wscleaner_logs/wscleaner_$(date -d now +%y%m%d).log" +runfolders="/media/data3/share" + +# Execute +wscleaner $runfolders --logfile $logfile From 2e5f0c13bddceb5ea6531037ac98650fc3eb1f56 Mon Sep 17 00:00:00 2001 From: Aledj2 Date: Mon, 9 Dec 2019 11:18:18 +0000 Subject: [PATCH 2/5] Fix wscleaner logfile name --- wscleaner/wscleaner_command.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wscleaner/wscleaner_command.sh b/wscleaner/wscleaner_command.sh index b4cb8c2..410f8b7 100755 --- a/wscleaner/wscleaner_command.sh +++ b/wscleaner/wscleaner_command.sh @@ -5,7 +5,7 @@ eval "$(conda shell.bash hook)" # Add conda environment to system path conda activate wscleaner # Set varianbles -logfile="/home/mokaguys/Documents/automate_demultiplexing_logfiles/wscleaner_logs/wscleaner_$(date -d now +%y%m%d).log" +logfile="/home/mokaguys/Documents/automate_demultiplexing_logfiles/wscleaner_logs/$(date -d now +%y%m%d)_wscleaner.log" runfolders="/media/data3/share" # Execute From 38978a1d89fc284e288ac58ed78fc0090bf3b1df Mon Sep 17 00:00:00 2001 From: Aledj2 Date: Fri, 3 Apr 2020 14:52:19 +0100 Subject: [PATCH 3/5] Update wscleaner command for crontab to use proper environment --- wscleaner/wscleaner_command.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wscleaner/wscleaner_command.sh b/wscleaner/wscleaner_command.sh index 410f8b7..c91ed02 100755 --- a/wscleaner/wscleaner_command.sh +++ b/wscleaner/wscleaner_command.sh @@ -1,10 +1,10 @@ #!/bin/bash # Activate wscleaner environment -eval "$(conda shell.bash hook)" # Add conda environment to system path +eval "$(/usr/local/bin/miniconda3/bin/conda shell.bash hook)" # Add conda environment to system path conda activate wscleaner -# Set varianbles +# Set variables logfile="/home/mokaguys/Documents/automate_demultiplexing_logfiles/wscleaner_logs/$(date -d now +%y%m%d)_wscleaner.log" runfolders="/media/data3/share" From a2ddd3ad646abf42d67658db8923cace310784aa Mon Sep 17 00:00:00 2001 From: Aledj2 Date: Fri, 3 Apr 2020 15:25:33 +0100 Subject: [PATCH 4/5] Reduce logfile count to 5. Add command line option to change logs. Fix #23 --- wscleaner/wscleaner/lib.py | 4 ++-- wscleaner/wscleaner/main.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/wscleaner/wscleaner/lib.py b/wscleaner/wscleaner/lib.py index 7fc1040..1b542a3 100644 --- a/wscleaner/wscleaner/lib.py +++ b/wscleaner/wscleaner/lib.py @@ -201,11 +201,11 @@ def check_fastqs(self, runfolder): self.logger.debug(f'{runfolder.name} FASTQ BOOL: {fastq_bool}') return fastq_bool - def check_logfiles(self, runfolder): + def check_logfiles(self, runfolder, logfile_count): """Returns true if a runfolder's DNAnexus project contains 6 logfiles in the expected location""" dx_logfiles = runfolder.dx_project.count_logfiles() - logfile_bool = dx_logfiles >= 6 + logfile_bool = (dx_logfiles == logfile_count) self.logger.debug(f'{runfolder.name} LOGFILE BOOL: {logfile_bool}') return logfile_bool diff --git a/wscleaner/wscleaner/main.py b/wscleaner/wscleaner/main.py index cf8a51e..640facb 100644 --- a/wscleaner/wscleaner/main.py +++ b/wscleaner/wscleaner/main.py @@ -35,6 +35,7 @@ def cli_parser(): parser.add_argument('root', help='A directory containing runfolders to process') parser.add_argument('--logfile', help='A path for the application logfile', default='mokaguys_logger.log') parser.add_argument('--min-age', help='The age (days) a runfolder must be to be deleted', type=int, default=14) + parser.add_argument('--logfile-count', help='The number of logfiles a runfolder must have in /Logfiles', type=int, default=5) # Get version from setup.py as version CLI response version_number = pkg_resources.require("wscleaner")[0].version parser.add_argument('--version', help='Print version', action='version', version=f"wscleaner v{version_number}") @@ -66,7 +67,7 @@ def main(): # runfolder.dx_project is evaluated first as following criteria checks depend on it if runfolder.dx_project: fastqs_uploaded = RFM.check_fastqs(runfolder) - logfiles_uploaded = RFM.check_logfiles(runfolder) + logfiles_uploaded = RFM.check_logfiles(runfolder, args.logfile_count) if fastqs_uploaded and logfiles_uploaded: RFM.delete(runfolder) elif not fastqs_uploaded: @@ -83,4 +84,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() From 16948a9aeadd4031d9b3c569ed1792711ea4bf54 Mon Sep 17 00:00:00 2001 From: Aledj2 Date: Fri, 3 Apr 2020 15:40:23 +0100 Subject: [PATCH 5/5] Authenticate using local file. Add argument for auth file. Remove psutil from requirements. Bump version to 1.1 --- wscleaner/setup.py | 4 +- wscleaner/wscleaner/auth.py | 83 ------------------------------------- wscleaner/wscleaner/main.py | 16 +++---- 3 files changed, 8 insertions(+), 95 deletions(-) delete mode 100644 wscleaner/wscleaner/auth.py diff --git a/wscleaner/setup.py b/wscleaner/setup.py index 186120a..9ee5b21 100644 --- a/wscleaner/setup.py +++ b/wscleaner/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='wscleaner', - version='1.0', + version='1.1', description='Package to remove uploaded runfolders from \ the Viapath Genome Informatics NGS workstation', url='https://github.com/NMNS93/wscleaner', @@ -13,7 +13,7 @@ python_requires = '>=3.6.8', install_requires = ['docutils>=0.3', 'dxpy==0.279.0', 'pytest==4.4.0', 'pytest-cov==2.6.1', - 'Sphinx==2.0.1', 'psutil==5.6.1'], + 'Sphinx==2.0.1'], package_data = {}, diff --git a/wscleaner/wscleaner/auth.py b/wscleaner/wscleaner/auth.py deleted file mode 100644 index f823574..0000000 --- a/wscleaner/wscleaner/auth.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -"""auth.py - -Utlily classes for the workstation cleaner module. - -Methods: - get_config(): Read the DNANexus API token from the application cache file - dx_set_auth(): Set the DNAnexus authentication token used in each instance of the application - -Classes: - SetKeyAction: Set the DNAnexus authentication token used in future instances of the application and exit - PrintKeyAction: Print the cached DNAnexus authentication key -""" - -from pathlib import Path -from pkg_resources import resource_filename -import json -import argparse -import dxpy - -import logging -logger = logging.getLogger(__name__) - -CONFIG_FILE = 'config.json' - -def get_config(config=CONFIG_FILE): - """Read the DNANexus API token from the application cache file - - Returns: - filename (object): A python file object - Raises: - AttirbuteError: Config file not found. - """ - # Return the file object containing the cached DNAnexus token if it exists - filename = resource_filename('wscleaner',config) - logger.debug(f'Config: {Path(filename).name}') - if Path(filename).is_file(): - return filename - else: - raise AttributeError('Config file not found. Set auth key with --set-key.') - -def dx_set_auth(auth_token=None): - """Set the DNAnexus authentication token used in future instances of the application and exit - - Args: - auth_token (str): A DNAnexus authentication key""" - if auth_token: - security_context = {'auth_token_type': 'Bearer', 'auth_token': auth_token} - else: - filename = get_config() - with open(filename, 'r') as f: - # Password is written to the cache as a dictionary. Loaded here using json module - pwd = json.load(f) - security_context = {'auth_token_type': 'Bearer', 'auth_token': pwd['auth_token']} - dxpy.set_security_context(security_context) - -class SetKeyAction(argparse.Action): - """Set the DNAnexus authentication key based on command line arguments and exit the program. - - Inherits from argparse.Action, which initiates __call__() when the linked argument is present. - """ - # Override argparse.Action.__call__() with desired behaviour - def __call__(self, parser, namespace, values, option_string=None, config=CONFIG_FILE): - filename = resource_filename('wscleaner',config) - with open(filename, 'w') as f: - # 'values' contains authentication token given on the command line. Store for future - # wscleaner calls to set as the DNAnexus dxpy security context. - json.dump({'auth_token': values}, f) - parser.exit() - - -class PrintKeyAction(argparse.Action): - """Print the cached DNAnexus authentication key - - Inherits from argparse.Action, which initiates __call__() when the linked argument is present. - """ - # Override argparse.Action.__call__() with desired behaviour - def __call__(self, parser, namespace, values, option_string=None): - filename = get_config() - with open(filename, 'r') as f: - pwd = json.load(f) - print(pwd) - parser.exit() diff --git a/wscleaner/wscleaner/main.py b/wscleaner/wscleaner/main.py index 640facb..e5b5fd2 100644 --- a/wscleaner/wscleaner/main.py +++ b/wscleaner/wscleaner/main.py @@ -11,8 +11,8 @@ import argparse import logging import pkg_resources +import dxpy from wscleaner import mokaguys_logger -from wscleaner.auth import SetKeyAction, PrintKeyAction, dx_set_auth from wscleaner.lib import RunFolder, RunFolderManager @@ -24,13 +24,7 @@ def cli_parser(): Otherwise, --set-key and --print-key exit after actions are performed. """ parser = argparse.ArgumentParser() - # argparse API for adding custom routines. SetKeyAction and PrintKeyAction are classes with - # routines that exit the software after an action is performed. - parser.register('action', 'setkey', SetKeyAction) - parser.register('action', 'printkey', PrintKeyAction) - # Define CLI arguments - parser.add_argument('--set-key', action='setkey', help='Cache a DNA Nexus API key') - parser.add_argument('--print-key', nargs=0, action='printkey', help='Print the cached DNA Nexus API key') + parser.add_argument('--auth', help='A text file containing the DNANexus authentication token', type=str, default='/usr/local/src/mokaguys/.dnanexus_auth_token') parser.add_argument('--dry-run', help='Perform a dry run without deleting files', action='store_true', default=False) parser.add_argument('root', help='A directory containing runfolders to process') parser.add_argument('--logfile', help='A path for the application logfile', default='mokaguys_logger.log') @@ -51,8 +45,10 @@ def main(): logger = logging.getLogger(__name__) logger.info(f'START') - # Setup dxpy with cached authentication token - dx_set_auth() + # Setup dxpy authentication token read from command line file. + with open(args.auth) as f: + auth_token = f.read().rstrip() + dxpy.set_security_context({'auth_token_type': 'Bearer', 'auth_token': auth_token}) # Set root directory and search it for runfolders # If dry-run CLI flag is given, no directories are deleted by the runfolder manager.