Skip to content

Commit

Permalink
Merge pull request #340 from oicr-gsi/release-1.4.0
Browse files Browse the repository at this point in the history
Release 1.4.0
  • Loading branch information
iainrb authored Jan 31, 2024
2 parents caebf29 + b7f3881 commit 4fd14d0
Show file tree
Hide file tree
Showing 71 changed files with 911 additions and 570 deletions.
20 changes: 20 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,25 @@
# CHANGELOG

## v1.4.0: 2024-01-31

- GCGI-611: fix fusion frameshift hardcode
- GCGI-443: merge duplicate fusion rows
- fixed bug removing oncogenic fusions
- GCGI-1217: Remove NORMAL_DEPTH variable
- GCGI-1194: Research Report PDF Footer says RUO
- GCGI-1261: Update plugin tester base class, so it is unaffected by changes in core HTML
- removed "Quality Failure" text in failed report plugin html
- GCGI-1282: fixed config generation for pWGS in setup mode, and removed automatic generation of failed report sentence
- GCGI-1278: Fix issues with fusion plugin in benchmark tests; update `test_env.sh`; add convenience test scripts

### Improvements for mini-Djerba

- GCGI-1265: Automatically fill in report dates (preserving date of original report draft)
- GCGI-1268: Use report ID for name of JSON output file
- GCGI-1269: Default to user-friendly minimal error text
- GCGI-1270: `--version` option in `djerba.py` and mini-Djerba
- GCGI-1271: Mini-Djerba modes changed to setup/render/update

## v1.3.1: 2024-01-19

- Fixed support for 40X assay
Expand Down
9 changes: 7 additions & 2 deletions src/bin/djerba.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,20 @@

sys.path.pop(0) # do not import from script directory
from djerba.core.main import main, arg_processor
from djerba.version import get_djerba_version
import djerba.util.constants as constants

def get_parser():
"""Construct the parser for command-line arguments"""
parser = argparse.ArgumentParser(
description='Djerba: A tool for making bioinformatics clinical reports',
epilog='Run any subcommand with -h/--help for additional information'
epilog='For details, run any subcommand with -h/--help, or visit https://djerba.readthedocs.io'
)
parser.add_argument('-d', '--debug', action='store_true', help='More verbose logging')
parser.add_argument('-v', '--verbose', action='store_true', help='Verbose logging')
parser.add_argument('-q', '--quiet', action='store_true', help='Logging for error messages only')
parser.add_argument('-l', '--log-path', help='Output file for log messages; defaults to STDERR')
parser.add_argument('--version', action='store_true', help='Print the version number and exit')
subparsers = parser.add_subparsers(title='subcommands', help='sub-command help', dest='subparser_name')
setup_parser = subparsers.add_parser(constants.SETUP, help='setup for a Djerba report')
setup_parser.add_argument('-a', '--assay', metavar='NAME', required=True, choices=['WGTS', 'WGS', 'TAR', 'PWGS'], help='Name of assay')
Expand All @@ -30,7 +32,7 @@ def get_parser():
config_parser.add_argument('-w', '--work-dir', metavar='PATH', required=True, help='Path to workspace directory')
extract_parser = subparsers.add_parser(constants.EXTRACT, help='extract metrics from configuration')
extract_parser.add_argument('-i', '--ini', metavar='PATH', required=True, help='INI config file with fully specified inputs')
extract_parser.add_argument('-j', '--json', metavar='PATH', help='Path for JSON output; defaults to djerba_report.json in the plugin workspace')
extract_parser.add_argument('-j', '--json', metavar='PATH', help='Path for JSON output; defaults to ${REPORT_ID}_report.json in the plugin workspace')
extract_parser.add_argument('-w', '--work-dir', metavar='PATH', required=True, help='Path to workspace directory')
extract_parser.add_argument('--no-archive', action='store_true', help='Do not archive the JSON report file')
render_parser = subparsers.add_parser(constants.RENDER, help='read JSON and write HTML, with optional PDF')
Expand Down Expand Up @@ -63,5 +65,8 @@ def get_parser():
parser.print_help(sys.stderr)
sys.exit(1)
args = parser.parse_args()
if args.version:
print("Djerba core version {0}".format(get_djerba_version()))
sys.exit(0)
ap = arg_processor(args)
main(ap.get_work_dir(), ap.get_log_level(), ap.get_log_path()).run(args)
62 changes: 48 additions & 14 deletions src/bin/mini_djerba.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,49 +8,83 @@
sys.path.pop(0) # do not import from script directory

import argparse
import logging
from tempfile import TemporaryDirectory
import djerba.util.mini.constants as constants
from djerba.util.mini.main import main, arg_processor, MiniDjerbaScriptError
from djerba.util.mini.mdc import MDCFormatError
from djerba.version import get_djerba_version

def get_parser():
"""Construct the parser for command-line arguments"""
parser = argparse.ArgumentParser(
description='Mini-Djerba: A tool for updating bioinformatics clinical reports',
epilog='Run any subcommand with -h/--help for additional information'
epilog='For details, run any subcommand with -h/--help, or visit https://djerba.readthedocs.io/en/latest/mini_djerba.html'
)
parser.add_argument('-d', '--debug', action='store_true', help='More verbose logging')
parser.add_argument('-v', '--verbose', action='store_true', help='Verbose logging')
parser.add_argument('-q', '--quiet', action='store_true', help='Logging for error messages only')
parser.add_argument('-l', '--log-path', help='Output file for log messages; defaults to STDERR')
parser.add_argument('--version', action='store_true', help='Print the version number and exit')
subparsers = parser.add_subparsers(title='subcommands', help='sub-command help', dest='subparser_name')
ready_parser = subparsers.add_parser(constants.READY, help='Ready an MDC (mini-Djerba config) file')
ready_parser.add_argument('-j', '--json', metavar='PATH', help='Existing report JSON. Optional, if not given will generate a blank config file.')
ready_parser.add_argument('-o', '--out', metavar='PATH', default='config.mdc', help='Output path. Optional, defaults to config.mdc in the current directory.')
update_parser = subparsers.add_parser(constants.UPDATE, help='Update an existing JSON report file; render HTML and optional PDF')
setup_parser = subparsers.add_parser(constants.SETUP, help='Set up an MDC (mini-Djerba config) file')
setup_parser.add_argument('-j', '--json', metavar='PATH', help='Existing report JSON. Optional, if not given will generate a blank config file.')
setup_parser.add_argument('-o', '--out', metavar='PATH', default='config.mdc', help='Output path. Optional, defaults to config.mdc in the current directory.')
render_parser = subparsers.add_parser(constants.RENDER, help='Render a JSON report file to HTML and optional PDF, with no changes')
render_parser.add_argument('-j', '--json', metavar='PATH', required=True, help='Path to the Djerba report JSON file to be updated')
render_parser.add_argument('-o', '--out-dir', metavar='DIR', default='.', help='Directory for output files. Optional, defaults to the current directory.')
render_parser.add_argument('-w', '--work-dir', metavar='PATH', help='Path to workspace directory; optional, defaults to a temporary directory')
render_parser.add_argument('--no-pdf', action='store_true', help='Do not generate PDF output from HTML')
update_parser = subparsers.add_parser(constants.UPDATE, help='Render a JSON file to HTML and optional PDF, with updates from an MDC file')
update_parser.add_argument('-c', '--config', metavar='PATH', required=True, help='Path to an MDC (mini-Djerba config) file')
update_parser.add_argument('-f', '--force', action='store_true', help='Force update of mismatched plugin versions')
update_parser.add_argument('-j', '--json', metavar='PATH', required=True, help='Path to the Djerba report JSON file to be updated')
update_parser.add_argument('-o', '--out-dir', metavar='DIR', default='.', help='Directory for output files. Optional, defaults to the current directory.')
update_parser.add_argument('-p', '--pdf', action='store_true', help='Generate PDF output from HTML')
update_parser.add_argument('-u', '--write-json', action='store_true', help='Write updated JSON to the output directory')
update_parser.add_argument('-w', '--work-dir', metavar='PATH', help='Path to workspace directory; optional, defaults to a temporary directory')
update_parser.add_argument('--no-pdf', action='store_true', help='Do not generate PDF output from HTML')
return parser

if __name__ == '__main__':
parser = get_parser()
if len(sys.argv) == 1:
parser.print_help(sys.stderr)
sys.exit(1)
# suppress the error stacktrace unless verbose logging is in effect
error_message = None
args = parser.parse_args()
ap = arg_processor(args)
if args.version:
print("Djerba core version {0}".format(get_djerba_version()))
sys.exit(0)
if not (args.verbose or args.debug or args.quiet):
args.silent = True
else:
args.silent = False
try:
with TemporaryDirectory(prefix='mini_djerba_') as tmp_dir:
main(tmp_dir, ap.get_log_level(), ap.get_log_path()).run(args)
if hasattr(args, 'no_pdf'):
args.pdf = not args.no_pdf
ap = arg_processor(args)
if hasattr(args, 'work_dir') and args.work_dir != None:
main(args.work_dir, ap.get_log_level(), ap.get_log_path()).run(args)
else:
with TemporaryDirectory(prefix='mini_djerba_') as tmp_dir:
main(tmp_dir, ap.get_log_level(), ap.get_log_path()).run(args)
except MDCFormatError as err:
msg = "Configuration error: Please check the -c/--config file and try again."
print(msg, file=sys.stderr)
raise
error_message = "Error reading the -c/--config file: {0}".format(err)
if not args.silent:
raise
except OSError as err:
error_message = "Filesystem error: {0}".format(err)
if not args.silent:
raise
except Exception as err:
msg = "Unexpected Mini-Djerba error! Contact the developers."
raise MiniDjerbaScriptError(msg) from err
error_message = "Unexpected Mini-Djerba error! Run with --verbose or --debug for details.\n"+\
"If errors persist:\n"+\
"- Email [email protected]\n"+\
"- Please DO NOT include personal health information (PHI)"
if not args.silent:
raise MiniDjerbaScriptError(error_message) from err
if error_message:
# only called if logging is "silent"
print(error_message, file=sys.stderr)
sys.exit(1)
4 changes: 3 additions & 1 deletion src/lib/djerba/core/configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,9 @@ def _raise_missing_config_error(self, key, input_keys, complete=True):

def _raise_null_param_error(self, key):
msg = "INI section {0}, option {1} is null; ".format(self.identifier, key)+\
"null values are not permitted in fully-specified Djerba config"
"null values are not permitted in fully-specified Djerba config. "+\
"Note that parameter defaults will NOT overwrite a null value "+\
"explicitly assigned in the config input."
self.logger.error(msg)
raise DjerbaConfigError(msg)

Expand Down
4 changes: 4 additions & 0 deletions src/lib/djerba/core/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@
MERGE_LIST = 'merge_list'
MERGED_FILENAME = 'merged_filename'
PAGE_FOOTER = 'page_footer'
PDF_FOOTERS = 'pdf_footers'

# keywords for plugin structure
RESULTS = 'results'
Expand All @@ -103,3 +104,6 @@

# keyword for OncoKB level
ONCOKB = 'OncoKB'

# JSON file suffix
REPORT_JSON_SUFFIX = '_report.json'
2 changes: 1 addition & 1 deletion src/lib/djerba/core/extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def _get_core_params(self, config):
# add the core release version
core_params[cc.CORE_VERSION] = get_djerba_version()
# add the timestamp in UTC
core_params[cc.EXTRACT_TIME] = time.strftime('%Y-%m-%d_%H:%M:%SZ', time.gmtime())
core_params[cc.EXTRACT_TIME] = time.strftime('%Y-%m-%d_%H:%M:%S %z', time.localtime())
return core_params

def _get_merger_params(self, config):
Expand Down
8 changes: 5 additions & 3 deletions src/lib/djerba/core/html/clinical_footer.html
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@

<%
from time import strftime
from time import strftime, strptime
fields = extract_time.split('_')
draft_date = strftime("%Y/%m/%d", strptime(fields[0], "%Y-%m-%d"))
%>

<br>
Expand All @@ -13,10 +15,10 @@ <h2>Report Sign-offs</h2>

<table class="suppl" width="100%">
<tr>
<td width="33%">Report drafted by ${author} on ${strftime("%Y/%m/%d")}</td>
<td width="33%">Report drafted by ${author} on ${draft_date}</td>
</tr>
<tr>
<td width="33%">Report electronically signed out by Trevor Pugh, PhD, FACMG (ABMS #1027812) on yyyy/mm/dd</td>
<td width="33%">Report electronically signed out by Trevor Pugh, PhD, FACMG (ABMS #1027812) on ${strftime("%Y/%m/%d")}</td>
</tr>
</table>

Expand Down
69 changes: 42 additions & 27 deletions src/lib/djerba/core/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import pdfkit
import os
import re
from glob import glob
from PyPDF2 import PdfMerger
import djerba.util.ini_fields as ini
from djerba.core.base import base as core_base
Expand Down Expand Up @@ -207,7 +208,7 @@ def base_render(self, data, out_dir=None, pdf=False):
self.logger.info("Wrote HTML output to {0}".format(html_path))
if pdf:
pdf_path = os.path.join(out_dir, prefix+'.pdf')
footer = output_data[cc.PAGE_FOOTER]
footer = output_data[cc.PDF_FOOTERS][prefix]
p_rend.render_file(html_path, pdf_path, footer)
self.logger.info("Wrote PDF output to {0}".format(pdf_path))
merge_list = output_data[cc.MERGE_LIST]
Expand Down Expand Up @@ -272,7 +273,8 @@ def update_data_from_file(self, new_data, json_path, force):
# ie. overwriting a given plugin is all-or-nothing
# also overwrite JSON config section for the plugin
# if plugin data did not exist in old JSON, it will be added
# TODO check plugin version numbers in old/new JSON
# check plugin version numbers in old/new JSON
# This updates plugins only; core data (including report timestamp) is not altered
for plugin in new_data[self.PLUGINS].keys():
old_version = data[self.PLUGINS][plugin][cc.VERSION]
new_version = new_data[self.PLUGINS][plugin][cc.VERSION]
Expand Down Expand Up @@ -309,17 +311,43 @@ def extract(self, config, json_path=None, archive=False):
if json_path: # do this *before* taking the time to generate output
self.path_validator.validate_output_file(json_path)
data = self.base_extract(config)
if json_path:
self.logger.debug('Writing JSON output to {0}'.format(json_path))
with open(json_path, 'w') as out_file:
out_file.write(json.dumps(data))
if not json_path:
json_path = self.get_default_json_output_path(data)
self.logger.debug('Writing JSON output to {0}'.format(json_path))
with open(json_path, 'w') as out_file:
out_file.write(json.dumps(data))
if archive:
self.upload_archive(data)
else:
self.logger.info("Omitting archive upload at extract step")
self.logger.info('Finished Djerba extract step')
return data

def get_json_input_path(self, json_arg):
if json_arg:
input_path = json_arg
else:
candidates = glob(os.path.join(self.work_dir, '*'+cc.REPORT_JSON_SUFFIX))
total = len(candidates)
if total == 0:
msg = 'Cannot find default JSON path; work_dir has no files '+\
'ending in "{0}"'+format(cc.REPORT_JSON_SUFFIX)
self.logger.error(msg)
raise RuntimeError(msg)
elif total > 1:
msg = 'Cannot find default JSON path; multiple candidates '+\
'ending in "{0}": {1}'+format((cc.REPORT_JSON_SUFFIX, candidates))
self.logger.error(msg)
raise RuntimeError(msg)
else:
input_path = candidates[0]
return input_path

def get_default_json_output_path(self, data):
filename = data[cc.CORE][cc.REPORT_ID]+cc.REPORT_JSON_SUFFIX
json_path = os.path.join(self.work_dir, filename)
return json_path

def render(self, data, out_dir=None, pdf=False, archive=False):
self.logger.info('Starting Djerba render step')
# Archive the JSON data structure, if needed
Expand Down Expand Up @@ -354,12 +382,12 @@ def run(self, args):
self.configure(ini_path, ini_path_out)
elif mode == constants.EXTRACT:
ini_path = ap.get_ini_path()
json_path = ap.get_json_path()
json_arg = ap.get_json()
archive = ap.is_archive_enabled()
config = self.read_ini_path(ini_path)
self.extract(config, json_path, archive)
self.extract(config, json_arg, archive)
elif mode == constants.RENDER:
json_path = ap.get_json_path()
json_path = self.get_json_input_path(ap.get_json())
archive = ap.is_archive_enabled()
with open(json_path) as json_file:
data = json.loads(json_file.read())
Expand All @@ -368,7 +396,7 @@ def run(self, args):
ini_path = ap.get_ini_path()
out_dir = ap.get_out_dir()
ini_path_out = os.path.join(out_dir, 'full_config.ini')
json_path = os.path.join(out_dir, 'djerba_report.json')
json_path = None # write JSON to default workspace location
archive = ap.is_archive_enabled()
config = self.configure(ini_path, ini_path_out)
# upload to archive at the extract step, not the render step
Expand All @@ -382,7 +410,7 @@ def run(self, args):
else:
config_path = ini_path
summary_only = False
jp = ap.get_json_path()
jp = self.get_json_input_path(ap.get_json())
out_dir = ap.get_out_dir()
archive = ap.is_archive_enabled()
pdf = ap.is_pdf_enabled()
Expand Down Expand Up @@ -454,6 +482,9 @@ def setup(self, assay, ini_path, compact):
component_list = [
'core',
'report_title',
'patient_info',
'pwgs_provenance_helper',
'pwgs_cardea_helper',
'pwgs.case_overview',
'pwgs.summary',
'pwgs.sample',
Expand Down Expand Up @@ -533,22 +564,6 @@ def get_ini_path(self):
def get_ini_out_path(self):
return self._get_arg('ini_out')

def get_json_path(self):
json_arg = self._get_arg('json')
if json_arg:
json_path = json_arg
else:
work_dir = self.get_work_dir()
if work_dir == None:
msg = "Cannot find default JSON path, work_dir undefined"
self.logger.error(msg)
raise RuntimeError(msg)
json_path = os.path.join(work_dir, self.DEFAULT_JSON_FILENAME)
return json_path

def get_summary_path(self):
return self._get_arg('summary')

def get_summary_path(self):
return self._get_arg('summary')

Expand Down
Loading

0 comments on commit 4fd14d0

Please sign in to comment.