Merge branch 'release/0.1.3'

assets/js/pages/gwas_summary.js

@@ -42,6 +42,15 @@ function createTopHitsTable(selector, data, region_url) {
                     }
                 },
             },
+            {
+                title: 'Nearest gene(s)',
+                field: 'nearest_genes',
+                formatter: cell => {
+                    const genes = cell.getValue() || [];  // There will be studies that predate this feature, and won't have a value
+                    // Convert the list of ensg/symbol objects to a string- eventually we can add links to ext DB
+                    return genes.map(gene => gene.symbol).join(', ');
+                }
+            },
             {title: '-log<sub>10</sub>(p)', field: 'neg_log_pvalue', formatter: cell => (+cell.getValue()).toFixed(3)},
         ],
         initialSort: [

assets/js/util/pheweb_plots.js

@@ -264,6 +264,36 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t
             return base;
         }
 
+        // Add gene name labels to the plot: the 7 most significant peaks, up to 2 gene labels per data point
+        // Note: this is slightly different from PheWeb's code (b/c nearest_genes is represented differently)
+        const variants_to_label = unbinned_variants.slice()
+            .filter(v => (!!v.peak && v.neg_log_pvalue > 7.301))  // Only label peaks above line of gwas significance
+            .sort((a, b) => b.neg_log_pvalue - a.neg_log_pvalue)  // most significant first
+            .slice(0, 7);
+        gwas_plot.append('g')
+            .attr('class', 'genenames')
+            .selectAll('text.genenames')
+            .data(variants_to_label)
+            .enter()
+            .append('text')
+            .attr('class', 'genename_text')
+            .style('font-style', 'italic')
+            .attr('text-anchor', 'middle')
+            .attr('transform', function(d) {
+                return fmt('translate({0},{1})',
+                           x_scale(get_genomic_position(d)),
+                           y_scale(d.neg_log_pvalue) - 5);
+            })
+            .text(function(d) {
+                // Old datasets: no symbol. New dataset: hash with symbol & ensg keys
+                const genes = (d.nearest_genes || []).map(gene => gene.symbol);
+                if (genes.length <= 2) {
+                    return genes.join(',');
+                } else {
+                    return genes.slice(0, 2).join(',') + ',...';
+                }
+            });
+
         function pp1() {
             gwas_plot.append('g').attr('class', 'variant_hover_rings')
                 .selectAll('a.variant_hover_ring')

assets/vue/gwas_region.vue

@@ -51,7 +51,7 @@
             class="float-right"
             :build="build"
             :max_range="500000"
-            search_url="https://portaldev.sph.umich.edu/api_internal_dev/v1/annotation/omnisearch/"/>
+            search_url="https://portaldev.sph.umich.edu/api/v1/annotation/omnisearch/"/>
       </div>
     </div>
     <div class="row" v-if="message">

locuszoom_plotting_service/taskapp/tasks.py

@@ -135,9 +135,10 @@ def summarize_gwas(self, instance: models.AnalysisFileset):
 @lz_file_prep("Prepare a manhattan plot")
 def manhattan_plot(self, instance: models.AnalysisFileset):
     # This is a separate task because it can be quite slow
+    metadata = instance.metadata
     normalized_path = instance.normalized_gwas_path
     manhattan_path = instance.manhattan_path
-    processors.generate_manhattan(normalized_path, manhattan_path)
+    processors.generate_manhattan(metadata.build, normalized_path, manhattan_path)
 
 
 @shared_task(bind=True)

locuszoom_plotting_service/templates/gwas/gwas_summary.html

@@ -149,9 +149,8 @@ <h3>QQ Plot:</h3>
 
     // Required as hardcoded global reference in PheWeb pheno.js
     window.model = {
-      // TODO: make sure more variants have ref/alt information
       urlprefix: '{% filter add_token:token %}{% url 'gwas:region' gwas.slug %}{% endfilter %}',
-      tooltip_underscoretemplate: '<b><%= d.chrom %>:<%= d.pos.toLocaleString() %></b><br>'
+      tooltip_underscoretemplate: '<b><%- d.chrom %>:<%- d.pos.toLocaleString() %> <%- d.ref %> / <%- d.alt %></b><br>-log<sub>10</sub>(p): <%- d.neg_log_pvalue && d.neg_log_pvalue.toFixed(3) %><br>Nearest gene(s): <%- d.nearest_genes && d.nearest_genes.length && d.nearest_genes.map(function(gene) { return gene.symbol; }).join(", ") %>'
     };
     window.pheno = '';
   </script>

locuszoom_plotting_service/templates/gwas/share.html

@@ -23,15 +23,23 @@
           </p>
         {% else %}
           <h2>Shareable links</h2>
-          <table class="table">
+          <table class="table" id="view-links-list">
             <tr>
               <th>Label</th>
               <th>URL</th>
             </tr>
             {% for link in viewlinks %}
               <tr>
                 <td>{% firstof link.label link.created %}</td>
-                <td><input type="text" value="{{ request.scheme }}://{{ request.META.HTTP_HOST }}{{ link.get_absolute_url }}"></td>
+                <td>
+                  <div class="input-group" id="copy-controls-{{ forloop.counter }}">
+                    <input type="text" id="copy-value-{{ forloop.counter }}"  class="form-control"
+                           value="{{ request.scheme }}://{{ request.get_host }}{{ link.get_absolute_url }}">
+                    <div class="input-group-append">
+                      <button id="copy-button-{{ forloop.counter }}" class="btn btn-outline-info ">Copy</button>
+                    </div>
+                  </div>
+                </td>
               </tr>
             {% empty %}
               <tr>
@@ -57,3 +65,17 @@ <h2>Create a shareable link</h2>
     </div>
   </div>
 {% endblock %}
+
+{% block javascript %}
+<script type="application/javascript">
+   document.querySelectorAll('div[id^=copy-controls]').forEach(function (row) {
+       // Connect each "copy value" button to the corresponding text field
+       var textbox = row.querySelector('[id^=copy-value]');
+       var button = row.querySelector('[id^=copy-button]');
+       button.addEventListener('click', function() {
+           textbox.select();
+           document.execCommand('copy');
+       });
+   })
+</script>
+{% endblock %}

package.json

@@ -31,8 +31,8 @@
     "d3": "3.5.16",
     "d3-tip": "0.9.0",
     "gwas-credible-sets": "^0.1.0",
-    "locuszoom": "https://github.com/statgen/locuszoom#8b06580",
-    "locuszoom-tabix": "https://github.com/statgen/localzoom#26c32d1",
+    "locuszoom": "0.10.0-beta.2",
+    "locuszoom-tabix": "https://github.com/statgen/localzoom#dbcc3e5",
     "pako": "^1.0.8",
     "underscore": "1.8.3",
     "vue": "^2.6.10",

requirements/base.txt

@@ -36,3 +36,4 @@ boltons~=19.1
 scipy~=1.2
 python-magic==0.4.15
 git+git://github.com/abought/zorp.git@ef0398a
+genelocator==1.1.1

tests/ingest/test_processors.py

@@ -35,7 +35,7 @@ def test_normalizes(self, tmpdir):
 
     def test_makes_manhattan(self, tmpdir):
         expected = tmpdir / 'manhattan.json'
-        status = processors.generate_manhattan(SAMPLE_NORM, expected)
+        status = processors.generate_manhattan('GRCh38', SAMPLE_NORM, expected)
         assert status is True
         assert expected.exists(), 'Manhattan data created'
 

tests/ingest/test_validators.py

@@ -7,7 +7,7 @@
 )
 
 from zorp import (
-    parsers, readers, sniffers
+    parsers, sniffers
 )
 
 
@@ -24,23 +24,22 @@ def test_valid_input(self):
 
     @pytest.mark.skip(reason="Unclear whether this is actually a requirement for PheWeb loaders; revisit")
     def test_wrong_chrom_order(self):
-        reader = readers.IterableReader([
-            "#chrom\tpos\tref\talt\tpvalue",
+        reader = sniffers.guess_gwas_generic([
+            "#chrom\tpos\tref\talt\tneg_log_pvalue",
             "2\t1\tA\tC\t7.3",
             "1\t1\tA\tC\t7.3"
-        ], parser=parsers.standard_gwas_parser_quick, skip_rows=1)
+        ], skip_rows=1)
 
         with pytest.raises(val_exc.ValidationException):
             validators.standard_gwas_validator._validate_contents(reader)
 
-    @pytest.mark.skip
-    def test_chroms_not_sorted(self):
-        reader = readers.IterableReader([
-            "#chrom\tpos\tref\talt\tpvalue",
+    def test_chroms_not_contiguous(self):
+        reader = sniffers.guess_gwas_generic([
+            "#chrom\tpos\tref\talt\tneg_log_pvalue",
             "1\t1\tA\tC\t7.3",
             "X\t1\tA\tC\t7.3",
             "1\t2\tA\tC\t7.3",
-        ], parser=parsers.standard_gwas_parser_quick, skip_rows=1)
+        ], skip_rows=1)
 
         with pytest.raises(val_exc.ValidationException):
             validators.standard_gwas_validator._validate_contents(reader)

util/ingest/processors.py

@@ -6,10 +6,13 @@
 import logging
 import math
 
+from genelocator import get_genelocator
+import genelocator.exception as gene_exc
 from zorp import (
     readers,
     sniffers
 )
+
 from . exceptions import TopHitException
 from . import (
     helpers,
@@ -47,7 +50,7 @@ def normalize_contents(reader: readers.BaseReader, dest_path: str) -> bool:
 
 
 @helpers.capture_errors
-def generate_manhattan(in_filename: str, out_filename: str) -> bool:
+def generate_manhattan(build: str, in_filename: str, out_filename: str) -> bool:
     """Generate manhattan plot data for the processed file"""
     # Strong assumption: there are no invalid lines when a file reaches this stage; this operates on normalized data
     reader = sniffers.guess_gwas_standard(in_filename)\
@@ -59,7 +62,23 @@ def generate_manhattan(in_filename: str, out_filename: str) -> bool:
 
     manhattan_data = binner.get_result()
 
+    gl = get_genelocator('GRCh38', coding_only=False)
     for v_dict in manhattan_data['unbinned_variants']:
+        # Annotate nearest gene(s) for all "top hits", and also clean up values so JS can handle them
+        # It's possible to have more than one nearest gene for a given position (if variant is inside, not just near)
+        try:
+            nearest_genes = [
+                {
+                    'symbol': res['symbol'],
+                    'ensg': res['ensg']
+                }
+                for res in gl.at(v_dict["chrom"], v_dict["pos"])
+            ]
+        except (gene_exc.BadCoordinateException, gene_exc.NoResultsFoundException):
+            nearest_genes = []
+
+        v_dict['nearest_genes'] = nearest_genes
+
         if math.isinf(v_dict['neg_log_pvalue']):
             # JSON has no concept of infinity; use a string that browsers can type-coerce into the correct number
             v_dict['neg_log_pvalue'] = 'Infinity'
@@ -72,15 +91,13 @@ def generate_manhattan(in_filename: str, out_filename: str) -> bool:
 @helpers.capture_errors
 def generate_qq(in_filename: str, out_filename) -> bool:
     """Largely borrowed from PheWeb code (load.qq.make_json_file)"""
-    # TODO: Currently the ingest pipeline never stores "af"/"maf" at all, which could affect this calculation
     # TODO: This step appears to load ALL data into memory (list on generator). This could be a memory hog; not sure if
     #   there is a way around it as it seems to rely on sorting values
     reader = sniffers.guess_gwas_standard(in_filename)\
         .add_filter("neg_log_pvalue")
 
     # TODO: Pheweb QQ code benefits from being passed { num_samples: n }, from metadata stored outside the
     #   gwas file. This is used when AF/MAF are present (which at the moment ingest pipeline does not support)
-
     variants = list(qq.augment_variants(reader))
 
     rv = {}

util/ingest/validators.py

@@ -42,9 +42,10 @@ def _validate_mimetype(self, mimetype: str) -> bool:
 
     @helpers.capture_errors
     def _validate_data_rows(self, reader) -> bool:
-        """Data must be sorted, all values must be readable, and all chroms must be known"""
+        """Data must be sorted, all values must be readable, and all chroms must be contiguous"""
         # Horked from PheWeb's `load.read_input_file.PhenoReader` class
         cp_groups = itertools.groupby(reader, key=lambda v: (v.chrom, v.pos))
+        chrom_seen = set()
 
         prev_chrom = None
         prev_pos = -1
@@ -54,6 +55,12 @@ def _validate_data_rows(self, reader) -> bool:
                 # Positions not in correct order for Pheweb to use
                 raise v_exc.ValidationException('Positions must be sorted prior to uploading')
 
+            if cur_chrom != prev_chrom:
+                if cur_chrom in chrom_seen:
+                    raise v_exc.ValidationException('Chromosomes must be sorted (so that all variants for the same chromosome are contiguous)')  # noqa
+                else:
+                    chrom_seen.add(cur_chrom)
+
             prev_chrom = cur_chrom
             prev_pos = cp[1]
 

yarn.lock

@@ -3253,24 +3253,25 @@ locate-path@^3.0.0:
     p-locate "^3.0.0"
     path-exists "^3.0.0"
 
-"locuszoom-tabix@https://github.com/statgen/localzoom#26c32d1":
-  version "0.3.0"
-  resolved "https://github.com/statgen/localzoom#26c32d11428e82d1cb95eb92bb5bf40330473e8d"
+"locuszoom-tabix@https://github.com/statgen/localzoom#dbcc3e5":
+  version "0.4.0"
+  resolved "https://github.com/statgen/localzoom#dbcc3e59a2061903d831ca4f8d6874f271f2590f"
   dependencies:
     "@sentry/browser" "^4.5.2"
     bootstrap "^4.1.3"
     bootstrap-vue "^2.0.0-rc.11"
     gwas-credible-sets "^0.1.0"
-    locuszoom "https://github.com/statgen/locuszoom#8b06580"
+    locuszoom "0.10.0-beta.2"
     lodash "^4.17.11"
     tabix-reader "https://github.com/abought/tabix-reader"
     tabulator-tables "^4.1.4"
     vue "^2.6.10"
     vue-bootstrap-typeahead "^0.2.6"
 
-"locuszoom@https://github.com/statgen/locuszoom#8b06580":
-  version "0.10.0-beta.1"
-  resolved "https://github.com/statgen/locuszoom#8b065804188279ff33cb30619aa555566c834d36"
+[email protected]:
+  version "0.10.0-beta.2"
+  resolved "https://registry.yarnpkg.com/locuszoom/-/locuszoom-0.10.0-beta.2.tgz#7c007c421ee3c1e39f9adab6612874761bb455c6"
+  integrity sha512-SkyquD7OCPJQcCrFoJxHBlA9/U9CNJb+EE1n+O/jnX5CqiFlPUHHrzi8KzBUPZ25DQHdtCHXM7YE+0imHTrYDg==
   dependencies:
     d3 "3.5.17"
     promise-polyfill "^8.1.3"