Merge branch 'release/0.1.11'

.envs/.local/.django

@@ -25,3 +25,6 @@ GOOGLE_ANALYTICS_ID=
 # ------------------------------------------------------------------------------
 SENTRY_DSN=
 SENTRY_DSN_FRONTEND=
+
+# Set the location of download large lookup files (which are downloaded separately, after the build step)
+ZORP_ASSETS_DIR=/app/.lookups

.gitignore

@@ -37,6 +37,9 @@ locuszoom_plotting_service/media/
 scripts/data_loaders/raw/*
 scripts/data_loaders/sources/*
 
+### Things mounted by local-mode docker container
+.lookups/
+
 # Configuration files that should not be checked into Git
 .envs/*
 !.envs/.local/

README.md

@@ -29,6 +29,14 @@ $ docker-compose -f local.yml build
 $ docker-compose -f local.yml up
 ```
 
+On the first installation, you will also need to download some large asset files required for annotations. (see 
+deployment docs for the correct command to use with production assets)
+
+```bash
+$ docker-compose -f local.yml run --rm django zorp-assets download --type snp_to_rsid_test --tag genome_build GRCh37 --no-update
+$ docker-compose -f local.yml run --rm django zorp-assets download --type snp_to_rsid_test --tag genome_build GRCh38 --no-update
+```
+
 ### Setting Up Your Users
 
 - To create an **superuser account**, use the following command. This must be performed first, in order to set up
@@ -56,6 +64,7 @@ into your browser. Now the user's email should be verified and ready to go.
 `$ docker-compose -f local.yml run --rm django python manage.py makemigrations`
 
 Then verify the migration file is correct, and restart docker to apply the migrations automatically.
+(in production, you must apply the migrations manually; see deployment guide for details)
 
 
 ## Development and testing helpers

assets/js/pages/gwas_summary.js

@@ -41,6 +41,10 @@ function createTopHitsTable(selector, data, region_url) {
                     }
                 },
             },
+            {
+                title: 'rsID',
+                field: 'rsid',
+            },
             {
                 title: 'Nearest gene(s)',
                 field: 'nearest_genes',

docs/deploy/index.md

@@ -40,7 +40,7 @@ Make sure to build the UI code (`yarn install && yarn run prod`) before creating
 this step should be automated!!)
 
 Build the docker container in production (or download an appropriate pre-made image):
-`sudo docker-compose -f production.yml build`
+`sudo docker-compose -f production.yml build --pull`
 
 Start the container:
 `sudo docker-compose -f production.yml up -d`
@@ -51,6 +51,11 @@ This app uses internal django features to serve static assets, so those do not r
 ### Run migrations for the first time
 `$ sudo docker-compose -f production.yml run --rm django python manage.py migrate`
 
+### Load sample data required for base functionality
+```bash
+$ docker-compose -f production.yml run --rm django zorp-assets download --type snp_to_rsid --tag genome_build GRCh37 --no-update
+$ docker-compose -f production.yml run --rm django zorp-assets download --type snp_to_rsid --tag genome_build GRCh38 --no-update
+```
 ### Create an admin user
 You will need to enter some configuration into the admin panel before using the app for the first time.
 
@@ -78,7 +83,7 @@ A sample callback URL for OAuth registration (in local development) would be:
 ## Releasing a new version (checklist)
 - (future) Verify backup status on DB backups
 - `yarn install --from-lockfile && yarn run prod`
-- `docker-compose -f production.yml build`
+- `docker-compose -f production.yml build --pull`
 - `docker-compose -f production.yml up -d`
 - (optional) `docker-compose -f production.yml run --rm django python manage.py migrate`
 

locuszoom_plotting_service/api/serializers.py

@@ -40,6 +40,7 @@ class GwasFileSerializer(drf_serializers.Serializer):
     position = drf_serializers.IntegerField(source='pos', read_only=True)
     ref_allele = drf_serializers.CharField(source='ref', read_only=True)
     alt_allele = drf_serializers.CharField(source='alt', read_only=True)
+    rsid = drf_serializers.CharField(read_only=True)
     log_pvalue = drf_serializers.SerializerMethodField(method_name='get_neg_log_pvalue', read_only=True)
     variant = drf_serializers.CharField(source='marker', read_only=True)
     beta = drf_serializers.FloatField(read_only=True)

locuszoom_plotting_service/taskapp/tasks.py

@@ -74,6 +74,8 @@ def hash_contents(self, instance: models.AnalysisFileset):
 @shared_task(bind=True)
 @lz_file_prep("Normalize GWAS file format")
 def normalize_gwas(self, instance: models.AnalysisFileset):
+    metadata = instance.metadata
+
     src_path = os.path.join(settings.MEDIA_ROOT, instance.raw_gwas_file.name)
     dest_path = instance.normalized_gwas_path
     parser_options = instance.parser_options
@@ -106,7 +108,7 @@ def normalize_gwas(self, instance: models.AnalysisFileset):
 
     # For now the writer expects a temp file name, and it creates the .gz version internally
     tmp_normalized_path = dest_path.replace('.txt.gz', '.txt')
-    processors.normalize_contents(reader, tmp_normalized_path)
+    processors.normalize_contents(reader, tmp_normalized_path, metadata.build, debug_mode=settings.DEBUG)
 
 
 @shared_task(bind=True)

locuszoom_plotting_service/templates/base.html

@@ -37,7 +37,7 @@
 
   {% endif %}
 
-  <link rel="icon" href="{% static 'images/favicons/favicon.ico' %}">
+  <link rel="icon" href="{% static 'images/favicons/favicon.png' %}">
 
   <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css"
         integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh" crossorigin="anonymous">
@@ -125,7 +125,7 @@
       <a href="{% url 'about' %}">FAQ</a> -
       <a href="{% url 'gwas:upload' %}">Upload</a> -
       <a href="https://forms.gle/GWzg9nzaDU5qrXJM8" target="_blank" class="text-success">User Survey</a> -
-      <a href="mailto:locuszoom@googlegroups.com">Contact us</a>
+      <a href="https://groups.google.com/forum/#!forum/locuszoom">Contact us</a>
     </div>
   </div>
 </footer>

locuszoom_plotting_service/templates/pages/about.html

@@ -20,6 +20,11 @@ <h2 class="mt-3">What is this for?</h2>
           <a href="https://groups.google.com/forum/#!forum/locuszoom">mailing list</a> to receive announcements.
         </p>
 
+        <p>
+          This site is operated by the University of Michigan
+          <a href="https://sph.umich.edu/csg/">Center for Statistical Genetics.</a>
+        </p>
+
         <div class="accordion mb-3" id="faq-questions">
           <div class="card">
             <div class="card-header" id="heading-question-privacy">
@@ -170,8 +175,8 @@ <h2 class="mt-3">What is this for?</h2>
               <div class="card-body">
                 <p>
                   There are several ways to reach us, including our
-                  <a href="mailto:locuszoom@googlegroups.com">user group</a> and
-                  <a href="https://github.com/statgen/locuszoom-hosted">public bug tracker</a>. For questions
+                  <a href="https://groups.google.com/forum/#!forum/locuszoom">user group</a> and
+                  <a href="https://github.com/statgen/locuszoom">public bug tracker</a>. For questions
                   involving security or sensitive data, we may be reached at
                   <a href="mailto:[email protected]">[email protected]</a>.
                 </p>

package.json

@@ -31,8 +31,8 @@
     "d3": "3.5.16",
     "d3-tip": "0.9.0",
     "gwas-credible-sets": "^0.1.0",
-    "localzoom": "https://github.com/statgen/localzoom#v0.5.3",
-    "locuszoom": "https://github.com/statgen/locuszoom#42f7c64127893f5f1b00ddf9b81acf369179dc36",
+    "localzoom": "https://github.com/statgen/localzoom#d795a9e",
+    "locuszoom": "0.11.0",
     "pako": "^1.0.8",
     "underscore": "1.8.3",
     "vue": "^2.6.10",

requirements/base.txt

@@ -35,5 +35,5 @@ pyyaml  # Required for OpenAPI rendering
 boltons~=19.1
 scipy~=1.2
 python-magic==0.4.15
-git+git://github.com/abought/zorp.git@893d383
+zorp[perf,lookups]==0.3.3
 genelocator==1.1.1

tests/ingest/test_processors.py

@@ -27,6 +27,8 @@ def test_normalizes(self, tmpdir):
         status = processors.normalize_contents(
             reader,
             os.path.join(tmpdir, 'normalized.txt'),
+            'GRCh38',
+            debug_mode=True
         )
 
         assert status is True, 'Normalization completed successfully'

util/ingest/processors.py

@@ -9,6 +9,7 @@
 from genelocator import get_genelocator
 import genelocator.exception as gene_exc
 from zorp import (
+    lookups,
     readers,
     sniffers
 )
@@ -38,12 +39,17 @@ def get_file_sha256(src_path, block_size=2 ** 20) -> bytes:
 
 
 @helpers.capture_errors
-def normalize_contents(reader: readers.BaseReader, dest_path: str) -> bool:
+def normalize_contents(reader: readers.BaseReader, dest_path: str, build: str, debug_mode=False) -> bool:
     """
     Initial content ingestion: load the file and write variants in a standardized format
 
     This routine will deliberately exclude lines that could not be handled in a reliable fashion, such as pval=NA
+
+    In "debug mode", the ingest process will use a smaller (test environment optimized) version of the
+        rsid_finder lookup.
     """
+    rsid_finder = lookups.SnpToRsid(build, test=debug_mode)
+    reader.add_lookup('rsid', lambda variant: rsid_finder(variant.chrom, variant.pos, variant.ref, variant.alt))
     reader.write(dest_path, make_tabix=True)
     # In reality a failing task will usually raise an exception rather than returning False
     return True