Merge pull request #23 from EBI-Metagenomics/analysis-downloads

Analysis downloads/qc/taxa

.github/workflows/test.yml

@@ -36,7 +36,9 @@ jobs:
 
       - name: Run tests
         run: |
-          docker run -v ${{ github.workspace }}/coverage:/app/coverage --entrypoint bash \
+          docker run -v ${{ github.workspace }}/coverage:/app/coverage \
+          -v ${{ github.workspace }}/slurm-dev-environment/fs/nfs/public:/app/data \
+          --entrypoint bash \
           --add-host=host.docker.internal:host-gateway \
           -e DATABASE_URL=postgres://postgres:[email protected]:5432/emg --rm app:latest -c pytest
 

.talismanrc

@@ -11,6 +11,8 @@ fileignoreconfig:
   checksum: 18b7d98b4778cf50b81961e026ef5ce8936692b2a2db11038656aca7940b6301
 - filename: .github/workflows/release_k8s.yml
   checksum: 0e93196d244417801fd40ad6bfa01f53a6bf04257371378a1dc3e14cbc7a04d8
+- filename: slurm-dev-environment/fs/nfs/public/tests/amplicon_v6_output/SRR6180434/taxonomy-summary/SILVA-SSU/SRR6180434.html
+  checksum: be322a6ac4686d8fa31b271a862510603403c2da8dbdce60dbb7ad3877aa7b47
 allowed_patterns:
 - pagination_key
 - CommonMetadataKeys

analyses/admin/analysis.py

@@ -6,8 +6,12 @@
 from unfold.admin import ModelAdmin
 from unfold.decorators import action
 
-from analyses.admin.base import StatusListFilter, StudyFilter
-from analyses.models import Analysis, Run
+from analyses.admin.base import (
+    JSONFieldWidgetOverridesMixin,
+    StatusListFilter,
+    StudyFilter,
+)
+from analyses.models import Analysis
 
 
 class AnalysisStatusListFilter(StatusListFilter):
@@ -16,7 +20,7 @@ def get_statuses(self) -> Iterable[str]:
 
 
 @admin.register(Analysis)
-class AnalysisAdmin(ModelAdmin):
+class AnalysisAdmin(JSONFieldWidgetOverridesMixin, ModelAdmin):
     list_display = [
         "__str__",
         "assembly_or_run",
@@ -66,6 +70,7 @@ class AnalysisAdmin(ModelAdmin):
             },
         ),
         ("Files", {"classes": ["tab"], "fields": ["downloads", "results_dir"]}),
+        ("QC", {"classes": ["tab"], "fields": ["quality_control"]}),
     )
 
     class StudyFilterForAnalysis(StudyFilter):

analyses/admin/assembly.py

@@ -3,7 +3,12 @@
 from django.contrib import admin
 from unfold.admin import ModelAdmin
 
-from analyses.admin.base import ENABrowserLinkMixin, StatusListFilter, StudyFilter
+from analyses.admin.base import (
+    ENABrowserLinkMixin,
+    JSONFieldWidgetOverridesMixin,
+    StatusListFilter,
+    StudyFilter,
+)
 from analyses.models import Assembler, Assembly
 
 
@@ -13,7 +18,7 @@ def get_statuses(self) -> Iterable[str]:
 
 
 @admin.register(Assembly)
-class AssemblyAdmin(ENABrowserLinkMixin, ModelAdmin):
+class AssemblyAdmin(ENABrowserLinkMixin, JSONFieldWidgetOverridesMixin, ModelAdmin):
     class StudyFilterForAssembly(StudyFilter):
         study_accession_search_fields = [
             "ena_study__accession",

analyses/admin/base.py

@@ -1,14 +1,17 @@
-from typing import Iterable
+from typing import Iterable, Optional
 
 from django.contrib import admin
 from django.core.validators import EMPTY_VALUES
-from django.db.models import Q
+from django.db.models import JSONField, Q
+from django.forms import Field
+from django.http import HttpRequest
 from django.shortcuts import redirect
 from django_admin_inline_paginator.admin import InlinePaginated
-from unfold.admin import TabularInline
+from unfold.admin import ModelAdmin, TabularInline
 from unfold.contrib.filters.admin import TextFilter
 from unfold.decorators import action
 
+from analyses.admin.widgets import ENAAccessionsListWidget, JSONTreeWidget
 from analyses.base_models.base_models import ENADerivedModel
 
 
@@ -70,3 +73,14 @@ class ENABrowserLinkMixin:
     def view_on_ena_browser(self, request, object_id):
         instance: type[ENADerivedModel] = self.model.objects.get(pk=object_id)
         return redirect(instance.ena_browser_url)
+
+
+class JSONFieldWidgetOverridesMixin(ModelAdmin):
+    def formfield_for_dbfield(
+        self, db_field: Field, request: HttpRequest, **kwargs
+    ) -> Optional[Field]:
+        if isinstance(db_field, JSONField) and db_field.name == "ena_accessions":
+            kwargs["widget"] = ENAAccessionsListWidget
+        elif isinstance(db_field, JSONField):
+            kwargs["widget"] = JSONTreeWidget
+        return super().formfield_for_dbfield(db_field, request, **kwargs)

analyses/admin/run.py

@@ -2,12 +2,16 @@
 from unfold.admin import ModelAdmin
 from unfold.decorators import display
 
-from analyses.admin.base import ENABrowserLinkMixin, StudyFilter
+from analyses.admin.base import (
+    ENABrowserLinkMixin,
+    JSONFieldWidgetOverridesMixin,
+    StudyFilter,
+)
 from analyses.models import Run
 
 
 @admin.register(Run)
-class RunAdmin(ENABrowserLinkMixin, ModelAdmin):
+class RunAdmin(ENABrowserLinkMixin, JSONFieldWidgetOverridesMixin, ModelAdmin):
     class StudyFilterForRun(StudyFilter):
         study_accession_search_fields = [
             "ena_study__accession",

analyses/admin/sample.py

@@ -1,10 +1,10 @@
 from django.contrib import admin
 from unfold.admin import ModelAdmin
 
-from analyses.admin.base import ENABrowserLinkMixin
+from analyses.admin.base import ENABrowserLinkMixin, JSONFieldWidgetOverridesMixin
 from analyses.models import Sample
 
 
 @admin.register(Sample)
-class SampleAdmin(ENABrowserLinkMixin, ModelAdmin):
+class SampleAdmin(ENABrowserLinkMixin, JSONFieldWidgetOverridesMixin, ModelAdmin):
     pass

analyses/admin/study.py

@@ -13,6 +13,7 @@
 from analyses.admin.analysis import AnalysisStatusListFilter
 from analyses.admin.base import (
     ENABrowserLinkMixin,
+    JSONFieldWidgetOverridesMixin,
     StudyFilter,
     TabularInlinePaginatedWithTabSupport,
 )
@@ -111,7 +112,7 @@ class StudyReadsInline(TabularInlinePaginatedWithTabSupport):
 
 
 @admin.register(Study)
-class StudyAdmin(ENABrowserLinkMixin, ModelAdmin):
+class StudyAdmin(ENABrowserLinkMixin, JSONFieldWidgetOverridesMixin, ModelAdmin):
     inlines = [StudyRunsInline, StudyAssembliesInline, StudyReadsInline]
     list_display = ["accession", "updated_at", "title", "display_accessions"]
     list_filter = ["updated_at", "created_at"]
@@ -146,7 +147,7 @@ class StudyAdmin(ENABrowserLinkMixin, ModelAdmin):
     )
 
     @display(description="ENA Accessions", label=True)
-    def display_accessions(self, instance: Run):
+    def display_accessions(self, instance: Study):
         return instance.ena_accessions
 
     @action(

analyses/admin/widgets.py

@@ -0,0 +1,17 @@
+from django_json_widget.widgets import JSONEditorWidget
+
+
+class ENAAccessionsListWidget(JSONEditorWidget):
+    def __init__(self, *args, **kwargs):
+        super().__init__(
+            options={"mode": "tree", "mainMenuBar": False, "modes": ["tree"]},
+            height="100px",
+            width="30em",
+            *args,
+            **kwargs
+        )
+
+
+class JSONTreeWidget(JSONEditorWidget):
+    def __init__(self, *args, **kwargs):
+        super().__init__(options={"mode": "tree"}, *args, **kwargs)

analyses/base_models/with_downloads_models.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 from enum import Enum
+from pathlib import Path
 from typing import List, Optional, Union
 
 from django.db import models
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 
 class DownloadType(str, Enum):
@@ -26,6 +27,7 @@ class DownloadFileType(str, Enum):
     JSON = "json"
     SVG = "svg"
     TREE = "tree"  # e.g. newick
+    HTML = "html"
     OTHER = "other"
 
 
@@ -45,6 +47,12 @@ class DownloadFile(BaseModel):
         None  # e.g. the accession of an Analysis this download is for
     )
 
+    @field_validator("path", mode="before")
+    def coerce_path(cls, value):
+        if isinstance(value, Path):
+            return str(value)
+        return value
+
 
 class WithDownloadsModel(models.Model):
     """

analyses/fixtures/analysis/conftest.py

@@ -1,11 +1,9 @@
+from pathlib import Path
+
 import django
 import pytest
 
-from analyses.base_models.with_downloads_models import (
-    DownloadFile,
-    DownloadFileType,
-    DownloadType,
-)
+from workflows.data_io_utils.mgnify_v6_utils.amplicon import import_qc, import_taxonomy
 
 django.setup()
 
@@ -37,15 +35,20 @@ def raw_read_analyses(raw_read_run):
     mgyas[0].save()
     mgyas[1].save()
 
-    mgyas[0].add_download(
-        DownloadFile(
-            file_type=DownloadFileType.TSV,
-            download_type=DownloadType.FUNCTIONAL_ANALYSIS,
-            long_description="Some PFAMs that were found",
-            short_description="PFAM table",
-            alias=f"PFAMS_{mgyas[0].accession}.tsv",
-            path="functional/pfam/annos.tsv",
-        )
+    mgyas[0].results_dir = "/app/data/tests/amplicon_v6_output/SRR6180434"
+    mgyas[0].save()
+
+    import_qc(
+        analysis=mgyas[0],
+        dir_for_analysis=Path(mgyas[0].results_dir),
+        allow_non_exist=False,
+    )
+
+    import_taxonomy(
+        analysis=mgyas[0],
+        dir_for_analysis=Path(mgyas[0].results_dir),
+        source=mg_models.Analysis.TaxonomySources.SSU,
+        allow_non_exist=False,
     )
 
     return mgyas

analyses/migrations/0026_analysis_quality_control.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.2 on 2024-12-16 16:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyses", "0025_analysis_experiment_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="analysis",
+            name="quality_control",
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]

analyses/models.py

@@ -463,12 +463,16 @@ class TaxonomySources(Enum):
         ITS_ONE_DB: str = "its_one_db"
         UNITE: str = "unite"
         PR2: str = "pr2"
+        DADA2_SILVA: str = "dada2_silva"
+        DADA2_PR2: str = "dada2_pr2"
 
     TAXONOMIES_SSU = f"{TAXONOMIES}__{TaxonomySources.SSU.value}"
     TAXONOMIES_LSU = f"{TAXONOMIES}__{TaxonomySources.LSU.value}"
     TAXONOMIES_ITS_ONE_DB = f"{TAXONOMIES}__{TaxonomySources.ITS_ONE_DB.value}"
     TAXONOMIES_UNITE = f"{TAXONOMIES}__{TaxonomySources.UNITE.value}"
     TAXONOMIES_PR2 = f"{TAXONOMIES}__{TaxonomySources.PR2.value}"
+    TAXONOMIES_DADA2_SILVA = f"{TAXONOMIES}__{TaxonomySources.DADA2_SILVA.value}"
+    TAXONOMIES_DADA2_PR2 = f"{TAXONOMIES}__{TaxonomySources.DADA2_PR2.value}"
 
     @staticmethod
     def default_annotations():
@@ -485,6 +489,7 @@ def default_annotations():
         }
 
     annotations = models.JSONField(default=default_annotations.__func__)
+    quality_control = models.JSONField(default=dict, blank=True)
 
     class PipelineVersions(models.TextChoices):
         v5 = "V5", "v5.0"

analyses/schemas.py

@@ -94,6 +94,16 @@ class MGnifyAnalysisDetail(MGnifyAnalysis):
         alias="raw_run",
         description="Metadata associated with the original read run this analysis is based on, whether or not those reads were assembled.",
     )
+    quality_control_summary: Optional[dict] = Field(
+        ...,
+        alias="quality_control",
+        examples=[
+            {
+                "before_filtering": {"total_bases": 1000000},
+                "after_filtering": {"total_bases": 700000},
+            }
+        ],
+    )
 
     class Meta:
         model = analyses.models.Analysis
@@ -172,6 +182,8 @@ class MGnifyFunctionalAnalysisAnnotationType(Enum):
     taxonomies_itsonedb: str = analyses.models.Analysis.TAXONOMIES_ITS_ONE_DB
     taxonomies_unite: str = analyses.models.Analysis.TAXONOMIES_UNITE
     taxonomies_pr2: str = analyses.models.Analysis.TAXONOMIES_PR2
+    taxonomies_dada2_pr2: str = analyses.models.Analysis.TAXONOMIES_DADA2_PR2
+    taxonomies_dada2_silva: str = analyses.models.Analysis.TAXONOMIES_DADA2_SILVA
     antismash_gene_clusters: str = analyses.models.Analysis.ANTISMASH_GENE_CLUSTERS
     pfams: str = analyses.models.Analysis.PFAMS
 

emgapiv2/api.py

@@ -156,6 +156,7 @@ def get_mgnify_analysis(request, accession: str):
         "experiment_type": experiment_type,
         "raw_run": raw_run,
         "pipeline_version": analysis.pipeline_version,
+        "quality_control": analysis.quality_control,
     }
 
     return response

emgapiv2/config.py

@@ -75,6 +75,7 @@ class AmpliconPipelineConfig(BaseModel):
     asv_folder: str = "asv"
     primer_identification_folder: str = "primer-identification"
     taxonomy_summary_folder: str = "taxonomy-summary"
+    qc_folder: str = "qc"
 
     amplicon_nextflow_master_job_memory_gb: int = 1
     amplicon_pipeline_time_limit_days: int = 5

emgapiv2/settings.py

@@ -72,6 +72,7 @@ def show_toolbar(request):
     "django_ltree",  ## for hierarchical models like Biome
     "debug_toolbar",
     "django_admin_inline_paginator",
+    "django_json_widget",
     "corsheaders",
     "ena",
     "analyses",

emgapiv2/test_api.py

@@ -54,3 +54,6 @@ def test_api_analysis_detail(raw_read_analyses, ninja_api_client):
     )
     assert analysis["accession"] == raw_read_analyses[0].accession
     assert analysis["study_accession"] == raw_read_analyses[0].study.accession
+    assert (
+        analysis["quality_control_summary"]["before_filtering"]["total_reads"] == 66124
+    )

requirements.txt

@@ -11,6 +11,7 @@ pydantic~=2.9.2
 pydantic-settings~=2.6.0
 nextflowpy==0.8.1
 django-unfold==0.41.0
+django-json-widget==2.0.1
 tzdata==2024.1
 pandas==2.2.3
 django-ltree-2==0.1.9