From 2b0b493d3fa3da5322d2f56b72ea3aa1354f27c1 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Tue, 17 Dec 2024 10:42:54 +0000 Subject: [PATCH 1/4] added pre-qc-failed status filter --- analyses/models.py | 3 ++- workflows/flows/analysis_amplicon_study.py | 10 +++++++--- workflows/flows/assemble_study.py | 1 + workflows/tests/test_analysis_amplicon_study_flow.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/analyses/models.py b/analyses/models.py index 9eb3052..54defe5 100644 --- a/analyses/models.py +++ b/analyses/models.py @@ -505,7 +505,7 @@ class AnalysisStates(str, Enum): ANALYSIS_COMPLETED = "analysis_completed" ANALYSIS_BLOCKED = "analysis_blocked" ANALYSIS_FAILED = "analysis_failed" - ANALYSIS_QC_FAILED = "analysis_qc_failed" + ANALYSIS_PRE_QC_FAILED = "analysis_pre_qc_failed" ANALYSIS_POST_SANITY_CHECK_FAILED = "analysis_post_sanity_check_failed" @classmethod @@ -515,6 +515,7 @@ def default_status(cls): cls.ANALYSIS_COMPLETED: False, cls.ANALYSIS_BLOCKED: False, cls.ANALYSIS_FAILED: False, + cls.ANALYSIS_PRE_QC_FAILED: False, } status = models.JSONField( diff --git a/workflows/flows/analysis_amplicon_study.py b/workflows/flows/analysis_amplicon_study.py index d77f5f6..acd9962 100644 --- a/workflows/flows/analysis_amplicon_study.py +++ b/workflows/flows/analysis_amplicon_study.py @@ -61,8 +61,12 @@ def get_analyses_to_attempt( """ study.refresh_from_db() analyses_worth_trying = ( - study.analyses.exclude_by_statuses( - [AnalysisStates.ANALYSIS_COMPLETED, AnalysisStates.ANALYSIS_BLOCKED] + study.analyses.filter( + **{ + f"status__{analyses.models.Analysis.AnalysisStates.ANALYSIS_COMPLETED}": False, + f"status__{analyses.models.Analysis.AnalysisStates.ANALYSIS_BLOCKED}": False, + f"status__{analyses.models.Analysis.AnalysisStates.ANALYSIS_PRE_QC_FAILED}": False, + } ) .filter(experiment_type=for_experiment_type.value) .order_by("id") @@ -545,7 +549,7 @@ def set_post_analysis_states(amplicon_current_outdir: Path, amplicon_analyses: L if analysis.run.first_accession in qc_failed_runs: task_mark_analysis_status( analysis, - status=AnalysisStates.ANALYSIS_FAILED, + status=AnalysisStates.ANALYSIS_PRE_QC_FAILED, reason=qc_failed_runs[analysis.run.first_accession], ) elif analysis.run.first_accession in qc_completed_runs: diff --git a/workflows/flows/assemble_study.py b/workflows/flows/assemble_study.py index 8722b2b..bf960dc 100644 --- a/workflows/flows/assemble_study.py +++ b/workflows/flows/assemble_study.py @@ -126,6 +126,7 @@ def get_assemblies_to_attempt(study: analyses.models.Study) -> List[Union[str, i **{ f"status__{analyses.models.Assembly.AssemblyStates.ASSEMBLY_COMPLETED}": False, f"status__{analyses.models.Assembly.AssemblyStates.ASSEMBLY_BLOCKED}": False, + f"status__{analyses.models.Assembly.AssemblyStates.PRE_ASSEMBLY_QC_FAILED}": False, } ).values_list("id", flat=True) return assemblies_worth_trying diff --git a/workflows/tests/test_analysis_amplicon_study_flow.py b/workflows/tests/test_analysis_amplicon_study_flow.py index a0452a1..df3170c 100644 --- a/workflows/tests/test_analysis_amplicon_study_flow.py +++ b/workflows/tests/test_analysis_amplicon_study_flow.py @@ -505,7 +505,7 @@ async def test_prefect_analyse_amplicon_flow( # check failed runs assert ( await analyses.models.Analysis.objects.filter( - status__analysis_failed=True + status__analysis_pre_qc_failed=True ).acount() == 1 ) From 562ba4fda4db5268ccd2e8e3748c602bc34017f9 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Tue, 17 Dec 2024 14:44:03 +0000 Subject: [PATCH 2/4] added default for PRE_QC status --- analyses/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/analyses/models.py b/analyses/models.py index 54defe5..13965a1 100644 --- a/analyses/models.py +++ b/analyses/models.py @@ -273,6 +273,7 @@ def default_status(cls): cls.ASSEMBLY_UPLOADED: False, cls.ASSEMBLY_UPLOAD_FAILED: False, cls.ASSEMBLY_UPLOAD_BLOCKED: False, + cls.PRE_ASSEMBLY_QC_FAILED: False, } status = models.JSONField( From 4a53f7a7f22fbad639bc0db96eec5194101c546e Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Wed, 15 Jan 2025 16:11:43 +0000 Subject: [PATCH 3/4] fixes --- analyses/models.py | 6 +++--- workflows/flows/analysis_amplicon_study.py | 14 +++++++------- workflows/flows/assemble_study.py | 12 ++++++------ .../tests/test_analysis_amplicon_study_flow.py | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/analyses/models.py b/analyses/models.py index 13965a1..8200920 100644 --- a/analyses/models.py +++ b/analyses/models.py @@ -267,13 +267,13 @@ class AssemblyStates(str, Enum): def default_status(cls): return { cls.ASSEMBLY_STARTED: False, + cls.PRE_ASSEMBLY_QC_FAILED: False, cls.ASSEMBLY_FAILED: False, cls.ASSEMBLY_COMPLETED: False, cls.ASSEMBLY_BLOCKED: False, cls.ASSEMBLY_UPLOADED: False, cls.ASSEMBLY_UPLOAD_FAILED: False, cls.ASSEMBLY_UPLOAD_BLOCKED: False, - cls.PRE_ASSEMBLY_QC_FAILED: False, } status = models.JSONField( @@ -506,17 +506,17 @@ class AnalysisStates(str, Enum): ANALYSIS_COMPLETED = "analysis_completed" ANALYSIS_BLOCKED = "analysis_blocked" ANALYSIS_FAILED = "analysis_failed" - ANALYSIS_PRE_QC_FAILED = "analysis_pre_qc_failed" + ANALYSIS_QC_FAILED = "analysis_qc_failed" ANALYSIS_POST_SANITY_CHECK_FAILED = "analysis_post_sanity_check_failed" @classmethod def default_status(cls): return { cls.ANALYSIS_STARTED: False, + cls.ANALYSIS_QC_FAILED: False, cls.ANALYSIS_COMPLETED: False, cls.ANALYSIS_BLOCKED: False, cls.ANALYSIS_FAILED: False, - cls.ANALYSIS_PRE_QC_FAILED: False, } status = models.JSONField( diff --git a/workflows/flows/analysis_amplicon_study.py b/workflows/flows/analysis_amplicon_study.py index acd9962..7e36a6f 100644 --- a/workflows/flows/analysis_amplicon_study.py +++ b/workflows/flows/analysis_amplicon_study.py @@ -61,12 +61,12 @@ def get_analyses_to_attempt( """ study.refresh_from_db() analyses_worth_trying = ( - study.analyses.filter( - **{ - f"status__{analyses.models.Analysis.AnalysisStates.ANALYSIS_COMPLETED}": False, - f"status__{analyses.models.Analysis.AnalysisStates.ANALYSIS_BLOCKED}": False, - f"status__{analyses.models.Analysis.AnalysisStates.ANALYSIS_PRE_QC_FAILED}": False, - } + study.analyses.exclude_by_statuses( + [ + analyses.models.Analysis.AnalysisStates.ANALYSIS_QC_FAILED, + analyses.models.Analysis.AnalysisStates.ANALYSIS_COMPLETED, + analyses.models.Analysis.AnalysisStates.ANALYSIS_BLOCKED, + ] ) .filter(experiment_type=for_experiment_type.value) .order_by("id") @@ -549,7 +549,7 @@ def set_post_analysis_states(amplicon_current_outdir: Path, amplicon_analyses: L if analysis.run.first_accession in qc_failed_runs: task_mark_analysis_status( analysis, - status=AnalysisStates.ANALYSIS_PRE_QC_FAILED, + status=AnalysisStates.ANALYSIS_QC_FAILED, reason=qc_failed_runs[analysis.run.first_accession], ) elif analysis.run.first_accession in qc_completed_runs: diff --git a/workflows/flows/assemble_study.py b/workflows/flows/assemble_study.py index bf960dc..92fa782 100644 --- a/workflows/flows/assemble_study.py +++ b/workflows/flows/assemble_study.py @@ -122,12 +122,12 @@ def get_assemblies_to_attempt(study: analyses.models.Study) -> List[Union[str, i :return: """ study.refresh_from_db() - assemblies_worth_trying = study.assemblies_reads.filter( - **{ - f"status__{analyses.models.Assembly.AssemblyStates.ASSEMBLY_COMPLETED}": False, - f"status__{analyses.models.Assembly.AssemblyStates.ASSEMBLY_BLOCKED}": False, - f"status__{analyses.models.Assembly.AssemblyStates.PRE_ASSEMBLY_QC_FAILED}": False, - } + assemblies_worth_trying = study.assemblies_reads.exclude_by_statuses( + [ + analyses.models.Assembly.AssemblyStates.PRE_ASSEMBLY_QC_FAILED, + analyses.models.Assembly.AssemblyStates.ASSEMBLY_COMPLETED, + analyses.models.Assembly.AssemblyStates.ASSEMBLY_BLOCKED, + ] ).values_list("id", flat=True) return assemblies_worth_trying diff --git a/workflows/tests/test_analysis_amplicon_study_flow.py b/workflows/tests/test_analysis_amplicon_study_flow.py index df3170c..99a31fa 100644 --- a/workflows/tests/test_analysis_amplicon_study_flow.py +++ b/workflows/tests/test_analysis_amplicon_study_flow.py @@ -505,7 +505,7 @@ async def test_prefect_analyse_amplicon_flow( # check failed runs assert ( await analyses.models.Analysis.objects.filter( - status__analysis_pre_qc_failed=True + status__analysis_qc_failed=True ).acount() == 1 ) From 34869e7fb5a7e17e4b5fcd7beae8522b0606a06f Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Wed, 15 Jan 2025 16:41:23 +0000 Subject: [PATCH 4/4] fixes --- analyses/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyses/models.py b/analyses/models.py index 8200920..1ca7539 100644 --- a/analyses/models.py +++ b/analyses/models.py @@ -207,7 +207,7 @@ def __str__(self): return f"{self.name} {self.version}" if self.version is not None else self.name -class AssemblyManager(ENADerivedManager): +class AssemblyManager(SelectByStatusManagerMixin, ENADerivedManager): def get_queryset(self): return super().get_queryset().select_related("run")