diff --git a/bims/migrations/0450_sitesetting_auto_validate_taxa_on_upload.py b/bims/migrations/0450_sitesetting_auto_validate_taxa_on_upload.py new file mode 100644 index 000000000..7c246d0f4 --- /dev/null +++ b/bims/migrations/0450_sitesetting_auto_validate_taxa_on_upload.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.16 on 2025-01-20 12:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bims', '0449_alter_biologicalcollectionrecord_uuid'), + ] + + operations = [ + migrations.AddField( + model_name='sitesetting', + name='auto_validate_taxa_on_upload', + field=models.BooleanField(default=True, help_text='If True, taxa from CSV uploads are automatically validated.'), + ), + ] diff --git a/bims/models/site_setting.py b/bims/models/site_setting.py index 8df785f92..056d314da 100644 --- a/bims/models/site_setting.py +++ b/bims/models/site_setting.py @@ -31,6 +31,11 @@ class SiteSetting(Preferences): '"filter_values": ["bims"]}]' ) + auto_validate_taxa_on_upload = models.BooleanField( + default=True, + help_text='If True, taxa from CSV uploads are automatically validated.' + ) + default_location_site_cluster = models.CharField( max_length=100, help_text='SQL view name of the location site cluster which ' diff --git a/bims/scripts/taxa_upload.py b/bims/scripts/taxa_upload.py index ca9276aa5..6d513e777 100644 --- a/bims/scripts/taxa_upload.py +++ b/bims/scripts/taxa_upload.py @@ -17,7 +17,8 @@ IUCNStatus, IUCN_CATEGORIES, VernacularName, - ORIGIN_CATEGORIES, TaxonTag, SourceReference, SourceReferenceBibliography, + ORIGIN_CATEGORIES, TaxonTag, SourceReference, + SourceReferenceBibliography, Invasion ) from bims.templatetags import is_fada_site @@ -25,11 +26,12 @@ fetch_all_species_from_gbif, fetch_gbif_vernacular_names ) from bims.scripts.data_upload import DataCSVUpload -from bims.utils.gbif import get_vernacular_names from td_biblio.exceptions import DOILoaderError from td_biblio.models import Entry from td_biblio.utils.loaders import DOILoader +from preferences import preferences + logger = logging.getLogger('bims') @@ -37,6 +39,26 @@ class TaxaProcessor(object): all_keys = {} + def add_taxon_to_taxon_group(self, taxonomy: Taxonomy, taxon_group: TaxonGroup, validated = True): + """ + Add or update the relationship between a taxonomy and a taxon group, + ensuring the 'is_validated' field is properly set in both the + intermediate table (TaxonGroupTaxonomy) and on the Taxonomy object. + """ + taxon_group.taxonomies.add( + taxonomy, + through_defaults={ + 'is_validated': validated + } + ) + + def add_taxon_to_taxon_group_unvalidated(self, taxonomy, taxon_group): + """ + A helper function that calls `add_taxon_to_taxon_group` with + validated=True + """ + self.add_taxon_to_taxon_group(taxonomy, taxon_group, validated=True) + def handle_error(self, row, message): pass @@ -647,12 +669,7 @@ def process_data(self, row, taxon_group: TaxonGroup): ) # -- Add to taxon group - taxon_group.taxonomies.add( - taxonomy, - through_defaults={ - 'is_validated': True - } - ) + self.add_taxon_to_taxon_group_unvalidated(taxonomy, taxon_group) # -- Endemism endemism = self.endemism(row) @@ -746,8 +763,6 @@ def process_data(self, row, taxon_group: TaxonGroup): if accepted_taxon: taxonomy.accepted_taxonomy = accepted_taxon - taxonomy.validated = True - taxonomy.save() self.finish_processing_row(row, taxonomy) except Exception as e: # noqa @@ -757,6 +772,14 @@ def process_data(self, row, taxon_group: TaxonGroup): class TaxaCSVUpload(DataCSVUpload, TaxaProcessor): model_name = 'taxonomy' + def add_taxon_to_taxon_group_unvalidated(self, taxonomy, taxon_group): + """ + A helper function that calls `add_taxon_to_taxon_group` with + validated=False + """ + auto_validate = preferences.SiteSetting.auto_validate_taxa_on_upload + self.add_taxon_to_taxon_group(taxonomy, taxon_group, validated=auto_validate) + def finish_processing_row(self, row, taxonomy): # -- Add to taxon group taxon_group = self.upload_session.module_group diff --git a/bims/tests/data/taxa_upload_family_2.csv b/bims/tests/data/taxa_upload_family_2.csv new file mode 100644 index 000000000..8a185a448 --- /dev/null +++ b/bims/tests/data/taxa_upload_family_2.csv @@ -0,0 +1,2 @@ +Invasion,Taxon Rank,Vernacular name(s),Language,Kingdom,Phylum,Class,Order,Family,Genus,Species,SubSpecies,Taxon,Accepted Taxon,Taxonomic status,Scientific name and authority,Common NAME,Origin,Endemism,Conservation status global,Conservation status national,lentic,On GBIF,Lakes (Y/N),Author(s),ANT,AT +Category 1a invasive,Class,,,Animalia,Annelida,Oligochaeta,,,,,,Oligochaeta2,,accepted,Oligochaeta2,"Earthworm (eng), Earthworm2, ミミズ (jpn)",Native,Unknown,Not evaluated,,,No,,,, \ No newline at end of file diff --git a/bims/tests/test_source_references.py b/bims/tests/test_source_references.py index 40332d483..21d4d252f 100644 --- a/bims/tests/test_source_references.py +++ b/bims/tests/test_source_references.py @@ -4,6 +4,7 @@ from unittest.mock import patch, MagicMock from django_tenants.test.cases import FastTenantTestCase +from datetime import datetime from bims.factories import EntryFactory from bims.tasks import get_source_reference_filter, generate_source_reference_filter @@ -67,5 +68,5 @@ def test_generate_source_reference_filter(self, mock_get_current, mock_set, mock f'source_reference_filter_{self.tenant}', [ {'id': self.references[0].id, 'reference': f'- | {self.entry.publication_date.year} | Test', 'type': 'Peer-reviewed scientific article'}, - {'id': self.references[1].id, 'reference': f'- | 2024 | {self.document.source}', 'type': 'Published book, report or thesis'}] + {'id': self.references[1].id, 'reference': f'- | {datetime.now().year} | {self.document.source}', 'type': 'Published book, report or thesis'}] , timeout=None) diff --git a/bims/tests/test_taxa_upload.py b/bims/tests/test_taxa_upload.py index ef1ce3dc2..0c32b7e25 100644 --- a/bims/tests/test_taxa_upload.py +++ b/bims/tests/test_taxa_upload.py @@ -172,6 +172,35 @@ def test_taxa_upload(self, mock_fetch_all_species_from_gbif, mock_finish): ).exists() ) + @mock.patch('bims.scripts.data_upload.DataCSVUpload.finish') + @mock.patch('bims.scripts.taxa_upload.fetch_all_species_from_gbif') + @mock.patch('bims.scripts.taxa_upload.preferences') + def test_taxa_upload_unvalidated(self, mock_preferences, mock_fetch_all_species_from_gbif, mock_finish): + mock_preferences.SiteSetting.auto_validate_taxa_on_upload = False + mock_finish.return_value = None + mock_fetch_all_species_from_gbif.return_value = self.taxonomy + + with open(os.path.join( + test_data_directory, 'taxa_upload_family_2.csv' + ), 'rb') as file: + upload_session = UploadSessionF.create( + uploader=self.owner, + process_file=File(file), + module_group=self.taxon_group + ) + + taxa_csv_upload = TaxaCSVUpload() + taxa_csv_upload.upload_session = upload_session + taxa_csv_upload.start('utf-8-sig') + + self.assertTrue( + TaxonGroupTaxonomy.objects.filter( + taxonomy__canonical_name='Oligochaeta2', + taxongroup=self.taxon_group, + is_validated=False + ).exists() + ) + @mock.patch('bims.scripts.data_upload.DataCSVUpload.finish') @mock.patch('bims.scripts.taxa_upload.fetch_all_species_from_gbif') def test_taxa_upload_variety_and_forma(self, mock_fetch_all_species_from_gbif, mock_finish):