From fce3b6883210c1ad9193b93b302f4b7f60b3a422 Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Mon, 10 Jun 2024 22:36:57 +0100 Subject: [PATCH 1/7] interim commit - update Django to v5.0 --- .../0078_alter_project_polymorphic_ctype.py | 20 +++++++++++++++++++ webapp/api/core/urls.py | 15 -------------- webapp/requirements.txt | 10 +++++----- 3 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 webapp/api/api/migrations/0078_alter_project_polymorphic_ctype.py diff --git a/webapp/api/api/migrations/0078_alter_project_polymorphic_ctype.py b/webapp/api/api/migrations/0078_alter_project_polymorphic_ctype.py new file mode 100644 index 00000000..179467f2 --- /dev/null +++ b/webapp/api/api/migrations/0078_alter_project_polymorphic_ctype.py @@ -0,0 +1,20 @@ +# Generated by Django 5.0.6 on 2024-06-07 11:18 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0077_projectgroup_create_associated_projects'), + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.AlterField( + model_name='project', + name='polymorphic_ctype', + field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polymorphic_%(app_label)s.%(class)s_set+', to='contenttypes.contenttype'), + ), + ] diff --git a/webapp/api/core/urls.py b/webapp/api/core/urls.py index 732dff7e..1f6bc754 100644 --- a/webapp/api/core/urls.py +++ b/webapp/api/core/urls.py @@ -1,18 +1,3 @@ -"""MedCATtrainer URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/2.2/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" from django.contrib import admin from django.contrib.auth import views as pw_views from django.urls import path, include, re_path diff --git a/webapp/requirements.txt b/webapp/requirements.txt index 6fb26e5f..a2f8aeb2 100644 --- a/webapp/requirements.txt +++ b/webapp/requirements.txt @@ -1,9 +1,9 @@ --extra-index-url https://download.pytorch.org/whl/cpu/ uwsgi~=2.0 -Django~=2.2 -django-filter~=2.4 +Django~=5.0 +django-filter~=24.2 django-polymorphic~=3.0 -djangorestframework~=3.10 -django-background-tasks~=1.2 -openpyxl~=3.0 +djangorestframework~=3.15 +django-background-tasks-updated~=1.2 +openpyxl~=3.1 medcat~=1.10 From 0cbf9b5352b9cd1538654f47ce8a29c757b5a9f6 Mon Sep 17 00:00:00 2001 From: tomolopolis Date: Wed, 19 Jun 2024 00:33:49 +0100 Subject: [PATCH 2/7] CU-8694gtycm: Update django to 5.0. Tested with all trainer functions manually. --- webapp/api/core/settings.py | 2 ++ webapp/frontend/src/components/common/ProjectList.vue | 6 +++--- webapp/frontend/src/views/Home.vue | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/webapp/api/core/settings.py b/webapp/api/core/settings.py index d0f07d93..04c493e6 100644 --- a/webapp/api/core/settings.py +++ b/webapp/api/core/settings.py @@ -15,6 +15,8 @@ log = logging.getLogger(__name__) +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/webapp/frontend/src/components/common/ProjectList.vue b/webapp/frontend/src/components/common/ProjectList.vue index 4af2b566..51770804 100644 --- a/webapp/frontend/src/components/common/ProjectList.vue +++ b/webapp/frontend/src/components/common/ProjectList.vue @@ -177,7 +177,9 @@ export default { components: {LoadingOverlay, Modal}, props: { projectItems: Array, - isAdmin: Boolean + isAdmin: Boolean, + cdbSearchIndexStatus: Object, + cdbLoaded: Object, }, data () { return { @@ -213,8 +215,6 @@ export default { modelCacheLoadError: false, metricsJobId: null, saving: false, - cdbSearchIndexStatus: {}, - cdbLoaded: {}, clearModelModal: false, selectedProjects: [], loadingProjects: false diff --git a/webapp/frontend/src/views/Home.vue b/webapp/frontend/src/views/Home.vue index 2c1cdca6..149104b9 100644 --- a/webapp/frontend/src/views/Home.vue +++ b/webapp/frontend/src/views/Home.vue @@ -24,11 +24,13 @@

{{selectedProjectGroup.description}}

- +
- + From a0087d4fe10f98748d636fb349c3c4bfcbce790f Mon Sep 17 00:00:00 2001 From: tomolopolis Date: Wed, 19 Jun 2024 00:37:30 +0100 Subject: [PATCH 3/7] CU-8694gtycm: fix default value for cdbSearchIndex and cdbLoaded state --- webapp/frontend/src/views/Home.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webapp/frontend/src/views/Home.vue b/webapp/frontend/src/views/Home.vue index 149104b9..cfb98231 100644 --- a/webapp/frontend/src/views/Home.vue +++ b/webapp/frontend/src/views/Home.vue @@ -71,6 +71,8 @@ export default { loadingProjects: false, isAdmin: false, selectedProjectGroup: null, + cdbSearchIndexStatus: {}, + cdbLoaded: {}, } }, created () { From 40e986dacf580cfd49f5119d0962f70c077e8480 Mon Sep 17 00:00:00 2001 From: tomolopolis Date: Tue, 2 Jul 2024 00:04:16 +0100 Subject: [PATCH 4/7] CU-8694gtycm: merge clashing migration numbers --- .../api/api/migrations/0079_merge_20240701_2259.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 webapp/api/api/migrations/0079_merge_20240701_2259.py diff --git a/webapp/api/api/migrations/0079_merge_20240701_2259.py b/webapp/api/api/migrations/0079_merge_20240701_2259.py new file mode 100644 index 00000000..0d011c63 --- /dev/null +++ b/webapp/api/api/migrations/0079_merge_20240701_2259.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.28 on 2024-07-01 22:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0078_metacatmodel_modelpack'), + ('api', '0078_alter_project_polymorphic_ctype'), + ] + + operations = [ + ] From 562367fa33e14683a69729edae48e2bceeb12341 Mon Sep 17 00:00:00 2001 From: tomolopolis Date: Tue, 2 Jul 2024 18:46:12 +0100 Subject: [PATCH 5/7] CU-8694gtycm: latest Django CSRF protections to be used now. --- envs/env | 3 +++ webapp/api/core/settings.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/envs/env b/envs/env index 404f8884..3d58bcb5 100644 --- a/envs/env +++ b/envs/env @@ -7,6 +7,9 @@ MEDCAT_CONFIG_FILE=/home/configs/base.txt ### Deployment Realm ### ENV=non-prod +# Complete once this is deployed +CSRF_TRUSTED_ORIGINS= + ### Django debug setting - to live-reload etc. ### DEBUG=True diff --git a/webapp/api/core/settings.py b/webapp/api/core/settings.py index 04c493e6..403d6661 100644 --- a/webapp/api/core/settings.py +++ b/webapp/api/core/settings.py @@ -20,9 +20,11 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +environ_origins = os.environ.get('CSRF_TRUSTED_ORIGINS', None) +trusted_origins = [] if environ_origins is None else environ_origins.split(',') +CSRF_TRUSTED_ORIGINS = ['https://127.0.0.1:8001', 'http://localhost:8001'] + trusted_origins -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ +SECURE_CROSS_ORIGIN_OPENER_POLICY = None # SECURITY WARNING: keep the secret key used in production secret! realm = os.environ.get('ENV', 'non-prod') From cd90f24d1229ff352068628e835eacdfa17828b9 Mon Sep 17 00:00:00 2001 From: tomolopolis Date: Tue, 2 Jul 2024 23:09:04 +0100 Subject: [PATCH 6/7] CU-8694gtycm: latest Django CSRF protections to be used now. --- envs/env-prod | 3 +++ 1 file changed, 3 insertions(+) diff --git a/envs/env-prod b/envs/env-prod index 19d6460b..0728c195 100644 --- a/envs/env-prod +++ b/envs/env-prod @@ -8,6 +8,9 @@ ENV=prod # SECRET KEY - edit this for prod deployments, # SECRET_KEY= +# Complete once this is deployed +CSRF_TRUSTED_ORIGINS= + # Django Debug mode should be False for prod DEBUG=False From e9ad7cb55a70dc852ade1e4ea049086f651dfdbe Mon Sep 17 00:00:00 2001 From: tomolopolis Date: Wed, 3 Jul 2024 14:49:50 +0100 Subject: [PATCH 7/7] CU-8694gtycm: fix edge case bug for cascading delete if a concept_db or vocab of an uploaded model pack are deleted first instead of the model pack being deleted --- webapp/api/api/signals.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/webapp/api/api/signals.py b/webapp/api/api/signals.py index a2f488bd..6ae3b278 100644 --- a/webapp/api/api/signals.py +++ b/webapp/api/api/signals.py @@ -3,6 +3,7 @@ import os import shutil +from django.core.exceptions import ObjectDoesNotExist from django.db.models.fields.files import FileField from django.db.models.signals import post_save, post_delete, pre_save from django.dispatch import receiver @@ -46,13 +47,19 @@ def save_exported_projects(sender, instance, **kwargs): @receiver(post_delete, sender=ModelPack) def remove_model_pack_assets(sender, instance, **kwargs): - if instance.concept_db: - instance.concept_db.delete(using=None, keep_parents=False) - if instance.vocab: - instance.vocab.delete(using=None, keep_parents=False) - if len(instance.meta_cats.all()) > 0: - for m_c in instance.meta_cats.all(): - m_c.delete(using=None, keep_parents=False) + try: + if instance.concept_db: + instance.concept_db.delete(using=None, keep_parents=False) + except ObjectDoesNotExist: + pass # if a ConceptDB of a model pack is removed, this will cascade ModelPack removal. + try: + if instance.vocab: + instance.vocab.delete(using=None, keep_parents=False) + if len(instance.meta_cats.all()) > 0: + for m_c in instance.meta_cats.all(): + m_c.delete(using=None, keep_parents=False) + except ObjectDoesNotExist: + pass # if a vocab of a model pack is removed, this will cascade ModelPack removal. try: # rm the model pack unzipped dir & model pack zip shutil.rmtree(instance.model_pack.path.replace(".zip", ""))