Merge branch 'develop' into data-containers-to-use-django

jwql/jwql_monitors/monitor_filesystem.py

@@ -49,22 +49,27 @@
 import numpy as np
 from sqlalchemy.exc import DataError
 
-from jwql.database.database_interface import engine
-from jwql.database.database_interface import session
-from jwql.database.database_interface import FilesystemCharacteristics
-from jwql.database.database_interface import FilesystemGeneral
-from jwql.database.database_interface import FilesystemInstrument
-from jwql.database.database_interface import CentralStore
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
 from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS
 from jwql.utils.utils import filename_parser
 from jwql.utils.utils import get_config
 from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table
 from jwql.utils.protect_module import lock_module
 from jwql.website.apps.jwql.data_containers import get_instrument_proposals
 
+if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    # Need to set up django apps before we can access the models
+    import django  # noqa: E402 (module level import not at top of file)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    django.setup()
+
+    # Import * is okay here because this module specifically only contains database models
+    # for this monitor
+    from jwql.website.apps.jwql.monitor_models.common import *  # noqa: E402 (module level import not at top of file)
+
 SETTINGS = get_config()
 FILESYSTEM = SETTINGS['filesystem']
 PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary')
@@ -74,6 +79,7 @@
 PREVIEW_IMAGES = SETTINGS['preview_image_filesystem']
 THUMBNAILS = SETTINGS['thumbnail_filesystem']
 LOGS = SETTINGS['log_dir']
+WORKING = SETTINGS['working']
 
 
 def files_per_filter():
@@ -232,7 +238,8 @@ def get_area_stats(central_storage_dict):
              'logs': LOGS,
              'preview_images': PREVIEW_IMAGES,
              'thumbnails': THUMBNAILS,
-             'all': CENTRAL}
+             'all': CENTRAL,
+             'working':WORKING}
 
     counteddirs = []
 
@@ -368,7 +375,7 @@ def initialize_results_dicts():
         A dictionary for the ``central_storage`` database table
     """
 
-    now = datetime.datetime.now()
+    now = datetime.datetime.now(datetime.timezone.utc)
 
     general_results_dict = {}
     general_results_dict['date'] = now
@@ -430,9 +437,9 @@ def update_central_store_database(central_storage_dict):
         new_record['size'] = central_storage_dict[area]['size']
         new_record['used'] = central_storage_dict[area]['used']
         new_record['available'] = central_storage_dict[area]['available']
-        with engine.begin() as connection:
-            connection.execute(CentralStore.__table__.insert(), new_record)
-    session.close()
+
+        entry = CentralStorage(**new_record)
+        entry.save()
 
 
 def update_characteristics_database(char_info):
@@ -447,7 +454,7 @@ def update_characteristics_database(char_info):
         using that filter/pupil.
     """
     logging.info('\tUpdating the characteristics database')
-    now = datetime.datetime.now()
+    now = datetime.datetime.now(datetime.timezone.utc)
 
     # Add data to filesystem_instrument table
     for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:
@@ -458,11 +465,9 @@ def update_characteristics_database(char_info):
         new_record['instrument'] = instrument
         new_record['filter_pupil'] = optics
         new_record['obs_per_filter_pupil'] = values
-        with engine.begin() as connection:
-            connection.execute(
-                FilesystemCharacteristics.__table__.insert(), new_record)
 
-    session.close()
+        entry = FilesystemCharacteristics(**new_record)
+        entry.save()
 
 
 def update_database(general_results_dict, instrument_results_dict, central_storage_dict):
@@ -478,8 +483,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
     """
     logging.info('\tUpdating the database')
 
-    with engine.begin() as connection:
-        connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict)
+    fs_general_entry = FilesystemGeneral(**general_results_dict)
+    fs_general_entry.save()
 
     # Add data to filesystem_instrument table
     for instrument in JWST_INSTRUMENT_NAMES:
@@ -493,13 +498,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
 
             # Protect against updated enum options that have not been propagated to
             # the table definition
-            try:
-                with engine.begin() as connection:
-                    connection.execute(FilesystemInstrument.__table__.insert(), new_record)
-            except DataError as e:
-                logging.error(e)
-
-    session.close()
+            fs_instrument_entry = FilesystemInstrument(**new_record)
+            fs_instrument_entry.save()
 
 
 @lock_module

jwql/shared_tasks/shared_tasks.py

@@ -184,7 +184,7 @@ def _caller(*args, **kwargs):
 
 
 def create_task_log_handler(logger, propagate):
-    log_file_name = configure_logging('shared_tasks')
+    log_file_name = configure_logging('shared_tasks', include_time=False)
     working_dir = os.path.join(get_config()['working'], 'calibrated_data')
     ensure_dir_exists(working_dir)
     celery_log_file_handler = FileHandler(log_file_name)

jwql/utils/logging_functions.py

@@ -102,7 +102,7 @@ def filter(record):
     return filter
 
 
-def configure_logging(module):
+def configure_logging(module, include_time=True):
     """
     Configure the log file with a standard logging format. The format in question is
     set up as follows:
@@ -116,11 +116,8 @@ def configure_logging(module):
     ----------
     module : str
         The name of the module being logged.
-    production_mode : bool
-        Whether or not the output should be written to the production
-        environement.
-    path : str
-        Where to write the log if user-supplied path; default to working dir.
+    include_time : bool, default True
+        Whether or not the file name should include the time as well as the date.
 
     Returns
     -------
@@ -129,7 +126,7 @@ def configure_logging(module):
     """
 
     # Determine log file location
-    log_file = make_log_file(module)
+    log_file = make_log_file(module, include_time=include_time)
 
     # Get the logging configuration dictionary
     logging_config = get_config()['logging']
@@ -171,7 +168,7 @@ def get_log_status(log_file):
         return 'FAILURE'
 
 
-def make_log_file(module):
+def make_log_file(module, include_time=True):
     """Create the log file name based on the module name.
 
     The name of the ``log_file`` is a combination of the name of the
@@ -181,12 +178,8 @@ def make_log_file(module):
     ----------
     module : str
         The name of the module being logged.
-    production_mode : bool
-        Whether or not the output should be written to the production
-        environment.
-    path : str
-        Where to write the log if user-supplied path; default to
-        working dir.
+    include_time : bool, default True
+        Whether or not the file name should include the time as well as the date.
 
     Returns
     -------
@@ -195,7 +188,10 @@ def make_log_file(module):
     """
 
     # Build filename
-    timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M')
+    if include_time:
+        timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M')
+    else:
+        timestamp = datetime.datetime.now().strftime('%Y-%m-%d')
     hostname = socket.gethostname()
     filename = '{0}_{1}_{2}.log'.format(module, hostname, timestamp)
 

jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py

@@ -0,0 +1,39 @@
+# Generated by Django 5.1.4 on 2025-01-16 21:35
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0027_alter_fgsbadpixelstats_source_files_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='filesystemcharacteristics',
+            name='filter_pupil',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='empty', help_text='filter and/or pupil name', max_length=7), blank=True, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='filesystemcharacteristics',
+            name='instrument',
+            field=models.CharField(),
+        ),
+        migrations.AlterField(
+            model_name='filesystemcharacteristics',
+            name='obs_per_filter_pupil',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='filesysteminstrument',
+            name='filetype',
+            field=models.CharField(),
+        ),
+        migrations.AlterField(
+            model_name='filesysteminstrument',
+            name='instrument',
+            field=models.CharField(default='empty', help_text='JWST instrument name', max_length=7),
+        ),
+    ]