Merge pull request #75 from DiamondLightSource/prep_new_v1.20.0

Prepare new version 1.20.0

HISTORY.rst

@@ -5,6 +5,44 @@ History
 Unreleased / master
 -------------------
 
+1.20.0 (2021-04-13)
+-------------------
+
+* Tables, views and indices:
+
+  * New cryoEM table `ParticleClassificationGroup`. Some columns were moved to this table from `ParticlePicker` and `ParticleClassification`.
+  * Add `capillary` as enum option to `BLSampleGroup_has_BLSample.type`
+  * XChem DB: First version of schema for production (not part of release)
+  * Analytics schema: Add access to all lookup tables, `RobotAction`, `ProcessingJob*`, `Protein` tables through views (not part of release)
+  * Drop duplicate indices, add missing indices that existed only in DLS prod
+
+* Data in lookup tables:
+
+  * Add additional container types: some historical, some new
+  * Update `ExperimentType.proposalType`: Change scm to saxs, MX to mx
+  * Update `ContainerType.proposalType`: Change scm to saxs
+  * Add two XPDF and five MX container types
+  * Update `SpaceGroup` table: Remove newlines in names. Set `MX_used` = 1 for all SGs.
+
+* Stored procedures:
+
+  * `retrieve_scm_*``: Preferentially get experiement type and container type+capacity from new lookup tables `ExperimentType` and `ContainerType`, otherwise fall-back to `experimentType` and `containerType`+`capacity` columns in the `Container` table.
+  * `upsert_sample_image`: set `BLSampleImage.modifiedTimeStamp` if the upsert is an update
+
+* Tools and documentation:
+
+  * New script `bin/missed_updates.sh` to identify update .sql files that haven't been run
+  * `bin/backup.sh`: Use `--add-drop-trigger` flag to drop trigger if exists before creation
+  * `CONTRIBUTING.md`: Simplify pull request procedure for table changes
+  * Updated simplified MX database diagram files
+  * Move list + string with lookup tables to separate file so it can be reused
+  * All bash scripts have been given new shebang lines to run on systems where bash is not at `/bin/bash`.
+
+* Users, roles and grants:
+
+  * More `INSERT` grants for `ispyb_web` role: DC, DCG, `ProcessingJob`, `ProcessingJobParameter`
+  * New `ispyb_ro_nopii` role (read-only-except-PII)
+
 1.19.0 (2021-03-05)
 -------------------
 

bin/add_schema_status_stmts.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 fname=$(basename ${1})
 

bin/backup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Usage example:
 # backup.sh ../dbbackups

bin/db_procs_to_rst.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 db=$1
 

bin/db_tables_to_rst.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 db=$1
 mysql --defaults-file=../.my.cnf -D $db --skip-column-names --batch --raw < sql/db_tables_to_tsv.sql > /tmp/tmp1.tsv

bin/export_session.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Usage:
 # export_session.sh <.cnf file> <db_name> <proposalCode><proposalNumber> <session number> <output dir>

bin/forget_all_labs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes details about laboratories on a particular proposal
 # and session.

bin/forget_all_persons.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Caution: This script attempts to remove all Personally Identifiable
 # Information (PII) in the given ispyb database for the given proposal

bin/forget_lab.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes details about a particular laboratory.
 # Usage:

bin/forget_pdb.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular PDB.
 #

bin/forget_person.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes Personally Identifiable Information (PII) for a particular person.
 #

bin/forget_protein.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular protein.
 #

bin/forget_sample.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular sample (BLSample).
 #

bin/forget_shipping.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular shipping.
 #

bin/forget_subsample.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes information for a particular sub-sample (BLSubSample).
 #

bin/forget_unused_shipments.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script removes all descriptive information related to shipments for
 # a particular proposal, except that related to samples on which data was

bin/lookup_tables.sh

@@ -4,6 +4,8 @@ SchemaStatus
 ComponentType
 ComponentSubType
 ConcentrationType
+ContainerType
+ExperimentType
 InspectionType
 PlateType
 SessionType

bin/mdb_cluster_backup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash 
+#!/usr/bin/env bash
 
 # This script assumes a MariaDB Galera Cluster with read-write split. The
 # script must be run locally on one of the database read nodes.

bin/missed_updates.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Get the database name
+if [ -z "${DB}" ]
+then
+  DB="ispyb_build"
+fi
+
+# Get this scripts dir
+dir=$(dirname $(realpath -e ${0}))
+
+# Get the project'r root dir
+project_root=$(dirname "${dir}")
+
+# Load some function definitions in case we need them
+source ${project_root}/bin/functions.sh
+
+# Get a list of all update files
+all_sql_files=`cd ${project_root}/schemas/ispyb/updates && ls *.sql && cd ../..`
+
+# Get a list of all update files recorded in the SchemaStatus table
+done_sql_files=`mysql --defaults-file=${project_root}/.my.cnf -D $DB --skip-column-names --silent --raw -e "SELECT scriptName FROM SchemaStatus WHERE schemaStatus = 'DONE' ORDER BY recordTimeStamp;"`
+
+# Make a list of all update files not recorded in the SchemaStatus table
+arr=()
+while read -r sql_file; do
+  contains "$done_sql_files" "$sql_file"
+  if [[ $? -ne 0 ]]
+  then
+    echo "$sql_file"
+  fi
+done <<< "$all_sql_files"

build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Script to create the database from the schema .sql files and identify any
 # update .sql files that haven't been run. Also generate documentation from
@@ -11,44 +11,32 @@
 
 source bin/functions.sh
 
-if [ -z "${db}" ]
+if [ -z "${DB}" ]
 then
-  db="ispyb_build"
+  DB="ispyb_build"
 fi
 
 echo "Dropping + creating build database"
-mysql --defaults-file=.my.cnf -e "DROP DATABASE IF EXISTS $db; CREATE DATABASE $db; SET GLOBAL log_bin_trust_function_creators=ON;"
+mysql --defaults-file=.my.cnf -e "DROP DATABASE IF EXISTS $DB; CREATE DATABASE $DB; SET GLOBAL log_bin_trust_function_creators=ON;"
 
 if [[ $? -eq 0 ]]
 then
-  mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/tables.sql
-  mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/lookups.sql
-  mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/data.sql
-  mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/routines.sql
-  mysql --defaults-file=.my.cnf -D $db < grants/ispyb_acquisition.sql
-  mysql --defaults-file=.my.cnf -D $db < grants/ispyb_processing.sql
-  mysql --defaults-file=.my.cnf -D $db < grants/ispyb_web.sql
-  mysql --defaults-file=.my.cnf -D $db < grants/ispyb_import.sql
+  mysql --defaults-file=.my.cnf -D $DB < schemas/ispyb/tables.sql
+  mysql --defaults-file=.my.cnf -D $DB < schemas/ispyb/lookups.sql
+  mysql --defaults-file=.my.cnf -D $DB < schemas/ispyb/data.sql
+  mysql --defaults-file=.my.cnf -D $DB < schemas/ispyb/routines.sql
+  mysql --defaults-file=.my.cnf -D $DB < grants/ispyb_acquisition.sql
+  mysql --defaults-file=.my.cnf -D $DB < grants/ispyb_processing.sql
+  mysql --defaults-file=.my.cnf -D $DB < grants/ispyb_web.sql
+  mysql --defaults-file=.my.cnf -D $DB < grants/ispyb_import.sql
 
-  # Identify update .sql files that haven't been run, and run them
-  all_sql_files=`cd schemas/ispyb/updates && ls *.sql && cd ../..`
-
-  done_sql_files=`mysql --defaults-file=.my.cnf -D $db --skip-column-names --silent --raw -e "SELECT scriptName FROM SchemaStatus WHERE schemaStatus = 'DONE' ORDER BY recordTimeStamp;"`
-
-  arr=()
-  while read -r sql_file; do
-    contains "$done_sql_files" "$sql_file"
-    if [[ $? -ne 0 ]]
-    then
-      arr+=("$sql_file")
-      mysql --defaults-file=.my.cnf -D $db < schemas/ispyb/updates/$sql_file
-    fi
-  done <<< "$all_sql_files"
+  arr=$(bin/missed_updates.sh)
 
   if [ -n "$arr" ]; then
-    echo "The following schemas/ispyb/updates/*.sql files were sourced:"
-    for val in "${arr[@]}"; do
-      echo "$val"
+    echo "Running schemas/ispyb/updates/*.sql files that haven't yet been run:"
+    for sql_file in ${arr[@]}; do
+      echo "$sql_file"
+      mysql --defaults-file=.my.cnf -D ${DB} < "schemas/ispyb/updates/${sql_file}"
     done
   else
     echo "No new schemas/ispyb/updates/*.sql files."
@@ -57,8 +45,8 @@ then
   # Generate table and sproc documentation
   if [ -d "bin" ] && [ -d "docs" ]; then
     cd bin
-    ./db_procs_to_rst.sh $db > ../docs/list_of_procs.rst
-    ./db_tables_to_rst.sh $db > ../docs/list_of_tables_and_columns.rst
+    ./db_procs_to_rst.sh $DB > ../docs/list_of_procs.rst
+    ./db_tables_to_rst.sh $DB > ../docs/list_of_tables_and_columns.rst
     cd ..
   fi
 

docs/list_of_tables_and_columns.rst

@@ -1317,26 +1317,29 @@
    ``y``,float,""
    **ParticleClassification**,table,"Results of 2D or 2D classification"
    ``particleClassificationId``,int,""
-   ``particlePickerId``,int,""
-   ``type``,enum,"Indicates the type of particle classification"
-   ``batchNumber``,int,"Corresponding to batch number"
    ``classNumber``,int,"Identified of the class. A unique ID given by Relion"
    ``classImageFullPath``,varchar,"The PNG of the class"
-   ``numberOfParticlesPerBatch``,int,"total number of particles per batch (a large integer)"
-   ``numberOfClassesPerBatch``,int,""
    ``particlesPerClass``,int,"Number of particles within the selected class, can then be used together with the total number above to calculate the percentage"
    ``rotationAccuracy``,int,"???"
    ``translationAccuracy``,float,"Unit: Angstroms"
    ``estimatedResolution``,float,"???, Unit: Angstroms"
    ``overallFourierCompleteness``,float,""
+   ``particleClassificationGroupId``,int,""
+   **ParticleClassificationGroup**,table,""
+   ``particleClassificationGroupId``,int,""
+   ``particlePickerId``,int,""
+   ``programId``,int,""
+   ``type``,enum,"Indicates the type of particle classification"
+   ``batchNumber``,int,"Corresponding to batch number"
+   ``numberOfParticlesPerBatch``,int,"total number of particles per batch (a large integer)"
+   ``numberOfClassesPerBatch``,int,""
    ``symmetry``,varchar,""
    **ParticleClassification_has_CryoemInitialModel**,table,""
    ``particleClassificationId``,int,""
    ``cryoemInitialModelId``,int,""
    **ParticlePicker**,table,"An instance of a particle picker program that was run"
    ``particlePickerId``,int,""
-   ``particlePickerProgramId``,int,""
-   ``particleClassificationProgramId``,int,""
+   ``programId``,int,""
    ``firstMotionCorrectionId``,int,""
    ``particlePickingTemplate``,varchar,"Cryolo model"
    ``particleDiameter``,float,"Unit: nm"

release.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Script to make a release from code currently in the repo if none exists for
 # the current schema version.

schemas/ispyb/data.sql

@@ -422,14 +422,6 @@ INSERT INTO `ContainerRegistry` (`containerRegistryId`, `barcode`, `comments`, `
 /*!40000 ALTER TABLE `ContainerReport` DISABLE KEYS */;
 /*!40000 ALTER TABLE `ContainerReport` ENABLE KEYS */;
 
---
--- Dumping data for table `ContainerType`
---
-
-/*!40000 ALTER TABLE `ContainerType` DISABLE KEYS */;
-INSERT INTO `ContainerType` (`containerTypeId`, `name`, `proposalType`, `active`, `capacity`, `wellPerRow`, `dropPerWellX`, `dropPerWellY`, `dropHeight`, `dropWidth`, `dropOffsetX`, `dropOffsetY`, `wellDrop`) VALUES (1,'B21_8+1','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(2,'B21_96','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(3,'B21_1tube','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(4,'I22_Capillary_Rack_20','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(5,'I22_Grid_100','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(6,'I22_Grid_45','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(7,'P38_Capillary_Rack_27','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(8,'P38_Solids','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(9,'P38_Powder','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(10,'B22_6','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(11,'I11_Capillary_Rack_6','scm',1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(12,'Puck','mx',1,16,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),(13,'ReferencePlate','mx',1,16,2,1,1,1,1,0,0,-1),(14,'CrystalQuickX','mx',1,192,12,2,1,0.5,1,0,0,-1),(15,'MitegenInSitu','mx',1,192,12,2,1,0.5,1,0,0,-1),(16,'FilmBatch','mx',1,96,12,1,1,1,1,0,0,-1),(17,'MitegenInSitu_3_Drop','mx',1,288,12,3,1,0.5,1,0,0,-1),(18,'Greiner 3 Drop','mx',1,288,12,3,1,0.5,1,0,0,-1),(19,'MRC Maxi','mx',1,48,6,1,1,1,0.5,0,0,-1),(20,'MRC 2 Drop','mx',1,192,12,1,2,1,0.5,0.5,0,-1),(21,'Griener 1536','mx',1,1536,12,4,4,1,1,0,0,-1),(22,'3 Drop Square','mx',1,288,12,2,2,1,1,0,0,3),(23,'SWISSCI 3 Drop','mx',1,288,12,2,2,1,1,0,0,1),(24,'1 drop','mx',1,96,12,1,1,0.5,0.5,0,0,-1),(25,'LCP Glass','mx',1,96,12,1,1,1,1,0,0,-1),(26,'PCRStrip','scm',1,9,9,1,1,1,1,0,0,-1);
-/*!40000 ALTER TABLE `ContainerType` ENABLE KEYS */;
-
 --
 -- Dumping data for table `CourierTermsAccepted`
 --
@@ -591,14 +583,6 @@ INSERT INTO `EnergyScan` (`energyScanId`, `sessionId`, `blSampleId`, `fluorescen
 /*!40000 ALTER TABLE `ExperimentKindDetails` DISABLE KEYS */;
 /*!40000 ALTER TABLE `ExperimentKindDetails` ENABLE KEYS */;
 
---
--- Dumping data for table `ExperimentType`
---
-
-/*!40000 ALTER TABLE `ExperimentType` DISABLE KEYS */;
-INSERT INTO `ExperimentType` (`experimentTypeId`, `name`, `proposalType`, `active`) VALUES (1,'Default',NULL,1),(2,'MXPressE','mx',1),(3,'MXPressO','mx',1),(4,'MXPressE_SAD','mx',1),(5,'MXScore','mx',1),(6,'MXPressM','mx',1),(7,'MAD','mx',1),(8,'SAD','mx',1),(9,'Fixed','mx',1),(10,'Ligand binding','mx',1),(11,'Refinement','mx',1),(12,'OSC','MX',1),(13,'MAD - Inverse Beam','mx',1),(14,'SAD - Inverse Beam','mx',1),(15,'MESH','mx',1),(16,'XFE','mx',1),(17,'Stepped transmission','mx',1),(18,'XChem High Symmetry','mx',1),(19,'XChem Low Symmetry','mx',1),(20,'Commissioning','mx',1),(21,'HPLC','scm',1),(22,'Robot','scm',1),(23,'Rack','scm',1),(24,'Grid','scm',1),(25,'Solids','scm',1),(26,'Powder','scm',1),(27,'Peltier','scm',1),(28,'Spectroscopy','scm',1),(29,'CD Spectroscopy','scm',1),(30,'Microscopy','scm',1),(31,'Imaging','scm',1),(32,'CD Thermal Melt','scm',1),(33,'Fixed Energy At Ambient With Robot','scm',1);
-/*!40000 ALTER TABLE `ExperimentType` ENABLE KEYS */;
-
 --
 -- Dumping data for table `Frame`
 --
@@ -870,6 +854,13 @@ INSERT INTO `PDB` (`pdbId`, `name`, `contents`, `code`) VALUES (6,'ceo2','\r\nda
 /*!40000 ALTER TABLE `ParticleClassification` DISABLE KEYS */;
 /*!40000 ALTER TABLE `ParticleClassification` ENABLE KEYS */;
 
+--
+-- Dumping data for table `ParticleClassificationGroup`
+--
+
+/*!40000 ALTER TABLE `ParticleClassificationGroup` DISABLE KEYS */;
+/*!40000 ALTER TABLE `ParticleClassificationGroup` ENABLE KEYS */;
+
 --
 -- Dumping data for table `ParticleClassification_has_CryoemInitialModel`
 --
@@ -1457,4 +1448,4 @@ INSERT INTO `v_run` (`runId`, `run`, `startDate`, `endDate`) VALUES (1,'2008-01'
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2021-03-05 16:12:02
+-- Dump completed on 2021-04-13 17:47:06