Merge branch 'dev' of github.com:biocore/qiita

.github/workflows/qiita-ci.yml

@@ -96,7 +96,9 @@ jobs:
           export REDBIOM_HOST="http://localhost:7379"
 
           pip install . --no-binary redbiom
-          conda install -c conda-forge --yes biom-format
+          # 10.2022
+          # this is for redbiom / biom-format (so fine to delete in the future)
+          pip install future
           pwd
           mkdir ~/.qiita_plugins
 
@@ -182,6 +184,8 @@ jobs:
           conda activate qiita
           export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
           export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
+          # for testing we only need to have this set, not actually exist
+          export QIITA_JOB_SCHEDULER_EPILOGUE=`/path/to/epilogue/file`
           export REDBIOM_HOST="http://localhost:7379"
 
           nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=${COVER_PACKAGE// / --cover-package=} -e 'test_submit_EBI_parse_EBI_reply_failure' -e 'test_full_submission'

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Qiita changelog
 
+Version 2022.09
+---------------
+
+* Moved Qiita's code base and plugins to SLURM (from Torque). The plugins updated are: qiita-spots/qp-woltka, biocore/mg-scripts, qiita-spots/qp-spades, qiita-spots/qp-meta, qiita-spots/qp-fastp-minimap2.
+* Pinned the paramiko version to < 2.9 [as newer versions were causing issues with older systems](https://github.com/paramiko/paramiko/issues/1961#issuecomment-1008119073).
+* Pinned the scipy version to < 1.8 to avoid issues with the biom-format library.
+* Updates to the INSTALL instructions (thank you @aliu104 !)
+
 Version 2022.07
 ---------------
 

INSTALL.md

@@ -6,7 +6,7 @@ Qiita is pip installable, but depends on specific versions of python and non-pyt
 
 ## Install and setup miniconda
 
-Download the appropriate installer [here](http://conda.pydata.org/docs/install/quick.html) corresponding to your operating system and execute it.
+Download the appropriate installer [here](https://repo.anaconda.com/miniconda/) corresponding to your operating system and execute it.
 
 Next, ensure conda is up-to-date.
 
@@ -19,6 +19,7 @@ conda update conda
 Setup a virtual environment in conda named `qiita` by executing the following:
 
 ```bash
+conda config --add channels conda-forge
 conda create -q --yes -n qiita python=3.6 pip libgfortran numpy nginx
 ```
 
@@ -40,7 +41,7 @@ $ which python
 (qiita)
 ```
 
-If you don't see this output, your `$PATH` variable was setup incorrectly or you haven't restarted your shell. Consult the [conda documentation](http://conda.pydata.org/docs/install/quick.html).
+If you don't see this output, your `$PATH` variable was setup incorrectly or you haven't restarted your shell. Consult the [conda documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html).
 
 As long as you are in the active qiita environment, commands such as `pip install` or `python` will refer to and be contained within this virtual environment.
 
@@ -54,7 +55,7 @@ source deactivate
 Install the non-python dependencies
 -----------------------------------
 
-* [PostgreSQL](http://www.postgresql.org/download/) (minimum required version 9.5.14, we have tested most extensively with 9.5.15)
+* [PostgreSQL](http://www.postgresql.org/download/) (currently using v13)
 * [redis-server](http://redis.io) (we have tested most extensively with 2.8.17)
 * [webdis] (https://github.com/nicolasff/webdis) (latest version should be fine but we have tested the most with 9ee6fe2 - Feb 6, 2016)
 
@@ -64,9 +65,43 @@ There are several options to install these dependencies depending on your needs:
 - Alternatively, you could install them via conda. However, the conda repository may not have the exact versions of these dependencies that you want.
 - You could setup a full development environment with [Vagrant](https://www.vagrantup.com/), and continue using conda under it to primarily manage python dependencies. Note that we don't cover Vagrant in these instructions.
 
+### PostgreSQL installation on Linux
+The following instructions have been adapted from [this site](https://computingforgeeks.com/how-to-install-postgresql-13-on-ubuntu/) and tested on Ubuntu v20.04.4 for Postgres v13.
+
+First, ensure that you have updated packages and reboot the system with:
+```bash
+sudo apt update && sudo apt -y full-upgrade
+[ -f /var/run/reboot-required ] && sudo reboot -f
+```
+You can reboot the system with `sudo reboot` in case any packages were updated.
+
+Next, we need to add the Postgres repository to our system:
+```bash
+sudo apt update
+sudo apt install curl gpg gnupg2 software-properties-common apt-transport-https lsb-release ca-certificates
+curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc|sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg
+echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee  /etc/apt/sources.list.d/pgdg.list
+```
+Adding the repository has added many different packages, which allows us to now install Postgres v13 with the following commands:
+```bash
+sudo apt update
+sudo apt install postgresql-13 postgresql-client-13
+```
+Now, we need to reconfigure the `pg_hba.conf` file and change all occurrences of `md5` and `peer` to `trust`. You can access the file with:
+```bash
+sudo vim /etc/postgresql/13/main/pg_hba.conf
+```
+To make sure all changes have been reflected, restart the Postgres server:
+```bash
+sudo service postgresql restart
+```
+Installing Postgres is now complete. Note that you will need to start the Postgres server every time you start the Qiita server. You can do this with the following command:
+```bash
+sudo service postgresql start
+```
 ### PostgreSQL installation on Mac OS X
 
-For Mac OS X, you can either install postgres through the [Postgres.app](https://postgresapp.com/downloads.html). These instructions were tested with the Postgres.app v9.5.
+For Mac OS X, you can either install postgres through the [Postgres.app](https://postgresapp.com/downloads.html). These instructions were tested with the Postgres.app v9.5, v13.
 
 You'll then need to ensure that the postgres binaries (for example, ``psql``) are in your executable search path (``$PATH`` environment variable). If you are using Postgres.app on OS X, you can do this by running the following, though you may have to replace`~/.bash_profile`with `~/.zshrc` if you're using zshell rather than the built-in bash, and you may have to change the version number `Versions/9.3/` to the exact one that you are installing:
 
@@ -75,14 +110,20 @@ echo 'export PATH="$PATH:/Applications/Postgres.app/Contents/Versions/9.5/bin/"'
 source ~/.bash_profile
 ```
 
-### Redis-server installation on Mac OS X
+### Redis-server installation using Homebrew (Mac OS X, Linux)
 
-Assuming you have [homebrew](http://www.brew.sh) installed, you can install redis-server v2.8.x as follows:
+Assuming you have [homebrew](http://brew.sh) installed, you can install the latest version of the redis-server as follows:
 
 ```bash
 brew update
 brew install homebrew/versions/redis28
 ```
+### Redis-server installation using apt-get (Linux)
+
+Alternatively, you can sudo install redis:
+```bash
+sudo apt-get install redis-server
+```
 
 ### webdis
 
@@ -112,7 +153,7 @@ Install Qiita development version and its python dependencies
 Clone the git repository with the development version of Qiita into your current directory:
 
 ```bash
-git clone https://github.com/biocore/qiita.git
+git clone https://github.com/qiita-spots/qiita.git
 ```
 
 Navigate to the cloned directory and ensure your conda environment is active:
@@ -121,12 +162,17 @@ Navigate to the cloned directory and ensure your conda environment is active:
 cd qiita
 source activate qiita
 ```
-
+If you are using Ubuntu or a Windows Subsystem for Linux (WSL), you will need to ensure that you have a C++ compiler and that development libraries and include files for PostgreSQL are available. Type `cc` into your system to ensure that it doesn't result in `program not found`. The following commands will install a C++ compiler and  `libpq-dev`:
+```bash
+sudo apt install gcc              # alternatively, you can install clang instead
+sudo apt-get install libpq-dev
+```
 Install Qiita (this occurs through setuptools' `setup.py` file in the qiita directory):
 
 ```bash
 pip install . --no-binary redbiom
 ```
+Note that if you get any errors or warnings with 'certifi', you can add the `--ignore-installed` tag to the command above.
 
 At this point, Qiita will be installed and the system will start. However,
 you will need to install plugins in order to process any kind of data. For a list
@@ -148,7 +194,7 @@ Move the Qiita sample configuration file to a different directory by executing:
  cp ./qiita_core/support_files/config_test.cfg ~/.qiita_config_test.cfg
 ```
 
-Note that you will need to change `BASE_URL = https://localhost:8383` to `BASE_URL = https://localhost:21174` if you are not using NGINX.
+Note that you will need to change `BASE_URL = https://localhost:8383` to `BASE_URL = https://localhost:21174` in the new copy of the configuration file if you are not using NGINX. Additionally, you will also need to change all URLs that start with `/home/runner/work/qiita/qiita/...` into wherever your qiita directory is (e.g. `/home/<username>/qiita/...`).
 
 
 Set your `QIITA_CONFIG_FP` environment variable to point to that file (into `.bashrc` if using bash; `.zshrc` if using zshell):
@@ -162,6 +208,10 @@ Set your `QIITA_CONFIG_FP` environment variable to point to that file (into `.ba
 
 Update paths in the newly copied configuration file to match your settings, e.g. replace /home/travis/ with your user home directory.
 
+If you are working on WSL, you will need to start the redis server with the following command before making a test environment:
+```bash
+redis-server --daemonize yes --port 7777
+```
 Next, make a test environment:
 
 ```bash

qiita_core/__init__.py

@@ -6,4 +6,4 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-__version__ = "2022.07"
+__version__ = "2022.09"

qiita_core/configuration_manager.py

@@ -42,11 +42,11 @@ class ConfigurationManager(object):
         Max upload size
     valid_upload_extension : str
         The extensions that are valid to upload, comma separated
-    trq_owner : str
-        Email address of submitter of Torque jobs
-    trq_poll_val : int
-        Interval (in seconds) to wait between calls to Torque's qstat program
-    trq_dependency_q_cnt : int
+    job_scheduler_owner : str
+        Email address of submitter of jobs
+    job_scheduler_poll_val : int
+        Interval (in seconds) to wait between calls to job_scheduler program
+    job_scheduler_dependency_q_cnt : int
         Hard upper-limit on the number of an artifact's concurrent validation
         processes.
     user : str
@@ -145,7 +145,7 @@ def __init__(self):
 
         self._get_main(config)
         self._get_smtp(config)
-        self._get_torque(config)
+        self._get_job_scheduler(config)
         self._get_postgres(config)
         self._get_redis(config)
         self._get_ebi(config)
@@ -234,22 +234,22 @@ def _get_main(self, config):
             self.key_file = join(install_dir, 'qiita_core', 'support_files',
                                  'server.key')
 
-    def _get_torque(self, config):
-        """Get the configuration of the torque section"""
-        self.trq_owner = config.get('torque', 'TORQUE_JOB_OWNER')
-        self.trq_poll_val = int(config.get('torque', 'TORQUE_POLLING_VALUE'))
-        self.trq_dependency_q_cnt = config.get('torque',
-                                               'TORQUE_PROCESSING_QUEUE_COUNT')
-        self.trq_dependency_q_cnt = int(self.trq_dependency_q_cnt)
-
-        if not self.trq_owner:
-            self.trq_owner = None
-
-        if not self.trq_poll_val:
-            self.trq_poll_val = None
-
-        if not self.trq_dependency_q_cnt:
-            self.trq_dependency_q_cnt = None
+    def _get_job_scheduler(self, config):
+        """Get the configuration of the job_scheduler section"""
+        self.job_scheduler_owner = config.get(
+            'job_scheduler', 'JOB_SCHEDULER_JOB_OWNER', fallback=None)
+        self.job_scheduler_poll_val = config.get(
+            'job_scheduler', 'JOB_SCHEDULER_POLLING_VALUE', fallback=None)
+        self.job_scheduler_dependency_q_cnt = config.get(
+            'job_scheduler', 'JOB_SCHEDULER_PROCESSING_QUEUE_COUNT',
+            fallback=None)
+
+        if self.job_scheduler_poll_val is not None:
+            self.job_scheduler_poll_val = int(self.job_scheduler_poll_val)
+
+        if self.job_scheduler_dependency_q_cnt is not None:
+            self.job_scheduler_dependency_q_cnt = int(
+                self.job_scheduler_dependency_q_cnt)
 
     def _get_postgres(self, config):
         """Get the configuration of the postgres section"""

qiita_core/support_files/config_test.cfg

@@ -123,16 +123,16 @@ PASSWORD = postgres
 # The postgres password for the admin_user
 ADMIN_PASSWORD = postgres
 
-# ----------------------------- Torque settings -----------------------------
-[torque]
-# The email address of the submitter of Torque jobs
-TORQUE_JOB_OWNER = torque_user@somewhere.org
+# ----------------------------- Job Scheduler Settings -----------------------------
+[job_scheduler]
+# The email address of the submitter of jobs
+JOB_SCHEDULER_JOB_OWNER = user@somewhere.org
 
-# The number of seconds to wait between successive qstat calls
-TORQUE_POLLING_VALUE = 15
+# The number of seconds to wait between successive calls
+JOB_SCHEDULER__POLLING_VALUE = 15
 
 # Hard upper-limit on concurrently running validator jobs
-TORQUE_PROCESSING_QUEUE_COUNT = 2
+JOB_SCHEDULER_PROCESSING_QUEUE_COUNT = 2
 
 # ----------------------------- EBI settings -----------------------------
 [ebi]

qiita_core/tests/test_configuration_manager.py

@@ -59,10 +59,10 @@ def test_init(self):
         self.assertEqual(obs.cookie_secret, "SECRET")
         self.assertEqual(obs.key_file, "/tmp/server.key")
 
-        # Torque section
-        self.assertEqual(obs.trq_owner, "torque_user@somewhere.org")
-        self.assertEqual(obs.trq_poll_val, 15)
-        self.assertEqual(obs.trq_dependency_q_cnt, 2)
+        # job_scheduler section
+        self.assertEqual(obs.job_scheduler_owner, "user@somewhere.org")
+        self.assertEqual(obs.job_scheduler_poll_val, 15)
+        self.assertEqual(obs.job_scheduler_dependency_q_cnt, 2)
 
         # Postgres section
         self.assertEqual(obs.user, "postgres")
@@ -180,13 +180,13 @@ def test_get_main(self):
 
         self.assertEqual(obs.qiita_env, "")
 
-    def test_get_torque(self):
+    def test_get_job_scheduler(self):
         obs = ConfigurationManager()
 
-        conf_setter = partial(self.conf.set, 'torque')
-        conf_setter('TORQUE_JOB_OWNER', '')
-        obs._get_torque(self.conf)
-        self.assertIsNone(obs.trq_owner)
+        conf_setter = partial(self.conf.set, 'job_scheduler')
+        conf_setter('JOB_SCHEDULER_JOB_OWNER', '')
+        obs._get_job_scheduler(self.conf)
+        self.assertEqual('', obs.job_scheduler_owner)
 
     def test_get_postgres(self):
         obs = ConfigurationManager()
@@ -329,16 +329,16 @@ def test_get_portal(self):
 # The postgres password for the admin_user
 ADMIN_PASSWORD = thishastobesecure
 
-# ----------------------------- Torque settings -----------------------------
-[torque]
-# The email address of the submitter of Torque jobs
-TORQUE_JOB_OWNER = torque_user@somewhere.org
+# ------------------------- job_scheduler settings -------------------------
+[job_scheduler]
+# The email address of the submitter of jobs
+JOB_SCHEDULER_JOB_OWNER = user@somewhere.org
 
-# The number of seconds to wait between successive qstat calls
-TORQUE_POLLING_VALUE = 15
+# The number of seconds to wait between successive calls
+JOB_SCHEDULER_POLLING_VALUE = 15
 
 # Hard upper-limit on concurrently running validator jobs
-TORQUE_PROCESSING_QUEUE_COUNT = 2
+JOB_SCHEDULER_PROCESSING_QUEUE_COUNT = 2
 
 # ----------------------------- EBI settings -----------------------------
 [ebi]

qiita_db/__init__.py

@@ -27,7 +27,7 @@
 from . import user
 from . import processing_job
 
-__version__ = "2022.07"
+__version__ = "2022.09"
 
 __all__ = ["analysis", "artifact",  "archive", "base", "commands",
            "environment_manager", "exceptions", "investigation", "logger",

qiita_db/environment_manager.py

@@ -203,16 +203,12 @@ def make_environment(load_ontologies, download_reference, add_demo_user):
 
             # Insert the settings values to the database
             sql = """INSERT INTO settings
-                     (test, base_data_dir, base_work_dir, trq_owner,
-                     trq_poll_val, trq_dependency_q_cnt)
-                     VALUES (%s, %s, %s, %s, %s, %s)"""
+                     (test, base_data_dir, base_work_dir)
+                     VALUES (%s, %s, %s)"""
             qdb.sql_connection.TRN.add(
                 sql, [test,
                       qiita_config.base_data_dir,
-                      qiita_config.working_dir,
-                      qiita_config.trq_owner,
-                      qiita_config.trq_poll_val,
-                      qiita_config.trq_dependency_q_cnt])
+                      qiita_config.working_dir])
             qdb.sql_connection.TRN.execute()
             create_layout(test=test, verbose=verbose)
 

qiita_db/metadata_template/base_metadata_template.py

@@ -1537,7 +1537,7 @@ def _get_accession_numbers(self, column):
         return result
 
     def _update_accession_numbers(self, column, values):
-        """Update accession numbers stored in `column` with the ones in `values`
+        """Update accession numbers stored in `column` with `values`
 
         Parameters
         ----------