diff --git a/celery_worker_scripts/dev/START_CELERY_WORKERS_SLURM.sh b/celery_worker_scripts/dev/START_CELERY_WORKERS_SLURM.sh index 08cb894..4d83803 100644 --- a/celery_worker_scripts/dev/START_CELERY_WORKERS_SLURM.sh +++ b/celery_worker_scripts/dev/START_CELERY_WORKERS_SLURM.sh @@ -38,6 +38,7 @@ celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL} # are not sent to the batch step (the shell script). With this # option scancel also signals the batch script and its children processes." # See https://slurm.schedmd.com/scancel.html#OPT_full +echo "sending SIGTERM signal to dev celery workers" scancel --name=sumstats_service_celery_worker --signal=TERM --full echo "START spinning up dev celery workers:" diff --git a/celery_worker_scripts/dev/start_celery_worker_slurm.sh b/celery_worker_scripts/dev/start_celery_worker_slurm.sh index feadb40..ff40ff1 100644 --- a/celery_worker_scripts/dev/start_celery_worker_slurm.sh +++ b/celery_worker_scripts/dev/start_celery_worker_slurm.sh @@ -21,9 +21,14 @@ celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL} # are not sent to the batch step (the shell script). With this # option scancel also signals the batch script and its children processes." # See https://slurm.schedmd.com/scancel.html#OPT_full +echo "sending SIGTERM signal to dev celery workers" scancel --name=sumstats_service_celery_worker --signal=TERM --full # Submit new SLURM jobs for celery workers +echo "START spinning up dev celery workers:" for WORKER_ID in {1..2}; do + echo $WORKER_ID sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}" done +echo "DONE spinning up dev celery workers" + diff --git a/celery_worker_scripts/prod/START_CELERY_WORKERS_SLURM.sh b/celery_worker_scripts/prod/START_CELERY_WORKERS_SLURM.sh index 2e63477..d855861 100644 --- a/celery_worker_scripts/prod/START_CELERY_WORKERS_SLURM.sh +++ b/celery_worker_scripts/prod/START_CELERY_WORKERS_SLURM.sh @@ -44,7 +44,7 @@ singularity_cmd="singularity exec --env-file $ENV_FILE $SINGULARITY_CACHEDIR/gwa # Submit new SLURM jobs for HH celery workers echo "spinning up HH celery workers:" for WORKER_ID in {4..6}; do - echo $WORKER_ID + echo $WORKER_ID sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker_prod --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}" done echo "DONE spinning up HH celery workers" diff --git a/celery_worker_scripts/prod/start_celery_worker_slurm.sh b/celery_worker_scripts/prod/start_celery_worker_slurm.sh index ba0cb28..36d313b 100644 --- a/celery_worker_scripts/prod/start_celery_worker_slurm.sh +++ b/celery_worker_scripts/prod/start_celery_worker_slurm.sh @@ -21,18 +21,25 @@ celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL} # are not sent to the batch step (the shell script). With this # option scancel also signals the batch script and its children processes." # See https://slurm.schedmd.com/scancel.html#OPT_full +echo "sending SIGTERM signal to prod celery workers" scancel --name=sumstats_service_celery_worker_prod --signal=TERM --full # Submit new SLURM jobs for HX celery workers +echo "START spinning up HX celery workers:" for WORKER_ID in {1..3}; do + echo $WORKER_ID sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker_prod --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}" done +echo "DONE spinning up HX celery workers" # spin up the workers for hl rabbitmq ENV_FILE="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/cel_envs_hh" singularity_cmd="singularity exec --env-file $ENV_FILE $SINGULARITY_CACHEDIR/gwas-sumstats-service_latest.sif" # Submit new SLURM jobs for HH celery workers +echo "START spinning up HH celery workers:" for WORKER_ID in {4..6}; do + echo $WORKER_ID sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker_prod --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}" done +echo "DONE spinning up HH celery workers" diff --git a/celery_worker_scripts/scrontab b/celery_worker_scripts/scrontab index d3cce31..299a581 100644 --- a/celery_worker_scripts/scrontab +++ b/celery_worker_scripts/scrontab @@ -27,4 +27,4 @@ 0 10 */3 * * sbatch --mem=1200M --time=00:30:00 -o /hps/software/users/parkinso/spot/gwas/dev/scripts/cron/sumstats_service/deploy.out -e /hps/software/users/parkinso/spot/gwas/dev/scripts/cron/sumstats_service/deploy.err --wrap="/hps/software/users/parkinso/spot/gwas/dev/scripts/cron/sumstats_service/start_celery_worker_slurm.sh" # Celery worker start for sumstats service (PROD): -0 10 */3 * * sbatch --mem=1200M --time=02:00:00 -o /hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/deploy.out -e /hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/deploy.err --wrap="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/start_celery_worker_slurm.sh" \ No newline at end of file +0 10 */3 * * sbatch --mem=1200M --time=00:30:00 -o /hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/deploy.out -e /hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/deploy.err --wrap="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/start_celery_worker_slurm.sh" \ No newline at end of file