Merge branch '275-batch-mode-launcher-is-broken'

batch/docker-entrypoint.batch.sh

@@ -1,3 +1,6 @@
 #!/bin/env bash
-bash wait-for-it.sh 0.0.0.0:${WEB_APP_PORT} --timeout=0 &&
-python3 run_batch.py /input.csv
+
+set -eo pipefail
+
+bash /wait-for-it.sh ${WEB_APP_HOST}:${WEB_APP_PORT} --timeout=0
+python3 /run_batch.py /input.csv

batch/example_input.csv

@@ -1,2 +1,3 @@
 name,ra,dec
-20dfgfdgdf,176.4545667755,-30.68695948493490
+2022example1,123.453454644,-30.34324332353
+2022example2,120.32304852353,20.03204833089302

batch/run_batch.py

@@ -4,6 +4,7 @@
 import time
 from urllib.request import Request
 from urllib.request import urlopen
+from urllib.error import HTTPError
 
 
 def post_transient_from_csv(path_to_input_csv: str, base_url: str) -> None:
@@ -27,6 +28,9 @@ def post_transient_from_csv(path_to_input_csv: str, base_url: str) -> None:
                 data = json.loads(response.read())
                 post_message = data.get("message", "no message returned by blast")
                 print(f"{post_message}")
+            except HTTPError as e:
+                print(e.code)
+                print(e.read())
             except Exception as e:
                 print(f"{name}: {e}")
 
@@ -76,28 +80,32 @@ def transient_processing_progress(path_to_output_csv: str) -> float:
     with open(path_to_output_csv, newline="") as csv_file:
         reader = csv.DictReader(csv_file)
         for transient in reader:
-            status = transient["transient_processing_status"]
-
-            if status == "processing":
+            progress = int(transient["transient_progress"])
+            # TODO: This progress monitor is not ideal, because if a transient fails to begin (0) or the workflow
+            #       fails to complete (<100), the script will idle indefinitely.
+            if progress >= 0:
                 processing = +1
-            else:
+            elif progress == 100:
                 completed = +1
 
     return completed / (processing + completed)
 
 
 if __name__ == "__main__":
-    localhost = "http://0.0.0.0:8000"
+    import os
+    from datetime import datetime, timezone
+    localhost = f'''http://{os.getenv('WEB_APP_HOST')}:{os.getenv('WEB_APP_PORT')}'''
     post_endpoint = "/api/transient/post/"
     get_endpoint = "/api/transient/get/"
 
     input_csv = str(sys.argv[1])
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
     post_transient_from_csv(input_csv, f"{localhost}{post_endpoint}")
-    download_data_snapshot(input_csv, "/results.csv", f"{localhost}{get_endpoint}")
-    batch_progress = transient_processing_progress("/results.csv")
+    download_data_snapshot(input_csv, f"/results/blast_results.{timestamp}.csv", f"{localhost}{get_endpoint}")
+    batch_progress = transient_processing_progress(f"/results/blast_results.{timestamp}.csv")
 
     while batch_progress < 1.0:
         print(batch_progress)
         time.sleep(10)
-        download_data_snapshot(input_csv, "/results.csv", f"{localhost}{get_endpoint}")
-        batch_progress = transient_processing_progress("/results.csv")
+        download_data_snapshot(input_csv, f"/results/blast_results.{timestamp}.csv", f"{localhost}{get_endpoint}")
+        batch_progress = transient_processing_progress(f"/results/blast_results.{timestamp}.csv")

docker/docker-compose.yml

@@ -117,12 +117,13 @@ services:
   batch:
     image: python:3.9-buster
     command: bash docker-entrypoint.batch.sh
-    network_mode: "host"
+    networks:
+      - internal
     volumes:
-      - ../batch/run_batch.py:/run_batch.py
-      - ../app/entrypoints/wait-for-it.sh:/wait-for-it.sh
-      - ../batch/docker-entrypoint.batch.sh:/docker-entrypoint.batch.sh
-      - ${BATCH_CSV}:/input.csv
-      - ${OUTPUT_CSV}:/results.csv
+      - ../batch/run_batch.py:/run_batch.py:ro
+      - ../app/entrypoints/wait-for-it.sh:/wait-for-it.sh:ro
+      - ../batch/docker-entrypoint.batch.sh:/docker-entrypoint.batch.sh:ro
+      - ${BATCH_CSV}:/input.csv:ro
+      - ${OUTPUT_DIR}:/results
     profiles:
       - "batch"

docs/usage/batch.rst

@@ -39,11 +39,12 @@ path to you input batch file, absolute or relative to the docker-compose.yml fil
 
     BATCH_CSV=<path_to_your_transient_input_file>
 
-You must also specify the path to the empty results file
+You must also specify the path to the target output directory, in which a timestamped
+CSV file will be generated (:code:`/tmp/blast_results` by default):
 
 .. code::
 
-    OUTPUT_CSV=<path_to_your_results_file>
+    OUTPUT_DIR=<path_to_output_results>
 
 Running in batch mode
 ---------------------

env/.env.default

@@ -50,7 +50,7 @@ ALLOW_API_POST = YES
 
 #Batch mode
 BATCH_CSV = ../batch/example_input.csv
-OUTPUT_CSV = ../batch/results.csv
+OUTPUT_DIR = /tmp/blast_results
 
 
 #Cutout settings, false if cutouts shouldn't be re download, True if they should

run/blastctl

@@ -42,6 +42,7 @@ case "$PROFILE" in
     COMPOSE_LOG_LIST="sphinx"
     ;;
   "batch")
+    COMPOSE_FILE="docker-compose.dev.yaml"
     ENV_FILE=".env.dev"
     COMPOSE_PROJECT_NAME="blast-dev"
     COMPOSE_ARGS_UP="${COMPOSE_ARGS_UP} --abort-on-container-exit"