Merge pull request #13 from jmetz/uploader-object

Status on EBI, files on Hypha (for now)

.github/scripts/update_status.py

@@ -0,0 +1,63 @@
+import os
+import io
+import argparse
+from typing import Optional
+import json
+from minio import Minio
+# from minio.error import S3Error
+
+def create_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("model_name", help="Model name")
+    parser.add_argument("status", help="Status")
+    parser.add_argument("step", help="Step", nargs="?", default=0, type=int)
+    parser.add_argument("num_steps", help="Status", nargs="?", default=0, type=int)
+    return parser
+
+
+def get_args(argv: Optional[list] = None):
+    """
+    Get command-line arguments
+    """
+    parser = create_parser()
+    return parser.parse_args(argv)
+
+
+def main():
+    args = get_args()
+    model_name = args.model_name
+    step = args.step
+    num_steps = args.num_steps
+    filename = "status.json"
+    status = args.status
+    s3_host = os.getenv("S3_HOST")
+    s3_bucket = os.getenv("S3_BUCKET")
+    s3_root_folder = os.getenv("S3_ROOT_FOLDER")
+    s3_access_key_id = os.getenv("S3_ACCESS_KEY_ID")
+    s3_secret_access_key = os.getenv("S3_SECRET_ACCESS_KEY")
+
+    client = Minio(
+        s3_host,
+        access_key=s3_access_key_id,
+        secret_key=s3_secret_access_key,
+    )
+    found = client.bucket_exists(s3_bucket)
+    if not found:
+        raise Exception("target bucket does not exist: {s3_bucket}")
+
+    status_message = json.dumps({"status": status, step=step, num_steps=num_steps}).encode()
+
+    status_file_object = io.BytesIO(status_message)
+    s3_path = f"{s3_root_folder}/{model_name}/{filename}"
+
+    client.put_object(
+        s3_bucket,
+        s3_path,
+        status_file_object,
+        length=len(status_message),
+        content_type="application/json",
+    )
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/update_status.sh

@@ -1,3 +1,55 @@
 #!/bin/sh 
-curl -X PUT -H 'Content-Type: application/json' -d '{"status": "'"$2"'"}' "$1"  
+# Updated to use S3 creds:
+# S3_HOST
+# S3_BUCKET
+# S3_ROOT_FOLDER
+# S3_ACCESS_KEY_ID
+# S3_SECRET_ACCESS_KEY
+# First arg is now model_nickname
+
+FILENAME=status.json
+
+MODEL_NAME=$1
+STATUS=$2
+
+if [ -z "$MODEL_NAME" ]; then 
+    printf '%s\n' "MODEL_NAME is unset or empty" >&2;
+    exit 1
+fi
+if [ -z "$S3_HOST" ]; then 
+    printf '%s\n' "S3_HOST is unset or empty" >&2;
+    exit 1
+fi
+if [ -z "$S3_BUCKET" ]; then 
+    printf '%s\n' "S3_BUCKET is unset or empty" >&2;
+    exit 1
+fi
+if [ -z "$S3_ROOT_FOLDER" ]; then 
+    printf '%s\n' "S3_ROOT_FOLDER is unset or empty" >&2;
+    exit 1
+fi
+if [ -z "$S3_ACCESS_KEY_ID" ]; then 
+    printf '%s\n' "S3_ACCESS_KEY_ID is unset or empty" >&2;
+    exit 1
+fi
+if [ -z "$S3_SECRET_ACCESS_KEY" ]; then 
+    printf '%s\n' "S3_SECRET_ACCESS_KEY is unset or empty" >&2;
+    exit 1
+fi
+
+
+#curl -X PUT -H 'Content-Type: application/json' -d '{"status": "'"$2"'"}' "$1"  
+
+RESOURCE="/${S3_BUCKET}/${S3_ROOT_FOLDER}/${MODEL_NAME}/${FILENAME}"
+CONTENT_TYPE="application/json"
+DATE=`date -R`
+_SIGNATURE="PUT\n\n${CONTENT_TYPE}\n${DATE}\n${RESOURCE}"
+SIGNATURE=`echo -en ${_SIGNATURE} | openssl sha1 -hmac ${S3_SECRET_ACCESS_KEY} -binary | base64`
+
+curl -X PUT -d '{"status": "'"$STATUS"'"}' \
+          -H "Host: ${S3_HOST}" \
+          -H "Date: ${DATE}" \
+          -H "Content-Type: ${CONTENT_TYPE}" \
+          -H "Authorization: AWS ${S3_ACCESS_KEY_ID}:${SIGNATURE}" \
+          https://${S3_HOST}${RESOURCE}
 

.github/workflows/ci_runner.yaml

@@ -5,7 +5,7 @@ on:
         inputs:
             status_url:
                 description: 'URL endpoint for status to be posted to'
-                required: true
+                required: false
                 type: string
             model_nickname: 
                 description: 'Nickname of the model - to be used to access the model data on S3'
@@ -17,40 +17,51 @@ jobs:
         runs-on: ubuntu-latest
         strategy:
             fail-fast: false
+        env: 
+            S3_HOST: ${{secrets.S3_HOST}}
+            S3_BUCKET: ${{secrets.S3_BUCKET}}
+            S3_ROOT_FOLDER: ${{secrets.S3_ROOT_FOLDER}}
+            S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}}
+            S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
         steps:
             - uses: actions/checkout@v3
 
+            - name: Install workflow script dependencies
+              run: |
+                    echo "Installing workflow script dependencies"
+                    python -m pip install --upgrade pip
+                    python -m pip install "minio==7.2.0"
             - name: Update status
-              run: sh .github/scripts/update_status.sh  "${{ inputs.status_url }}" "Starting testing of ${{inputs.model_nickname}}"  
+              run: python .github/scripts/update_status.py  "${{ inputs.model_nickname }}" "Starting testing of ${{inputs.model_nickname}}" "1" "8"  
 
             - name: Install dependencies
               run: |
-                    sh .github/scripts/update_status.sh  "${{ inputs.status_url }}" "Installing dependencies of ${{inputs.model_nickname}}"  
+                    python .github/scripts/update_status.py  "${{ inputs.model_nickname }}" "Installing dependencies of ${{inputs.model_nickname}}" "2" "8"  
                     echo "Installing dependencies"
                     python -m pip install --upgrade pip
                     python -m pip install flake8 pytest
                     if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
 
             - name: Additional Steps
               run: |
-                  sh .github/scripts/update_status.sh  "${{ inputs.status_url }}" "Running additional steps of ${{inputs.model_nickname}}"  
+                  python .github/scripts/update_status.py  "${{ inputs.model_nickname }}" "Running additional steps of ${{inputs.model_nickname}}" "3" "8"  
                   echo "Running additional steps..."
                   sleep 30
                   echo "Done"
-                  sh .github/scripts/update_status.sh  "${{ inputs.status_url }}" "Done running additional steps of ${{inputs.model_nickname}}"  
+                  python .github/scripts/update_status.py  "${{ inputs.model_nickname }}" "Done running additional steps of ${{inputs.model_nickname}}" "4" "8"  
 
             - name: Main testing
               run: |
-                  sh .github/scripts/update_status.sh  "${{ inputs.status_url }}" "Running main testing of ${{inputs.model_nickname}}"  
+                  python .github/scripts/update_status.py  "${{ inputs.model_nickname }}" "Running main testing of ${{inputs.model_nickname}}" "5" "8"  
                   echo "Running main testing"
                   sleep 30
                   echo "Done"
-                  sh .github/scripts/update_status.sh  "${{ inputs.status_url }}" "Testing complete of ${{inputs.model_nickname}}"  
+                  python .github/scripts/update_status.py  "${{ inputs.model_nickname }}" "Testing complete of ${{inputs.model_nickname}}" "6" "8"  
 
             - name: Publish within S3
               run: |
-                  sh .github/scripts/update_status.sh  "${{ inputs.status_url }}" "Running publishing of ${{inputs.model_nickname}}"  
+                  python .github/scripts/update_status.py  "${{ inputs.model_nickname }}" "Running publishing of ${{inputs.model_nickname}}" "7" "8"  
                   echo "Somehow doing publising workflow 🤔"
                   sleep 30
                   echo "Done"
-                  sh .github/scripts/update_status.sh  "${{ inputs.status_url }}" "Publishing complete of ${{inputs.model_nickname}}"  
+                  python .github/scripts/update_status.py  "${{ inputs.model_nickname }}" "Publishing complete of ${{inputs.model_nickname}}" "8" "8"  

.gitignore

@@ -12,3 +12,4 @@ bun.lockb
 
 # Local Netlify folder
 .netlify
+dist/

README.md

@@ -1,38 +1,71 @@
-# create-svelte
+# Uploader Overview
 
-Everything you need to build a Svelte project, powered by [`create-svelte`](https://github.com/sveltejs/kit/tree/master/packages/create-svelte).
+```mermaid
+stateDiagram-v2
+    User --> Client
+    Upload --> CI
+    Upload --> ZipFile
+    ZipFile --> PresignedURL
+    CreateStatus --> status.json
+    status.json --> Status
+    Unzip --> FileSet
+    CreateTritonModel --> TritonModel
+    TestTritonModel --> BioEngineRunner
+    TritonModel --> BioEngineRunner
+    Publish --> PublishedFileSet
 
-## Creating a project
+    state Client {
+        Add --> Edit
+        Edit --> Verify
+        Verify --> Upload
+        Upload --> PresignedURL
+        PresignedURL --> Status
+    }
 
-If you're seeing this, you've probably already done this step. Congrats!
+    state CI{
+        CreateStatus --> Unzip
+        Unzip --> TestModel
+        TestModel --> CreateTritonModel
+        CreateTritonModel --> TestTritonModel
+        TestTritonModel --> Publish
+    }
 
-```bash
-# create a new project in the current directory
-npm create svelte@latest
+    state HyphaAppEngine {
+        HyphaS3
+        BioEngineRunner
+
+        state HyphaS3{
+            ZipFile
+        }
+    }
+
+    state EBI_S3 {
+        status.json
+        FileSet
+        TritonModel
+    }
 
-# create a new project in my-app
-npm create svelte@latest my-app
+    state Zenodo {
+        PublishedFileSet
+    }
 ```
 
+
 ## Developing
 
-Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
+Start a development server:
 
 ```bash
-npm run dev
+ntl dev
 
-# or start the server and open the app in a new browser tab
-npm run dev -- --open
+# or run without netlify functions: 
+npm run dev 
 ```
 
 ## Building
 
-To create a production version of your app:
+To create a production version of the app:
 
 ```bash
 npm run build
 ```
-
-You can preview the production build with `npm run preview`.
-
-> To deploy your app, you may need to install an [adapter](https://kit.svelte.dev/docs/adapters) for your target environment.

functions/create_status.js

@@ -3,6 +3,8 @@ const { S3_ENDPOINT,  S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY } = process.env;
 
 
 export default async (event, context) => {
+
+
     console.log(S3_ENDPOINT);
     console.log(S3_ACCESS_KEY_ID);
     console.log(S3_SECRET_ACCESS_KEY);

functions/generate_name.js

@@ -45,6 +45,7 @@ function generate_name(starts, animals){
 
 
 export default async () => {
+    console.log("Generating nickname from allowed lists...");
     const [adjectives, animals_with_icons, taken_names] = await Promise.all([
         get_adjectives(),
         get_animals(),
@@ -53,6 +54,8 @@ export default async () => {
     while(name in taken_names){
         name = generate_name(adjectives, animals_with_icons);
     }
+    console.log("Name generated:");
+    console.log(name);
     const res = Response.json(name);
     res.headers.set("Access-Control-Allow-Origin", "*");
     res.headers.append("Access-Control-Allow-Headers", "*");

functions/get_status.js

@@ -0,0 +1,20 @@
+export default async (event) => {
+    const data = await event.json();
+    const url = data.url;
+    let obj = {};
+    try{
+        obj = await (await fetch(url)).json();
+    }catch(err){
+        obj = {status: "CI not started yet"};
+    }
+    console.log("Got status for");
+    console.log(url);
+    console.log("Answer:");
+    console.log(obj);
+    const res = Response.json(obj);
+    res.headers.set("Access-Control-Allow-Origin", "*");
+    res.headers.append("Access-Control-Allow-Headers", "*");
+    res.headers.append("Access-Control-Allow-Methods", "*");
+    return res;
+
+}

functions/notify_ci.js

@@ -18,13 +18,14 @@ export default async (event, context) => {
         method: "POST",
         headers: headers, 
         body: JSON.stringify({
-            'ref': 'main',
+            'ref': 'uploader-object',
             'inputs':{
-                'status_url': data.status_url,
+                //'status_url': data.status_url,
                 'model_nickname': data.model_nickname,
             }
         })
     };
+    let resp_obj = {};
 
     if(!data.status_url){
         const res = Response.json({'message': "Failed: status_url not found in request json"});
@@ -38,7 +39,7 @@ export default async (event, context) => {
         try{
             let resp = await fetch(GITHUB_URL, options);
             try{
-                console.log(await resp.json());
+                resp_obj = await resp.json();
             }catch(err){
                 console.log("No JSON in response");
             }
@@ -55,7 +56,10 @@ export default async (event, context) => {
             return res;
         }
         // const res = new Response("Success");
-        const res = Response.json({"message":"Success"});
+        const reply_obj = {"status":`Contacted CI: ${resp_obj.message}`};
+        console.log("Response from CI:");
+        console.log(resp_obj);
+        const res = Response.json(reply_obj);
         res.headers.set("Access-Control-Allow-Origin", "*");
         res.headers.append("Access-Control-Allow-Headers", "*");
         res.headers.append("Access-Control-Allow-Methods", "*");