BN-106 Added github workflow realization

.github/actions/blendnet-test-linux/action.yml

@@ -0,0 +1,38 @@
+name: BlendNet Test Linux
+description: Runs test on linux with the specified blender version
+inputs:
+  blender_version:
+    required: true
+    description: |
+      Blender version to use
+      * <version> - exact version you need
+      * lts    - latest available LTS version
+      * latest - latest available version
+runs:
+  using: composite
+  steps:
+    - name: Preparing to the test execution
+      shell: bash
+      run: |
+        $GITHUB_ACTION_PATH/test_prepare.sh "${{ inputs.blender_version }}"
+        # Creating the docker container to store the data
+        docker create -v /srv --name blendnet-srv alpine /bin/true
+        mkdir -p scripts/addons
+        cp -a blendnet scripts/addons
+        docker cp scripts blendnet-srv:/srv
+        docker cp blender blendnet-srv:/srv
+        docker cp workspace blendnet-srv:/srv
+        mkdir -p results
+
+    - name: Run Addon test
+      shell: bash
+      run: $GITHUB_ACTION_PATH/test_execute_addon.sh "${{ inputs.blender_version }}" || exit 1
+
+    - name: Run API test
+      shell: bash
+      run: $GITHUB_ACTION_PATH/test_execute_api.sh "${{ inputs.blender_version }}" || exit 1
+
+    - name: Clean
+      shell: bash
+      run: |
+        docker rm -f blendnet-srv || true

.github/actions/blendnet-test-linux/test_execute_addon.sh

@@ -0,0 +1,56 @@
+#!/bin/sh -xe
+# Runs the actual test
+
+BLENDER_VERSION=$1
+[ "${BLENDER_VERSION}" != '' ] || exit 1
+
+ROOT=$(dirname "$0")/../../..
+
+# Run Agents & Manager containers
+"${ROOT}/.github/actions/blendnet-test-linux/test_run_agent.sh" addon 1 &
+"${ROOT}/.github/actions/blendnet-test-linux/test_run_agent.sh" addon 2 &
+"${ROOT}/.github/actions/blendnet-test-linux/test_run_manager.sh" addon &
+
+# Wait for blendnet-manager container
+while ! docker ps | grep blendnet-manager; do echo "waiting manager"; docker ps; sleep 1; done
+
+echo "Docker containers are running"
+docker ps
+
+results_dir=results/addon
+mkdir -p "${results_dir}"
+
+# Run the execution of BlendNet Addon test script
+docker run --name blendnet-executor -m 1G -i \
+    --link blendnet-manager:blendnet-manager-host \
+    --volumes-from blendnet-srv \
+    ubuntu:20.04 /srv/workspace/test_run_service.sh addon 2>&1 | tee "${results_dir}/addon.log" || true
+
+# Stop the Agents & Manager
+# TODO: when BN-64 is completed - stop Manager and Agents properly
+docker rm -f blendnet-manager || true
+docker rm -f blendnet-agent-1 || true
+docker rm -f blendnet-agent-2 || true
+
+# Check existing of the path in the log
+grep 'DATA: CI:' "${results_dir}/addon.log"
+
+# Copy the compose file to results
+remote_file="$(grep 'DATA: CI:' "${results_dir}/addon.log" | cut -d' ' -f3-)"
+docker cp "blendnet-executor:${remote_file}" "${results_dir}"
+docker rm -f blendnet-executor || true
+
+compose_file="${results_dir}/$(basename ${remote_file})"
+[ -f "${compose_file}" ] # Compose file exists
+ls -lh "${compose_file}"
+file "${compose_file}"
+file "${compose_file}" | grep -q 'PNG image data' # It's PNG format
+file "${compose_file}" | grep -q '1280 x 720' # Resolution is ok
+file "${compose_file}" | grep -q '8-bit/color RGB' # Color is encoded properly
+[ $(stat --format '%s' "${compose_file}") -gt $((540*1024)) ] # Compose PNG size > 540KB
+
+grep '^Traceback' "${results_dir}/addon.log" && exit 1 || echo "ok - no exceptions in ${results_dir}/addon.log"
+grep '^ERROR: ' "${results_dir}/addon.log" && exit 2 || echo "ok - no errors in ${results_dir}/addon.log"
+grep '^Fatal Python error: ' "${results_dir}/addon.log" && exit 3 || echo "ok - no python errors in ${results_dir}/addon.log"
+
+echo "Addon Compose images are received"

.github/actions/blendnet-test-linux/test_execute_api.sh

@@ -0,0 +1,124 @@
+#!/bin/sh -xe
+# Runs the actual test
+
+BLENDER_VERSION=$1
+[ "${BLENDER_VERSION}" != '' ] || exit 1
+
+ROOT=$(dirname "$0")/../../..
+
+# Run Agents & Manager containers
+"${ROOT}/.github/actions/blendnet-test-linux/test_run_agent.sh" api 1 &
+"${ROOT}/.github/actions/blendnet-test-linux/test_run_agent.sh" api 2 &
+"${ROOT}/.github/actions/blendnet-test-linux/test_run_manager.sh" api &
+
+# Wait for blendnet-manager container
+while ! docker ps | grep blendnet-manager; do echo "waiting manager"; docker ps; sleep 1; done
+
+docker run --name blendnet-executor -m 1G -i -d \
+    --link blendnet-manager:blendnet-manager-host \
+    --volumes-from blendnet-srv ubuntu:20.04 /bin/sleep 3600 || true
+
+# Wait for blendnet-executor container
+while ! docker ps | grep blendnet-executor; do echo "waiting executor"; docker ps; sleep 1; done
+
+# There is no direct access to the Manager port, so using it to run required commands
+# Install curl and unzip to the Manager
+docker exec blendnet-executor /bin/sh -c 'apt update; apt install -y curl unzip'
+echo "Docker containers are running"
+
+docker ps
+
+for retry in $(seq 1 10); do
+    sleep 5
+    echo "Check ${retry}"
+    docker exec blendnet-executor curl --user 'None:None' --insecure --max-time 5 --silent \
+        "https://blendnet-manager-host:8443/api/v1/status" || continue
+    echo "Looks like connected to blendnet-manager"
+    break
+done
+
+# Add the Agents to the Manager
+docker exec blendnet-executor curl --user 'None:None' --insecure --silent -X PUT \
+    "https://blendnet-manager-host:8443/api/v1/agent/agent-1/config" \
+    --data '{"address": "blendnet-agent-1-host", "port": 9443, "auth_user": "None", "auth_password": "None"}'
+docker exec blendnet-executor curl --user 'None:None' --insecure --silent -X PUT \
+    "https://blendnet-manager-host:8443/api/v1/agent/agent-2/config" \
+    --data '{"address": "blendnet-agent-2-host", "port": 9443, "auth_user": "None", "auth_password": "None"}'
+
+# Uploading the required task dependencies
+docker exec blendnet-executor /bin/sh -c '
+cd /srv/workspace/test-project
+for f in $(find . -type f -name "*0032*") tex/* test-project.blend; do
+    curl --user "None:None" --insecure \
+        --header "X-Checksum-Sha1:$(sha1sum "${f}" | cut -d " " -f 1)" \
+        --upload-file "${f}" "https://blendnet-manager-host:8443/api/v1/task/test-task-1/file/${f}"
+done
+'
+
+# Configure the task (render-ci and compose-ci uses 23 samples)
+docker exec blendnet-executor curl --user 'None:None' --insecure --silent -X PUT \
+    -d '{"samples": 23, "project": "test-project.blend", "frame": 32}' \
+    "https://blendnet-manager-host:8443/api/v1/task/test-task-1/config"
+
+# Run the task execution
+docker exec blendnet-executor curl --user 'None:None' --insecure --silent \
+    "https://blendnet-manager-host:8443/api/v1/task/test-task-1/run"
+
+results_dir=results/api
+mkdir -p "${results_dir}"
+
+# Watch the task execution and save render
+for retry in $(seq 1 50); do
+    sleep 5
+    echo "Check task render ${retry}"
+    r=$(docker exec blendnet-executor curl --user 'None:None' --insecure --silent \
+        "https://blendnet-manager-host:8443/api/v1/task/test-task-1/status")
+    [ "$(echo "${r}" | jq -r '.data.result.render')" != "null" ] || continue
+    docker exec blendnet-executor curl --user 'None:None' --insecure --output - --silent \
+        "https://blendnet-manager-host:8443/api/v1/task/test-task-1/status/result/render" > "${results_dir}/render.exr"
+    break
+done
+
+render_file="${results_dir}/render.exr"
+[ -f "${render_file}" ] # Render file exists
+ls -lh "${render_file}"
+file "${render_file}"
+file "${render_file}" | grep -q 'OpenEXR image data' # It's EXR format
+file "${render_file}" | grep -q 'compression: zip' # Compression is lossless
+if [ "${BLENDER_VERSION}" = '2.80' ]; then
+    [ $(stat --format '%s' "${render_file}") -gt $((2*1024*1024)) ] # Render EXR size > 2MB
+else
+    [ $(stat --format '%s' "${render_file}") -gt $((14*1024*1024)) ] # Render EXR size > 14MB
+fi
+
+# Watch the task execution and save compose
+fn='notexist'
+for retry in $(seq 1 10); do
+    sleep 5
+    echo "Check task compose ${retry}"
+    r=$(docker exec blendnet-executor curl --user 'None:None' --insecure --silent \
+        "https://blendnet-manager-host:8443/api/v1/task/test-task-1/status")
+    [ "$(echo "${r}" | jq -r '.data.result.compose')" != "null" ] || continue
+    fn=$(echo "${r}" | jq -r '.data.compose_filepath')
+    docker exec blendnet-executor curl --user 'None:None' --insecure --output - --silent \
+        "https://blendnet-manager-host:8443/api/v1/task/test-task-1/status/result/compose" > "${results_dir}/$(basename "${fn}")"
+    break
+done
+
+# Stop the Agents & Manager
+# TODO: when BN-64 is completed - stop Manager and Agents properly
+docker rm -f blendnet-executor || true
+docker rm -f blendnet-manager || true
+docker rm -f blendnet-agent-1 || true
+docker rm -f blendnet-agent-2 || true
+
+compose_file="${results_dir}/$(basename ${fn})"
+[ -f "${compose_file}" ] # Compose file exists
+ls -lh "${compose_file}"
+file "${compose_file}"
+file "${compose_file}" | grep -q 'PNG image data' # It's PNG format
+file "${compose_file}" | grep -q '1280 x 720' # Resolution is ok
+file "${compose_file}" | grep -q '8-bit/color RGB' # Color is encoded properly
+[ $(stat --format '%s' "${compose_file}") -gt $((540*1024)) ] # Compose PNG size > 540KB
+
+echo "Render & Compose images are received"

.github/actions/blendnet-test-linux/test_prepare.sh

@@ -0,0 +1,52 @@
+#!/bin/sh -xe
+# Prepares the necessary resources to run the tests
+
+BLENDER_VERSION=$1
+[ "${BLENDER_VERSION}" != '' ] || exit 1
+
+ROOT=$(dirname "$0")/../../..
+
+# Find the required blender version
+python3 "${ROOT}/BlendNet/list_blender_versions.py" 'lin' "${BLENDER_VERSION}" | tee /tmp/blender_versions.txt
+ver_line=$(grep '^DATA:' /tmp/blender_versions.txt | head -1)
+rm -f /tmp/blender_versions.txt
+version=$(echo "${ver_line}" | cut -d" " -f2)
+checksum=$(echo "${ver_line}" | cut -d" " -f3)
+url=$(echo "${ver_line}" | cut -d" " -f4-)
+
+[ "${version}" != '' ] || exit 2
+[ "${checksum}" != '' ] || exit 3
+[ "${url}" != '' ] || exit 4
+
+# Download the blender archive and unpack it
+out_archive=/tmp/$(basename "${url}")
+echo "${checksum} -" > sum.txt
+curl -fLs "${url}" | tee "${out_archive}" | sha256sum -c sum.txt
+rm -f sum.txt
+mkdir -p blender
+tar -C blender --strip-components=1 --checkpoint=10000 --checkpoint-action=echo='Unpacked %{r}T' -xf "${out_archive}"
+rm -f "${out_archive}"
+
+# Create the initial workspace
+mkdir -p workspace
+openssl req -x509 -nodes -newkey rsa:4096 \
+    -keyout workspace/server.key -out workspace/server.crt \
+    -days 365 -subj "/C=US/ST=N/L=N/O=N/OU=N/CN=blendet-service"
+# Required ca.crt for Manager
+cp workspace/server.crt workspace/ca.crt
+cp -a "${ROOT}/.github/actions/blendnet-test-linux/test_run_service.sh" workspace
+cp -a "${ROOT}/.github/scripts/test_script_addon.py" workspace
+
+# Download the test project
+testproject_url='https://github.com/state-of-the-art/BlendNet/wiki/files/blendnet-test-project'
+# (not perfect but simple & working for current CI)
+if [ "${BLENDER_VERSION}" = '2.80' ]; then
+    testproject_url="${testproject_url}-2.80"
+else
+    testproject_url="${testproject_url}-2.82"
+fi
+testproject_url="${testproject_url}-v0.3.zip"
+
+curl -fLo test-project.zip "${testproject_url}"
+unzip -d workspace test-project.zip
+mv workspace/blendnet-test-project* workspace/test-project

.github/actions/blendnet-test-linux/test_run_agent.sh

@@ -0,0 +1,16 @@
+#!/bin/sh -xe
+# Runs the agent in docker container
+
+TEST_NAME=$1
+AGENT_NUM=$2
+
+logfile="results/${TEST_NAME}/agent-${AGENT_NUM}.log"
+mkdir -p "$(dirname "${logfile}")"
+
+docker run --name blendnet-agent-${AGENT_NUM} -m 2G --rm -i \
+    --volumes-from blendnet-srv ubuntu:20.04 \
+    /srv/workspace/test_run_service.sh agent > "${logfile}" 2>&1 || true
+
+grep '^Traceback' "${logfile}" && exit 1 || echo "ok - no exceptions in ${logfile}"
+grep '^ERROR: ' "${logfile}" && exit 2 || echo "ok - no errors in ${logfile}"
+grep '^Fatal Python error: ' "${logfile}" && exit 3 || echo "ok - no python errors in ${logfile}"

.github/actions/blendnet-test-linux/test_run_manager.sh

@@ -0,0 +1,22 @@
+#!/bin/sh -xe
+# Runs the manager in docker container
+
+TEST_NAME=$1
+
+logfile="results/${TEST_NAME}/manager.log"
+mkdir -p "$(dirname "${logfile}")"
+
+# Wait for blendnet-agent-1 container
+while ! docker ps | grep blendnet-agent-1; do echo "waiting agent-1"; docker ps; sleep 1; done
+# Wait for blendnet-agent-2 container
+while ! docker ps | grep blendnet-agent-2; do echo "waiting agent-2"; docker ps; sleep 1; done
+
+docker run --name blendnet-manager -m 2G --rm -i \
+    --link blendnet-agent-1:blendnet-agent-1-host \
+    --link blendnet-agent-2:blendnet-agent-2-host \
+    --volumes-from blendnet-srv ubuntu:20.04 \
+    /srv/workspace/test_run_service.sh manager > "${logfile}" 2>&1 || true
+
+grep '^Traceback' "${logfile}" && exit 1 || echo "ok - no exceptions in ${logfile}"
+grep '^ERROR: ' "${logfile}" && exit 2 || echo "ok - no errors in ${logfile}"
+grep '^Fatal Python error: ' "${logfile}" && exit 3 || echo "ok - no python errors in ${logfile}"

.github/actions/blendnet-test-linux/test_run_service.sh

@@ -0,0 +1,22 @@
+#!/bin/sh -xe
+# Runs the required service (manager/agent/addon) in test env
+
+SVC=$1
+
+[ "${SVC}" = 'agent' -o "${SVC}" = 'manager' -o "${SVC}" = 'addon' ] || exit 1
+[ -d /srv/scripts/addons/blendnet ] || exit 2
+[ -d /srv/blender ] || exit 3
+[ -d /srv/workspace ] || exit 4
+
+cp -a /srv/workspace /workspace
+cd /workspace
+
+apt update
+apt install --no-install-recommends -y libxrender1 libxi6 libgl1
+
+if [ "${SVC}" = "addon" ]; then
+    export BLENDER_USER_SCRIPTS=/srv/scripts
+    /srv/blender/blender -b -noaudio test-project/test-project.blend -P /srv/workspace/test_script_addon.py
+else
+    /srv/blender/blender -b -noaudio -P /srv/scripts/addons/blendnet/${SVC}.py
+fi

.github/scripts/build.sh

@@ -0,0 +1,23 @@
+#!/bin/sh -xe
+# Show the version and pack the BlendNet distributive archive
+
+ROOT=$(dirname "$0")/../..
+
+# Get version from the addon init file
+VERSION=$(head -10 "${ROOT}/__init__.py" | grep -o 'version.: *(.*)' | tr ',' '.' | grep -o '[0-9]\|\.' | tr -d '\n')
+
+DEV_VER=''
+if [ "x$(head -10 "${ROOT}/__init__.py" | grep -o 'warning.: *.dev')" != 'x' ]; then
+    DEV_VER="-$(git -C "${ROOT}" rev-parse --short HEAD)"
+fi
+
+echo "INFO: BlendNet version: $VERSION$DEV_VER"
+if [ -d blendnet ]; then
+    mkdir -p results/dist
+
+    tar -cvzf results/dist/blendnet-${VERSION}${DEV_VER}.tar.gz --exclude='.*' --exclude='__pycache__' blendnet
+    zip -9 -r results/dist/blendnet-${VERSION}${DEV_VER}.zip blendnet --exclude '**/.*' '**/__pycache__/*'
+    echo "INFO: Created dist archives: $(ls results/dist | tr '\n' ', ')"
+else
+    echo 'WARN: Skip creating dist archives, no blendnet dir is here'
+fi