ssh-experiment

#!/bin/bash

INFO="
Sync a git repository to a remote server, execute a command
on the remote copy, and get results as a .tgz archive.

Usage:
    ssh-experiment <server> <command> [ ... <arguments>... ]
"


SSH_OPTS=()
while [ "x${1::1}" = x- ]; do
    SSH_OPTS+=("$1")
    shift
done

for opt in "${SSH_OPTS[@]}"; do
    echo "'$opt'"
done

SERVER="$1"
shift

if [ "x$SERVER" = x -o "x$1" = x ]; then
    echo "$INFO" 1>&2
    exit 1
fi

GIT_ROOT="$(git rev-parse --show-toplevel)" || exit 1
GIT_NAME=$(basename "$GIT_ROOT") || exit 1
EXEC_DIR="$(git rev-parse --show-prefix)" || exit 1
GIT_HEAD="$(git rev-parse --short HEAD)" || exit 1
GIT_STATUS="$(git status --porcelain)" || exit 1
CONTENT="$(find "${GIT_ROOT}" -mindepth 1 -printf '%P\n')" || exit 1
IGNORED="$(cd "${GIT_ROOT}" && git ls-files --others --ignored \
    --exclude-standard --directory)" || exit 1

EXP_NAME="$(date '+%Y-%m-%d_%H-%M-%S')" || exit 1
EXP_NAME="${EXP_NAME}_${GIT_NAME}_${GIT_HEAD}"

if [ "x$GIT_STATUS" != x ]; then
    echo "This repository has uncommitted changes:"
    echo "$GIT_STATUS"
    echo -n "Please name the experiment: "
    read USER_NAME
    USER_NAME=$(echo $USER_NAME)
    EXP_NAME="${EXP_NAME}_${USER_NAME// /_}"
fi

RUN_SCRIPT="
echo \"$@\" &>\"$EXP_NAME/command\"
( cd \"$EXP_NAME/$GIT_NAME/$EXEC_DIR\" && $@ ) &>\"$EXP_NAME/output\"
echo \$? >\"$EXP_NAME/exit_code\"
( cd \"$EXP_NAME\" && tar -c command output exit_code \"$GIT_NAME\" ) | gzip >\"$EXP_NAME/results.tmp\"
if [ \"x\${PIPESTATUS[0]}\${PIPESTATUS[1]}\" = x00 ]; then
    mv \"$EXP_NAME/results.tmp\" \"$EXP_NAME.tgz\"
    rm -rf \"$EXP_NAME\" || true
else
    rm \"$EXP_NAME/results.tmp\" || true
fi
"

echo "Sending experiment '$EXP_NAME' to '$SERVER'..." 1>&2

while ! rsync "${SSH_OPTS[@]}" \
        -rlxt \
        --omit-dir-times \
        --progress \
        --exclude="/$GIT_NAME/.git" \
        --exclude-from=<(echo "/ignored" && \
            while read f; do \
                [ "x$f" = x ] || echo "/${GIT_NAME}/${f%/}"; \
            done <<< "$IGNORED") \
        --exclude='__*.tgz' \
        "$GIT_ROOT" \
        "$SERVER:~/$EXP_NAME" </dev/null 1>&2; do
    echo 1>&2
    echo "RSYNC FAILED    (retry in 60s...)" 1>&2
    echo 1>&2
    sleep 60 || exit 1
done || exit 1

echo "Starting experiment '$EXP_NAME' on '$SERVER'..." 1>&2

ssh "${SSH_OPTS[@]}" \
    "$SERVER" \
    "nohup bash -c '$RUN_SCRIPT' &>/dev/null </dev/null &" || exit 1

echo "Experiment started!" 1>&2
START_TIME=$(date +'%s')

ssh "${SSH_OPTS[@]}" \
    "$SERVER" \
    "tail --follow=name \"$EXP_NAME/output\" 2>/dev/null" || true

echo "Waiting for results..." 1>&2

while sleep 2; do
    STATUS=$(ssh "${SSH_OPTS[@]}" "$SERVER" \
        "if [ -e \"$EXP_NAME.tgz\" ]; then echo OK; fi")
    if [ "x$STATUS" = xOK ]; then
        break
    fi
done

TOTAL_TIME=$(($(date +'%s') - $START_TIME))
TOTAL_MIN=$(($TOTAL_TIME / 60))
TOTAL_SEC=$(printf '%02d' $(($TOTAL_TIME % 60)))

echo "Downloading results (after ${TOTAL_MIN}m ${TOTAL_SEC}s)..." 1>&2

while ! scp "${SSH_OPTS[@]}" \
        "$SERVER:${EXP_NAME}.tgz" \
        "__${EXP_NAME}.tgz"; do
    echo 1>&2
    echo "SCP FAILED    (retry in 60s...)" 1>&2
    echo 1>&2
    sleep 60 || exit 1
done

play -q -n synth 0.1 sin C5 || true