config.yml

version: 2
jobs:

  # If you want to deploy your container to Docker Hub, you must
  #  1. define the CONTAINER_NAME environment variable for the project 
  #  2. Also define DOCKER_USER and DOCKER_PASS
  #  3. Optionally add DOCKER_TAG to use a custom tag, otherwise will push
  #     to the commit id and "latest"
  setup:
    environment:
      - TZ: "/usr/share/zoneinfo/America/Los_Angeles"
    docker:
      - image: docker:18.01.0-ce-git
    steps:
      - run:
          name: Test if the user has defined the repo name 
          command: apk add --no-cache pigz python3

  build:
    environment:
      - TZ: "/usr/share/zoneinfo/America/Los_Angeles"
    docker:
      - image: docker:18.01.0-ce-git
    working_directory: /tmp/src
    steps:
      - run:
          name: Install parallel gzip and python3
          command: apk add --no-cache pigz python3
      - restore_cache:
          keys:
            - docker-v1-{{ .Branch }}-{{ epoch }}
          paths:
            - /tmp/cache/container.tar.gz

      - checkout
      - setup_remote_docker
      - run:
          name: Load Docker image layer cache
          no_output_timeout: 30m
          command: |
            docker info
            set +o pipefail
            if [ -f /tmp/cache/container.tar.gz ]; then
              pigz -d --stdout /tmp/cache/container.tar.gz | docker load
            fi
            docker images
      - run:
          name: Build Docker Image
          no_output_timeout: 60m
          command: |
            echo "Working directory is ${PWD}"
            # If not set, define CONTAINER_NAME
            if [ ! -n "${CONTAINER_NAME:-}" ]
                then
                    CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
            fi
            echo "Container name is ${CONTAINER_NAME}"
            # If not set, define DOCKER_TAG
            if [ ! -n "${DOCKER_TAG:-}" ]
                then
                    DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10)
            fi
            # Pull experiment Factory builder
            echo "1. Preparing experiment building environment"
            EXPERIMENT_FOLDER=$PWD/experiments
            if [ -d "${EXPERIMENT_FOLDER}" ];
            then
                cd ${EXPERIMENT_FOLDER}
                for e in $(ls $PWD)
                  do
                      # We add /data to indicate the file exists locally
                      EXPS="${EXPS} /data/${e}"
                  done
            else
                mkdir "${EXPERIMENT_FOLDER}"
                cd "${EXPERIMENT_FOLDER}"
                EXPS=""
            fi
            # Add experiments from experiments.txt
            if [ -f "../experiments.txt" ]
              then
                ADDITIONAL=$(cat ../experiments.txt);
                # These experiments are not expected to exist
                EXPS="${EXPS} ${ADDITIONAL}";            
              fi            
            echo "Experiments for install are: ${EXPS}"
            echo "2. Starting the Experiment Factory builder container"
            docker run -td --name expfactory-builder --rm=false --entrypoint bash vanessa/expfactory-builder 
            docker ps
            # Check if the directory is empty
            if [ $(ls  "${EXPERIMENT_FOLDER}/" | wc -l) != "0" ];
              then
                 echo "3. Copy experiments from host to container..."
                 docker cp ${EXPERIMENT_FOLDER}/* expfactory-builder:/data
              fi
            docker exec expfactory-builder /bin/bash /entrypoint.sh build ${EXPS} 
            echo "4. Copy files from Docker container to host..."
            docker cp expfactory-builder:/data/Dockerfile ${EXPERIMENT_FOLDER}/Dockerfile
            docker cp expfactory-builder:/data/startscript.sh ${EXPERIMENT_FOLDER}/startscript.sh
            echo "Stopping builder..."
            docker stop expfactory-builder
            docker rm expfactory-builder
            echo "5. Building experiment container..."  
            cat ${EXPERIMENT_FOLDER}/Dockerfile
            # Build experiment docker image
            e=1 && for i in {1..5}; do
              docker build \
                --cache-from=${CONTAINER_NAME} \
                --rm=false \
                -t ${CONTAINER_NAME}:${DOCKER_TAG} . \
              && e=0 && break || sleep 15
            done && [ "$e" -eq "0" ]
      - run:
          name: Docker Save
          no_output_timeout: 40m
          command: |
            # If not set, define CONTAINER_NAME
            if [ ! -n "${CONTAINER_NAME:-}" ]
                then
                    CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
            fi
            if [ ! -n "${DOCKER_TAG:-}" ]
                then
                    DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10)
            fi
            echo "Saving ${CONTAINER_NAME} to container.tar.gz"
            mkdir -p /tmp/cache
            docker save ${CONTAINER_NAME}:${DOCKER_TAG} \
              | pigz -2 -p 3 > /tmp/cache/container.tar.gz
      - persist_to_workspace:
          root: /tmp
          paths:
            - cache/container.tar.gz
            - src

  update_cache:
    machine:
      # Ubuntu 14.04 with Docker 17.10.0-ce
      image: circleci/classic:201711-01
    working_directory: /tmp/src
    steps:
      - attach_workspace:
          at: /tmp
      - save_cache:
         key: docker-v1-{{ .Branch }}-{{ epoch }}
         paths:
            - /tmp/cache/container.tar.gz

  verify_experiments:
    machine:
      image: circleci/classic:201711-01
    working_directory: /home/circleci/out/tests
    steps:
      - attach_workspace:
          at: /tmp
      - run:
          name: Load Docker image layer cache
          no_output_timeout: 30m
          command: |
            docker info
            set +o pipefail
            if [ -f /tmp/cache/container.tar.gz ]; then
              sudo apt update && sudo apt -y install pigz
              pigz -d --stdout /tmp/cache/container.tar.gz | docker load
              docker images
            fi
      - run:
          name: List Experiments
          no_output_timeout: 2h
          command: |
            # If not set, define CONTAINER_NAME
            if [ ! -n "${CONTAINER_NAME:-}" ]
                then
                    CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
            fi
            # If not set, define DOCKER_TAG
            if [ ! -n "${DOCKER_TAG:-}" ]
                then
                    DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10)
            fi
            echo "Container name set to ${CONTAINER_NAME}"
            echo "The experiments installed in this container are:"
            docker run -it --rm=false \
              ${CONTAINER_NAME}:${DOCKER_TAG} ls /scif/apps
      - store_test_results:
          path: /home/circleci/out/tests

  deploy:
    machine:
      image: circleci/classic:201711-01
    working_directory: /tmp/src
    steps:
      - attach_workspace:
          at: /tmp
      - run:
          name: Load Docker image layer cache
          no_output_timeout: 30m
          command: |
            echo "Working directory is ${PWD}"
            docker info
            set +o pipefail
            if [ -f /tmp/cache/container.tar.gz ]; then
              sudo apt update && sudo apt -y install pigz
              pigz -d --stdout /tmp/cache/container.tar.gz | docker load
              docker images
            fi
      - run:
          name: Upload Metadata or Result
          no_output_timeout: 40m
          command: |
            # If not set, define CONTAINER_NAME
            if [ ! -n "${CONTAINER_NAME:-}" ]
                then
                    CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
            fi
            if [ ! -n "${DOCKER_TAG:-}" ]
                then
                    DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10)
            fi
            echo "Here we could upload metadata, for example:"
            docker run -it ${CONTAINER_NAME}:${DOCKER_TAG} inspect
      - run:
          name: Deploy to Docker Hub
          no_output_timeout: 40m
          command: |
            echo "Looking for Docker deployment options"
            # If not set, define CONTAINER_NAME
            if [[ ! -n "${CONTAINER_NAME:-}" ]]
                then
                    CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
            fi
            if [ ! -n "${DOCKER_TAG:-}" ]
                then
                    DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10)
            fi
            echo "Container name set to ${CONTAINER_NAME}"
            if [[ -n "$DOCKER_PASS" ]]; then
                  docker login -u $DOCKER_USER -p $DOCKER_PASS
                  docker push ${CONTAINER_NAME}:${DOCKER_TAG}
                  docker tag ${CONTAINER_NAME}:${DOCKER_TAG} ${CONTAINER_NAME}:latest
                  docker push ${CONTAINER_NAME}:latest
            fi
workflows:
  version: 2
  build_test_deploy:
    jobs:
      - build:
          filters:
            tags:
              only: /.*/

      - update_cache:
          requires:
            - build
          filters:
            branches:
              ignore: /docs?\/.*/
            tags:
              only: /.*/

      - verify_experiments:
          requires:
            - build
          filters:
            branches:
              ignore: /docs?\/.*/
            tags:
              only: /.*/

      # This is where we upload the container to some final resting spot :)
      - deploy:
          requires:
            - build
            - verify_experiments
          filters:
            branches:
              only: master
            tags:
              only: /.*/