diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3053ba76..10eceabd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -3,46 +3,97 @@ name: Build Data Image and Run End-to-end Tests on: push jobs: - e2e: + check-for-image: # Run in Sage repo on develop or main branches # and on all branches in user-owned forks if: ${{ github.ref_name == 'develop' || github.ref_name == 'main' || github.actor == github.repository_owner }} - timeout-minutes: 60 + timeout-minutes: 5 runs-on: ubuntu-20.04 + permissions: + packages: read environment: e2e env: - CI: true - LOCAL_DATA_PATH: './local/data' - LOCAL_TEAM_IMAGES_DIR: './local/data/team_images' - LOCAL_COLLECTIONS_PATH: './scripts/collections.csv' - LOCAL_IMPORT_SCRIPT_PATH: './docker/import-data.sh' - LOCAL_CREATE_INDEXES_PATH: './scripts/mongo-create-Indexes.js' - DOCKER_COMPOSE_PATH: './docker/docker-compose.yml' + ORG_NAME: 'Sage-Bionetworks' + NAMESPACE: ${{ github.repository_owner }} + DATA_IMAGE_NAME: '${{ vars.DB_NAME }}-data' + outputs: + DATA_MANIFEST_ID: ${{ steps.get-ids-from-package-json.outputs.DATA_MANIFEST_ID }} + DATA_VERSION: ${{ steps.get-ids-from-package-json.outputs.DATA_VERSION }} + DATA_IMAGE_PATH: ${{ steps.get-ids-from-package-json.outputs.DATA_IMAGE_PATH }} + IMAGE_EXISTS: ${{ steps.check-container-repo.outputs.IMAGE_EXISTS }} steps: - uses: actions/checkout@v4 - name: get data image tag, data manifest id, and data version + id: get-ids-from-package-json env: PACKAGE_JSON_PATH: 'package.json' - DATA_IMAGE_NAME: '${{ vars.DB_NAME }}/data' run: | DATA_MANIFEST_ID=$(jq -r '."data-file"' "${{ env.PACKAGE_JSON_PATH }}") - echo "DATA_MANIFEST_ID=${DATA_MANIFEST_ID}" >> "$GITHUB_ENV" + echo "DATA_MANIFEST_ID=${DATA_MANIFEST_ID}" >> "$GITHUB_OUTPUT" DATA_VERSION=$(jq -r '."data-version"' "${{ env.PACKAGE_JSON_PATH }}") - echo "DATA_VERSION=${DATA_VERSION}" >> "$GITHUB_ENV" + echo "DATA_VERSION=${DATA_VERSION}" >> "$GITHUB_OUTPUT" + + DATA_IMAGE_TAG="${DATA_MANIFEST_ID}.${DATA_VERSION}" + echo "DATA_IMAGE_TAG=${DATA_IMAGE_TAG}" >> "$GITHUB_OUTPUT" - DATA_IMAGE_TAG="${{ env.DATA_IMAGE_NAME }}:${DATA_MANIFEST_ID}-v${DATA_VERSION}" - echo "DATA_IMAGE_TAG=${DATA_IMAGE_TAG}" >> "$GITHUB_ENV" + DATA_IMAGE_PATH="ghcr.io/${{ env.NAMESPACE }}/${{ env.DATA_IMAGE_NAME }}:${DATA_IMAGE_TAG}" + echo "DATA_IMAGE_PATH=${DATA_IMAGE_PATH}" >> "$GITHUB_OUTPUT" + + - name: login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - # TODO - check remote repository - name: check whether image exists - id: image-exists + id: check-container-repo + env: + # https://docs.github.com/en/rest/packages/packages?apiVersion=2022-11-28#list-package-versions-for-a-package-owned-by-an-organization + ORG_PATH: '/orgs/${{ env.ORG_NAME }}/packages/container/${{ env.DATA_IMAGE_NAME }}/versions' + # https://docs.github.com/en/rest/packages/packages?apiVersion=2022-11-28#list-package-versions-for-a-package-owned-by-the-authenticated-user + USER_PATH: '/user/packages/container/${{ env.DATA_IMAGE_NAME }}/versions' + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DATA_IMAGE_TAG: ${{ steps.get-ids-from-package-json.outputs.DATA_IMAGE_TAG }} run: | - echo "==> data image tag: ${{ env.DATA_IMAGE_TAG }}" - echo "IMAGE_EXISTS=false" >> "$GITHUB_ENV" + GH_PATH=$( [[ "${{ env.NAMESPACE }}" == "${{ env.ORG_NAME }}" ]] && echo "${{ env.ORG_PATH }}" || echo "${{ env.USER_PATH }}" ) + + # allow error to handle case where gh throws an error when package has not yet been created + set +e + IMAGE_VERSIONS=$(gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "${GH_PATH}") + set -e + + PACKAGE_NOT_FOUND=$(echo "${IMAGE_VERSIONS}" | jq '.message? == "Package not found."') + if [[ "${PACKAGE_NOT_FOUND}" == "true" ]]; then + IMAGE_EXISTS="false" + else + IMAGE_EXISTS=$(echo "${IMAGE_VERSIONS}" | jq '[.[] | select(any(.metadata.container.tags[]; . == "${{ env.DATA_IMAGE_TAG }}"))] | length > 0') + fi + + echo "IMAGE_EXISTS=${IMAGE_EXISTS}" >> "$GITHUB_OUTPUT" + + build-image: + needs: check-for-image + if: ${{ needs.check-for-image.result == 'success' && needs.check-for-image.outputs.IMAGE_EXISTS == 'false' }} + timeout-minutes: 60 + runs-on: ubuntu-20.04 + permissions: + packages: write + environment: e2e + env: + LOCAL_DATA_PATH: './local/data' + LOCAL_TEAM_IMAGES_DIR: './local/data/team_images' + LOCAL_COLLECTIONS_PATH: './scripts/collections.csv' + LOCAL_IMPORT_SCRIPT_PATH: './docker/import-data.sh' + LOCAL_CREATE_INDEXES_PATH: './scripts/mongo-create-Indexes.js' + DATA_VERSION: ${{ needs.check-for-image.outputs.DATA_VERSION }} + DATA_MANIFEST_ID: ${{ needs.check-for-image.outputs.DATA_MANIFEST_ID }} + DATA_IMAGE_PATH: ${{ needs.check-for-image.outputs.DATA_IMAGE_PATH }} + steps: + - uses: actions/checkout@v4 - # TODO - move into job which conditionally runs based on IMAGE_EXISTS # https://github.com/actions/setup-python/issues/807 - name: create placeholder requirements.txt file run: touch ./requirements.txt @@ -81,16 +132,53 @@ jobs: DOCKERFILE_PATH: './docker/Dockerfile' run: | docker build \ - -t "${{ env.DATA_IMAGE_TAG }}" \ + -t "${{ env.DATA_IMAGE_PATH }}" \ -f "${{ env.DOCKERFILE_PATH }}" \ + --build-arg SOURCE_REPO="https://github.com/${{ github.repository }}" \ + --build-arg DESCRIPTION="${GITHUB_SHA::7}" \ --build-arg LOCAL_DATA_PATH="${{ env.LOCAL_DATA_PATH }}" \ --build-arg LOCAL_COLLECTIONS_PATH="${{ env.LOCAL_COLLECTIONS_PATH }}" \ --build-arg LOCAL_IMPORT_SCRIPT_PATH="${{ env.LOCAL_IMPORT_SCRIPT_PATH }}" \ --build-arg LOCAL_CREATE_INDEXES_PATH="${{ env.LOCAL_CREATE_INDEXES_PATH }}" \ . - # TODO - push image to remote repository - # TODO - move into job that pulls image from remote repository + - name: login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Note: if a new package is created using this workflow, then the package + # visibility will default to the source repository visibility. For example, + # a package created by a workflow in a public repo will also be public. + # Publicly visible container packages are anonymously downloadable. + - name: push to GitHub Container Registry + run: docker push "${{ env.DATA_IMAGE_PATH }}" + + run-e2e-tests: + needs: + - check-for-image + - build-image + # always() prevents skipping this job when build-image was skipped + # see https://github.com/actions/runner/issues/491 + if: ${{ always() && needs.check-for-image.result == 'success' && (needs.build-image.result == 'success' || needs.build-image.result == 'skipped')}} + timeout-minutes: 60 + runs-on: ubuntu-20.04 + permissions: + packages: read + environment: e2e + env: + CI: true + DOCKER_COMPOSE_PATH: './docker/docker-compose.yml' + DATA_IMAGE_PATH: ${{ needs.check-for-image.outputs.DATA_IMAGE_PATH }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASS: ${{ secrets.DB_PASS }} + DB_PORT: ${{ vars.DB_PORT }} + DB_NAME: ${{ vars.DB_NAME }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: 16 @@ -101,12 +189,14 @@ jobs: - name: install Playwright Browsers run: npx playwright install --with-deps + - name: login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: start database, server, and app - env: - DB_USER: ${{ secrets.DB_USER }} - DB_PASS: ${{ secrets.DB_PASS }} - DB_PORT: ${{ vars.DB_PORT }} - DB_NAME: ${{ vars.DB_NAME }} run: | echo "==> start database and data containers" docker-compose -f "${{ env.DOCKER_COMPOSE_PATH }}" up -d @@ -123,11 +213,6 @@ jobs: run: npx playwright test - name: clean up database, server, and app - env: - DB_USER: ${{ secrets.DB_USER }} - DB_PASS: ${{ secrets.DB_PASS }} - DB_PORT: ${{ vars.DB_PORT }} - DB_NAME: ${{ vars.DB_NAME }} run: | echo "==> stop app" pid=$(lsof -i :3000 -t) && kill ${pid} diff --git a/docker/Dockerfile b/docker/Dockerfile index b25d7970..41ba0461 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,9 @@ FROM mongo:latest +# GHCR labels +ARG SOURCE_REPO="https://github.com/Sage-Bionetworks/agora" +ARG DESCRIPTION="" + # Paths on local machine - may be overwritten at build time ARG LOCAL_DATA_PATH="./local/data" ARG LOCAL_COLLECTIONS_PATH="./scripts/collections.csv" @@ -23,5 +27,9 @@ COPY "${LOCAL_CREATE_INDEXES_PATH}" "${CREATE_INDEXES_FILE}" RUN chmod +x "${IMPORT_DATA_FILE}" RUN chmod +x "${CREATE_INDEXES_FILE}" +# Add GHCR labels +LABEL org.opencontainers.image.source="${SOURCE_REPO}" +LABEL org.opencontainers.image.description="${DESCRIPTION}" + # On container startup, import data into mongodb CMD ["/bin/bash", "import-data.sh"] \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 51d8c00d..a1c1afc2 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,7 +10,7 @@ services: - MONGO_INITDB_ROOT_PASSWORD=${DB_PASS} - MONGO_INITDB_DATABASE=${DB_NAME} mongo-seed: - image: ${DATA_IMAGE_TAG} + image: ${DATA_IMAGE_PATH} depends_on: - mongodb environment: