diff --git a/.github/workflows/bert-models-cicd.yaml b/.github/workflows/bert-models-cicd.yaml index 25a14a26..24bc5e73 100644 --- a/.github/workflows/bert-models-cicd.yaml +++ b/.github/workflows/bert-models-cicd.yaml @@ -50,41 +50,41 @@ jobs: # Note: when trying to run build_push-bert-base with other jobs below, an error is reported on line 90. # However, when run on its own it works fine. - # build_push-bert-base: - # name: "build/push bluebert-base container image" - # runs-on: ubuntu-latest - # env: - # IMAGE_NAME: bluebert-base - # DOCKERFILE: base.Dockerfile - # permissions: - # contents: 'read' - # id-token: 'write' - # steps: - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - # - id: 'auth' - # uses: 'google-github-actions/auth@v2' - # with: - # project_id: ${{ secrets.GCE_PROJECT }} - # service_account: ${{ secrets.SERVICE_ACCOUNT }} - # credentials_json: ${{ secrets.SERVICE_ACCOUNT_KEY }} - # - name: 'Set up Cloud SDK' - # uses: 'google-github-actions/setup-gcloud@v2' - # with: - # version: '>= 363.0.0' - # - name: Setup Python - # uses: actions/setup-python@v2 - # with: - # python-version: 3.7 - # - name: Configure Docker Authentication - # run: gcloud --quiet auth configure-docker - # - name: Set MODEL_VERSION env - # run: echo "MODEL_VERSION=$(grep 'BASE' MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV - # - name: Build Docker Image - # run: docker build --tag "gcr.io/$PROJECT_ID/$IMAGE_NAME:$MODEL_VERSION" -f ${{ env.DOCKERFILE }} . - # - name: Publish Docker Image to Google Container Registry - # run: docker push "gcr.io/$PROJECT_ID/$IMAGE_NAME:$MODEL_VERSION" + build_push-bert-base: + name: "build/push bluebert-base container image" + runs-on: ubuntu-latest + env: + IMAGE_NAME: bluebert-base + DOCKERFILE: base.Dockerfile + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: 'auth' + uses: 'google-github-actions/auth@v2' + with: + project_id: ${{ secrets.GCE_PROJECT }} + service_account: ${{ secrets.SERVICE_ACCOUNT }} + credentials_json: ${{ secrets.SERVICE_ACCOUNT_KEY }} + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Configure Docker Authentication + run: gcloud --quiet auth configure-docker + - name: Set MODEL_VERSION env + run: echo "MODEL_VERSION=$(grep 'BASE' MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV + - name: Build Docker Image + run: docker build --tag "gcr.io/$PROJECT_ID/$IMAGE_NAME:$MODEL_VERSION" -f ${{ env.DOCKERFILE }} . + - name: Publish Docker Image to Google Container Registry + run: docker push "gcr.io/$PROJECT_ID/$IMAGE_NAME:$MODEL_VERSION" # # ----------------------------------------------------------------------------------- # # # ----------------------------------------------------------------------------------- # @@ -2397,124 +2397,124 @@ jobs: # # # ----------------------------------------------------------------------------------- # # # # ----------------------------------------------------------------------------------- # - build_push_train-craft_100-train: - name: "build/push/train craft_100 model" - # needs: "build_push-bert-base" - runs-on: ubuntu-latest - permissions: - contents: "read" - id-token: "write" - env: - TASK_NAME: craft_100 - DOCKERFILE: train.Dockerfile - ENTRYPOINT_FILE: scripts/train.entrypoint.sh - BASE_DOCKERFILE: base.Dockerfile - DATA_FILE: data/craft_100/data.tsv - MODEL_VERSION_KEY: CRAFT_100 - AI_PLATFORM_JOB_NAME: "craft_100_train_${{ github.run_number }}" - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v0 - with: - workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER_LOCATION }} - service_account: "${{ secrets.SERVICE_ACCOUNT }}@${{ secrets.GCE_PROJECT }}.iam.gserviceaccount.com" - - - name: Configure Docker Authentication - run: | - gcloud --quiet auth configure-docker + # build_push_train-craft_100-train: + # name: "build/push/train craft_100 model" + # # needs: "build_push-bert-base" + # runs-on: ubuntu-latest + # permissions: + # contents: "read" + # id-token: "write" + # env: + # TASK_NAME: craft_100 + # DOCKERFILE: train.Dockerfile + # ENTRYPOINT_FILE: scripts/train.entrypoint.sh + # BASE_DOCKERFILE: base.Dockerfile + # DATA_FILE: data/craft_100/data.tsv + # MODEL_VERSION_KEY: CRAFT_100 + # AI_PLATFORM_JOB_NAME: "craft_100_train_${{ github.run_number }}" + # steps: + # - name: Checkout repository + # uses: actions/checkout@v2 + + # - name: Authenticate to Google Cloud + # uses: google-github-actions/auth@v0 + # with: + # workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER_LOCATION }} + # service_account: "${{ secrets.SERVICE_ACCOUNT }}@${{ secrets.GCE_PROJECT }}.iam.gserviceaccount.com" + + # - name: Configure Docker Authentication + # run: | + # gcloud --quiet auth configure-docker - - name: Set MODEL_VERSION env - run: | - echo "MODEL_VERSION=$(grep ${{ env.MODEL_VERSION_KEY }} MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV + # - name: Set MODEL_VERSION env + # run: | + # echo "MODEL_VERSION=$(grep ${{ env.MODEL_VERSION_KEY }} MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV - - name: Set BASE_VERSION env - run: | - echo "BASE_VERSION=$(grep 'BASE' MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV + # - name: Set BASE_VERSION env + # run: | + # echo "BASE_VERSION=$(grep 'BASE' MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV - - name: Build Docker Image - run: | - docker build --build-arg "PROJECT_ID=$PROJECT_ID" \ - --build-arg "TASK_NAME=$TASK_NAME" \ - --build-arg "BASE_VERSION=$BASE_VERSION" \ - --build-arg "TUNED_MODEL_VERSION=$MODEL_VERSION" \ - --tag "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" \ - -f ${{ env.DOCKERFILE }} . + # - name: Build Docker Image + # run: | + # docker build --build-arg "PROJECT_ID=$PROJECT_ID" \ + # --build-arg "TASK_NAME=$TASK_NAME" \ + # --build-arg "BASE_VERSION=$BASE_VERSION" \ + # --build-arg "TUNED_MODEL_VERSION=$MODEL_VERSION" \ + # --tag "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" \ + # -f ${{ env.DOCKERFILE }} . - - name: Publish Docker Image to Google Container Registry - run: | - docker push "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" + # - name: Publish Docker Image to Google Container Registry + # run: | + # docker push "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" - - name: Submit a training job to AI Plaform to train and cache the model - run: | - gcloud ai-platform jobs submit training ${{ env.AI_PLATFORM_JOB_NAME }} \ - --scale-tier basic_gpu --region "$GCE_REGION" \ - --master-image-uri "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" \ - -- \ - NO_ARG \ - "gs://$MODEL_STORAGE_BUCKET" + # - name: Submit a training job to AI Plaform to train and cache the model + # run: | + # gcloud ai-platform jobs submit training ${{ env.AI_PLATFORM_JOB_NAME }} \ + # --scale-tier basic_gpu --region "$GCE_REGION" \ + # --master-image-uri "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" \ + # -- \ + # NO_ARG \ + # "gs://$MODEL_STORAGE_BUCKET" - - name: Monitor the training job - run: | - sh -c ".github/workflows/monitor-ai-platform-job.sh ${{ env.AI_PLATFORM_JOB_NAME }}" - timeout-minutes: 180 + # - name: Monitor the training job + # run: | + # sh -c ".github/workflows/monitor-ai-platform-job.sh ${{ env.AI_PLATFORM_JOB_NAME }}" + # timeout-minutes: 180 - # # # ----------------------------------------------------------------------------------- # - # # # BUILD/PUSH CRAFT_RELATIONS INFLUENCE CLASSIFICATION CONTAINER IMAGE # - # # # ----------------------------------------------------------------------------------- # + # # # # ----------------------------------------------------------------------------------- # + # # # # BUILD/PUSH CRAFT_RELATIONS INFLUENCE CLASSIFICATION CONTAINER IMAGE # + # # # # ----------------------------------------------------------------------------------- # - build_push-craft_100-predict: - name: "build/push craft_100 predict container" - needs: "build_push_train-craft_100-train" - runs-on: ubuntu-latest - permissions: - contents: "read" - id-token: "write" - env: - TASK_NAME: craft_100 - DOCKERFILE: predict.Dockerfile - TRAIN_DOCKERFILE: train.Dockerfile - TRAIN_ENTRYPOINT_FILE: scripts/train.entrypoint.sh - ENTRYPOINT_FILE: scripts/predict.entrypoint.sh - BASE_DOCKERFILE: base.Dockerfile - MODEL_VERSION_KEY: CRAFT_100 - CLASSIFICATION_LABELS: "label1 label2 false" - DATA_FILE: data/craft_100/data.tsv - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v0 - with: - workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER_LOCATION }} - service_account: "${{ secrets.SERVICE_ACCOUNT }}@${{ secrets.GCE_PROJECT }}.iam.gserviceaccount.com" + # build_push-craft_100-predict: + # name: "build/push craft_100 predict container" + # needs: "build_push_train-craft_100-train" + # runs-on: ubuntu-latest + # permissions: + # contents: "read" + # id-token: "write" + # env: + # TASK_NAME: craft_100 + # DOCKERFILE: predict.Dockerfile + # TRAIN_DOCKERFILE: train.Dockerfile + # TRAIN_ENTRYPOINT_FILE: scripts/train.entrypoint.sh + # ENTRYPOINT_FILE: scripts/predict.entrypoint.sh + # BASE_DOCKERFILE: base.Dockerfile + # MODEL_VERSION_KEY: CRAFT_100 + # CLASSIFICATION_LABELS: "label1 label2 false" + # DATA_FILE: data/craft_100/data.tsv + # steps: + # - name: Checkout repository + # uses: actions/checkout@v2 + + # - name: Authenticate to Google Cloud + # uses: google-github-actions/auth@v0 + # with: + # workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER_LOCATION }} + # service_account: "${{ secrets.SERVICE_ACCOUNT }}@${{ secrets.GCE_PROJECT }}.iam.gserviceaccount.com" - - name: Configure Docker Authentication - run: | - gcloud --quiet auth configure-docker + # - name: Configure Docker Authentication + # run: | + # gcloud --quiet auth configure-docker - - name: Set MODEL_VERSION env - run: | - echo "MODEL_VERSION=$(grep ${{ env.MODEL_VERSION_KEY }} MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV + # - name: Set MODEL_VERSION env + # run: | + # echo "MODEL_VERSION=$(grep ${{ env.MODEL_VERSION_KEY }} MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV - - name: Set BASE_VERSION env - run: | - echo "BASE_VERSION=$(grep 'BASE' MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV + # - name: Set BASE_VERSION env + # run: | + # echo "BASE_VERSION=$(grep 'BASE' MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV - - name: Build Docker Image - run: | - docker build --build-arg "PROJECT_ID=$PROJECT_ID" \ - --build-arg "TASK_NAME=$TASK_NAME" \ - --build-arg "BASE_VERSION=$BASE_VERSION" \ - --build-arg "TUNED_MODEL_VERSION=$MODEL_VERSION" \ - --build-arg "MODEL_STORAGE_BUCKET=$MODEL_STORAGE_BUCKET" \ - --build-arg "CLASSIFICATION_LABELS=${{ env.CLASSIFICATION_LABELS }}" \ - --tag "gcr.io/$PROJECT_ID/$TASK_NAME-predict:$MODEL_VERSION" \ - -f ${{ env.DOCKERFILE }} . + # - name: Build Docker Image + # run: | + # docker build --build-arg "PROJECT_ID=$PROJECT_ID" \ + # --build-arg "TASK_NAME=$TASK_NAME" \ + # --build-arg "BASE_VERSION=$BASE_VERSION" \ + # --build-arg "TUNED_MODEL_VERSION=$MODEL_VERSION" \ + # --build-arg "MODEL_STORAGE_BUCKET=$MODEL_STORAGE_BUCKET" \ + # --build-arg "CLASSIFICATION_LABELS=${{ env.CLASSIFICATION_LABELS }}" \ + # --tag "gcr.io/$PROJECT_ID/$TASK_NAME-predict:$MODEL_VERSION" \ + # -f ${{ env.DOCKERFILE }} . - - name: Publish Docker Image to Google Container Registry - run: | - docker push "gcr.io/$PROJECT_ID/$TASK_NAME-predict:$MODEL_VERSION" \ No newline at end of file + # - name: Publish Docker Image to Google Container Registry + # run: | + # docker push "gcr.io/$PROJECT_ID/$TASK_NAME-predict:$MODEL_VERSION" \ No newline at end of file diff --git a/MODEL_VERSIONS b/MODEL_VERSIONS index 41ad5e9f..4451c750 100644 --- a/MODEL_VERSIONS +++ b/MODEL_VERSIONS @@ -1,4 +1,4 @@ -BASE=0.7 +BASE=0.8 BL_CHEMICAL_TO_DISEASE_OR_PHENOTYPIC_FEATURE=0.1 BL_CHEMICAL_TO_GENE=0.1 BL_DISEASE_TO_PHENOTYPIC_FEATURE=0.1 diff --git a/base.Dockerfile b/base.Dockerfile index e3af4875..42406770 100644 --- a/base.Dockerfile +++ b/base.Dockerfile @@ -1,4 +1,4 @@ -FROM tensorflow/tensorflow:1.15.4-gpu +FROM tensorflow/tensorflow:1.15.5-gpu-py3 RUN \ # Update nvidia GPG key - from: https://github.com/NVIDIA/nvidia-docker/issues/1631 @@ -15,13 +15,8 @@ RUN apt-get update && apt-get install -y \ less \ vim \ wget \ - python3.8 \ && rm -rf /var/lib/apt/lists/* -RUN rm /usr/bin/python3 && \ - ln -s python3.8 /usr/bin/python3 && \ - python3 -V - # set up directories that will be used by this container (and its children) RUN mkdir -p /home/dev/models/baseline && \ mkdir -p /home/dev/output && \ @@ -32,10 +27,10 @@ WORKDIR /home/dev # Installs google cloud sdk, this allows use of gsutil # from: https://cloud.google.com/ai-platform/training/docs/custom-containers-training RUN wget -nv \ - https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz && \ + https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-422.0.0-linux-x86_64.tar.gz && \ mkdir /root/tools && \ - tar xvzf google-cloud-sdk.tar.gz -C /root/tools && \ - rm google-cloud-sdk.tar.gz && \ + tar xvzf google-cloud-cli-422.0.0-linux-x86_64.tar.gz -C /root/tools && \ + rm google-cloud-cli-422.0.0-linux-x86_64.tar.gz && \ /root/tools/google-cloud-sdk/install.sh --usage-reporting=false \ --path-update=false --bash-completion=false \ --disable-installation-options && \