diff --git a/.github/workflows/bert-models-cicd.yaml b/.github/workflows/bert-models-cicd.yaml index b0f4a497..edb98382 100644 --- a/.github/workflows/bert-models-cicd.yaml +++ b/.github/workflows/bert-models-cicd.yaml @@ -341,134 +341,134 @@ jobs: # docker push "gcr.io/$PROJECT_ID/$TASK_NAME-predict:$MODEL_VERSION" - # ----------------------------------------------------------------------------------- # - # ----------------------------------------------------------------------------------- # - # ----------------------------------------------------------------------------------- # - # TRAIN BL_CHEMICAL_TO_DISEASE_OR_PHENOTYPIC_FEATURE MODEL # - # ----------------------------------------------------------------------------------- # - # ----------------------------------------------------------------------------------- # - # ----------------------------------------------------------------------------------- # - - build_push_train-bl_chemical_to_disease_or_phenotypic_feature-train: - name: "build/push/train bl_chemical_to_disease_or_phenotypic_feature model" - # needs: "build_push-bert-base" - runs-on: ubuntu-latest - env: - TASK_NAME: bl_chemical_to_disease_or_phenotypic_feature - DOCKERFILE: train.Dockerfile - ENTRYPOINT_FILE: scripts/train.entrypoint.sh - BASE_DOCKERFILE: base.Dockerfile - DATA_FILE: data/bl_chemical_to_disease_or_phenotypic_feature/data.tsv - MODEL_VERSION_KEY: BL_CHEMICAL_TO_DISEASE_OR_PHENOTYPIC_FEATURE - AI_PLATFORM_JOB_NAME: "bl_chemical_to_disease_or_phenotypic_feature_train_${{ github.run_number }}" - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - # - name: Get changed files - # id: changed-files - # uses: tj-actions/changed-files@v7 - - # - name: List all modified files - # run: | - # for file in "${{ steps.changed-files.outputs.all_modified_files }}"; do - # echo "$file was modified" - # done - - # - name: Run step when a file changes - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - # run: | - # echo "A modified file has triggered a rebuild of the ${{ env.TASK_NAME }} model." - - - name: Set up Cloud SDK - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - uses: google-github-actions/setup-gcloud@main - with: - project_id: ${{ secrets.GCE_PROJECT }} - service_account_key: ${{ secrets.GCE_SA_KEY }} - version: '290.0.1' - - - name: Configure Docker Authentication - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - run: | - gcloud --quiet auth configure-docker - - - name: Set MODEL_VERSION env - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - run: | - echo "MODEL_VERSION=$(grep ${{ env.MODEL_VERSION_KEY }} MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV - - - name: Set BASE_VERSION env - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - run: | - echo "BASE_VERSION=$(grep 'BASE' MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV - - - name: Build Docker Image - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - run: | - docker build --build-arg "PROJECT_ID=$PROJECT_ID" \ - --build-arg "TASK_NAME=$TASK_NAME" \ - --build-arg "BASE_VERSION=$BASE_VERSION" \ - --build-arg "TUNED_MODEL_VERSION=$MODEL_VERSION" \ - --tag "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" \ - -f ${{ env.DOCKERFILE }} . - - - name: Publish Docker Image to Google Container Registry - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - run: | - docker push "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" - - - name: Submit a training job to AI Plaform to train and cache the model - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - run: | - gcloud ai-platform jobs submit training ${{ env.AI_PLATFORM_JOB_NAME }} \ - --scale-tier basic_gpu --region "$GCE_REGION" \ - --master-image-uri "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" \ - -- \ - NO_ARG \ - "gs://$MODEL_STORAGE_BUCKET" - - - name: Monitor the training job - # if: | - # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || - # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) - run: | - sh -c ".github/workflows/monitor-ai-platform-job.sh ${{ env.AI_PLATFORM_JOB_NAME }}" - timeout-minutes: 500 + # # ----------------------------------------------------------------------------------- # + # # ----------------------------------------------------------------------------------- # + # # ----------------------------------------------------------------------------------- # + # # TRAIN BL_CHEMICAL_TO_DISEASE_OR_PHENOTYPIC_FEATURE MODEL # + # # ----------------------------------------------------------------------------------- # + # # ----------------------------------------------------------------------------------- # + # # ----------------------------------------------------------------------------------- # + + # build_push_train-bl_chemical_to_disease_or_phenotypic_feature-train: + # name: "build/push/train bl_chemical_to_disease_or_phenotypic_feature model" + # # needs: "build_push-bert-base" + # runs-on: ubuntu-latest + # env: + # TASK_NAME: bl_chemical_to_disease_or_phenotypic_feature + # DOCKERFILE: train.Dockerfile + # ENTRYPOINT_FILE: scripts/train.entrypoint.sh + # BASE_DOCKERFILE: base.Dockerfile + # DATA_FILE: data/bl_chemical_to_disease_or_phenotypic_feature/data.tsv + # MODEL_VERSION_KEY: BL_CHEMICAL_TO_DISEASE_OR_PHENOTYPIC_FEATURE + # AI_PLATFORM_JOB_NAME: "bl_chemical_to_disease_or_phenotypic_feature_train_${{ github.run_number }}" + # steps: + # - uses: actions/checkout@v2 + # with: + # fetch-depth: 0 + # # - name: Get changed files + # # id: changed-files + # # uses: tj-actions/changed-files@v7 + + # # - name: List all modified files + # # run: | + # # for file in "${{ steps.changed-files.outputs.all_modified_files }}"; do + # # echo "$file was modified" + # # done + + # # - name: Run step when a file changes + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # # run: | + # # echo "A modified file has triggered a rebuild of the ${{ env.TASK_NAME }} model." + + # - name: Set up Cloud SDK + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # uses: google-github-actions/setup-gcloud@main + # with: + # project_id: ${{ secrets.GCE_PROJECT }} + # service_account_key: ${{ secrets.GCE_SA_KEY }} + # version: '290.0.1' + + # - name: Configure Docker Authentication + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # run: | + # gcloud --quiet auth configure-docker + + # - name: Set MODEL_VERSION env + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # run: | + # echo "MODEL_VERSION=$(grep ${{ env.MODEL_VERSION_KEY }} MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV + + # - name: Set BASE_VERSION env + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # run: | + # echo "BASE_VERSION=$(grep 'BASE' MODEL_VERSIONS | cut -f 2 -d '=')" >> $GITHUB_ENV + + # - name: Build Docker Image + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # run: | + # docker build --build-arg "PROJECT_ID=$PROJECT_ID" \ + # --build-arg "TASK_NAME=$TASK_NAME" \ + # --build-arg "BASE_VERSION=$BASE_VERSION" \ + # --build-arg "TUNED_MODEL_VERSION=$MODEL_VERSION" \ + # --tag "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" \ + # -f ${{ env.DOCKERFILE }} . + + # - name: Publish Docker Image to Google Container Registry + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # run: | + # docker push "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" + + # - name: Submit a training job to AI Plaform to train and cache the model + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # run: | + # gcloud ai-platform jobs submit training ${{ env.AI_PLATFORM_JOB_NAME }} \ + # --scale-tier basic_gpu --region "$GCE_REGION" \ + # --master-image-uri "gcr.io/$PROJECT_ID/$TASK_NAME-train:$MODEL_VERSION" \ + # -- \ + # NO_ARG \ + # "gs://$MODEL_STORAGE_BUCKET" + + # - name: Monitor the training job + # # if: | + # # contains(steps.changed-files.outputs.all_modified_files, env.DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.BASE_DOCKERFILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.DATA_FILE) || + # # contains(steps.changed-files.outputs.all_modified_files, env.ENTRYPOINT_FILE) + # run: | + # sh -c ".github/workflows/monitor-ai-platform-job.sh ${{ env.AI_PLATFORM_JOB_NAME }}" + # timeout-minutes: 500 # -------------------------------------------------------------------------------------- # # BUILD/PUSH BL_CHEMICAL_TO_DISEASE_OR_PHENOTYPIC_FEATURE CLASSIFICATION CONTAINER IMAGE # @@ -476,7 +476,7 @@ jobs: build_push-bl_chemical_to_disease_or_phenotypic_feature-predict: name: "build/push bl_chemical_to_disease_or_phenotypic_feature predict container" - needs: "build_push_train-bl_chemical_to_disease_or_phenotypic_feature-train" + # needs: "build_push_train-bl_chemical_to_disease_or_phenotypic_feature-train" runs-on: ubuntu-latest env: TASK_NAME: bl_chemical_to_disease_or_phenotypic_feature