test CICD v4 w/ .kubernetes_environment

.gitlab-ci.yml

@@ -1,7 +1,8 @@
 include:
   - project: 'analytics/artificial-intelligence/ai-platform/aip-infrastructure/ci-templates/ci-cd-template'
 #    ref: &include_ref 'v4'
-    ref: &include_ref '4e25c7200369aec17fcf8de723db360a7726151a'
+    ref: &include_ref 'tz/cicd-fix-wfsdk'
+#    ref: &include_ref '4e25c7200369aec17fcf8de723db360a7726151a'
     file: 'environments/devex.yml'
   - project: 'analytics/artificial-intelligence/ai-platform/aip-infrastructure/ci-templates/ci-cd-template'
     ref: *include_ref
@@ -22,6 +23,7 @@ variables:
   DEPLOY_SANDBOX: "false"
   PUBLISH: "true"
   CI_TRACE_DEBUG: "true"
+  NAMESPACE_PREFIX: "metaflow-integration-testing"
 
 stages:
   - build
@@ -44,20 +46,26 @@ stages:
   PY_LIBRARY_VERSION="${PY_LIBRARY_VERSION}+${METAFLOW_VERSION}"
 
 .test:
-  extends: .generate_kubeconfig
+  extends: .kubernetes_environment
   stage: test
   variables:
     GIT_STRATEGY: none
     PIPELINE_VERSION: "1.0"
   before_script:
     - export IMAGE_REPOSITORY_TAG=$( cat ${IMAGE_REPOSITORY_TAG_PATH} )
     - export IMAGE_REPOSITORY_TAG_AIP_STEP=$( cat ${IMAGE_REPOSITORY_TAG_PATH_AIP_STEP} )
-    - !reference [.generate_kubeconfig, before_script]
+    - !reference [.kubernetes_environment, before_script]
   script:
+    - env
+    - |
+      echo "SHARED_PATH=${SHARED_PATH}"
+      echo "WORKING_DIR=${WORKING_DIR}"
+      echo "CI_PROJECT_PATH=${CI_PROJECT_PATH}"
     - docker run
         --rm
         -v ${SHARED_PATH}/.kube:/home/zservice/.kube
-        -v ${SHARED_PATH}/public:/home/zservice/public  
+        -v ${SHARED_PATH}/public:/home/zservice/public
+        -v ${SHARED_PATH}/.aws:/home/zservice/.aws
         -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID 
         -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY 
         -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN 
@@ -105,41 +113,45 @@ build:docker:
       }
 
     - |
-      export IMAGE_REPOSITORY_TAG=${DOCKER_REPO_URL}/${CI_PROJECT_NAMESPACE}/${PY_LIBRARY_NAME}:${IMAGE_TAG}
-      build_image $IMAGE_REPOSITORY_TAG $IMAGE_REPOSITORY_TAG_PATH
+      # export IMAGE_REPOSITORY_TAG=${DOCKER_REPO_URL}/${CI_PROJECT_NAMESPACE}/${PY_LIBRARY_NAME}:${IMAGE_TAG}
+      export IMAGE_REPOSITORY_TAG=analytics-docker.artifactory.zgtools.net/analytics/artificial-intelligence/ai-platform/sdks/zillow-metaflow:2.0.2548
+      # build_image $IMAGE_REPOSITORY_TAG $IMAGE_REPOSITORY_TAG_PATH
+      echo $IMAGE_REPOSITORY_TAG > $IMAGE_REPOSITORY_TAG_PATH
 
     # Create another image specifically to test aip_flow.py, this sets the image name within the built image
     # such that the test can assert that for specific steps it is using this overridden image
     - |
-      export IMAGE_REPOSITORY_TAG_AIP_STEP=${DOCKER_REPO_URL}/${CI_PROJECT_NAMESPACE}/${PY_LIBRARY_NAME}:${IMAGE_TAG}_aip_step
-      build_image $IMAGE_REPOSITORY_TAG_AIP_STEP $IMAGE_REPOSITORY_TAG_PATH_AIP_STEP
+      # export IMAGE_REPOSITORY_TAG_AIP_STEP=${DOCKER_REPO_URL}/${CI_PROJECT_NAMESPACE}/${PY_LIBRARY_NAME}:${IMAGE_TAG}_aip_step
+      export IMAGE_REPOSITORY_TAG_AIP_STEP=analytics-docker.artifactory.zgtools.net/analytics/artificial-intelligence/ai-platform/sdks/zillow-metaflow:2.0.2548_aip_step
+      # build_image $IMAGE_REPOSITORY_TAG_AIP_STEP $IMAGE_REPOSITORY_TAG_PATH_AIP_STEP
+      echo $IMAGE_REPOSITORY_TAG_AIP_STEP > $IMAGE_REPOSITORY_TAG_PATH_AIP_STEP
   artifacts:
     paths:
       - ${IMAGE_REPOSITORY_TAG_PATH}
       - ${IMAGE_REPOSITORY_TAG_PATH_AIP_STEP}
 
-build:publish:
-  extends: .aip_python_debian_image
-  stage: build
-  script:
-  - *version
-  # Now write back the zillow-metaflow version back to the original location we found the original so
-  # python package managers can reference it as they need the version stored internally.
-  - sed -i "s/\(version = ['\"]\)[^'\"]*\(['\"]\)/\1${PY_LIBRARY_VERSION}\2/" $METAFLOW_VERSION_PATH
-  - python setup.py sdist
-  # Set up the configuration for Artifactory to publish the python package internally.
-  - |
-    cat >~/.pypirc <<EOL
-    [distutils]
-    index-servers = local
-    [local]
-    repository: ${ANALYTICS_PYPI_REPOSITORY}
-    username: ${PYPI_USERNAME}
-    password: ${PYPI_PASSWORD}
-    EOL
-  - python setup.py sdist upload --repository "${ANALYTICS_PYPI_REPOSITORY}"
-  rules:
-    - if: '$PUBLISH == "true"'
+#build:publish:
+#  extends: .aip_python_debian_image
+#  stage: build
+#  script:
+#  - *version
+#  # Now write back the zillow-metaflow version back to the original location we found the original so
+#  # python package managers can reference it as they need the version stored internally.
+#  - sed -i "s/\(version = ['\"]\)[^'\"]*\(['\"]\)/\1${PY_LIBRARY_VERSION}\2/" $METAFLOW_VERSION_PATH
+#  - python setup.py sdist
+#  # Set up the configuration for Artifactory to publish the python package internally.
+#  - |
+#    cat >~/.pypirc <<EOL
+#    [distutils]
+#    index-servers = local
+#    [local]
+#    repository: ${ANALYTICS_PYPI_REPOSITORY}
+#    username: ${PYPI_USERNAME}
+#    password: ${PYPI_PASSWORD}
+#    EOL
+#  - python setup.py sdist upload --repository "${ANALYTICS_PYPI_REPOSITORY}"
+#  rules:
+#    - if: '$PUBLISH == "true"'
 
 test:sandbox:
   extends: