diff --git a/.github/workflows/kind-cluster/determine_models.py b/.github/workflows/kind-cluster/determine_models.py index 555c1bc27..92b3d7a6c 100644 --- a/.github/workflows/kind-cluster/determine_models.py +++ b/.github/workflows/kind-cluster/determine_models.py @@ -119,11 +119,14 @@ def main(): pr_branch = os.environ.get("PR_BRANCH", "main") # If not specified default to 'main' force_run_all = os.environ.get("FORCE_RUN_ALL", "false") # If not specified default to False force_run_all_phi = os.environ.get("FORCE_RUN_ALL_PHI", "false") # If not specified default to False + force_run_all_public = os.environ.get("FORCE_RUN_ALL_PUBLIC", "false") # If not specified default to False affected_models = [] if force_run_all != "false": affected_models = [model['name'] for model in YAML_PR['models']] - elif force_run_all_phi != "false": + elif force_run_all_public != "false": + affected_models = [model['name'] for model in YAML_PR['models'] if "llama" not in model['name']] + elif force_run_all_phi != "false": affected_models = [model['name'] for model in YAML_PR['models'] if 'phi-3' in model['name']] else: # Logic to determine affected models diff --git a/.github/workflows/kind-cluster/main.py b/.github/workflows/kind-cluster/main.py index c75f50dd1..f73688c48 100644 --- a/.github/workflows/kind-cluster/main.py +++ b/.github/workflows/kind-cluster/main.py @@ -13,7 +13,7 @@ def get_weights_path(model_name): return f"{WEIGHTS_FOLDER}/{model_name}/weights" def get_dockerfile_path(model_runtime): - return f"/kaito/docker/presets/workspace/models/{model_runtime}/Dockerfile" + return f"/kaito/docker/presets/models/{model_runtime}/Dockerfile" def generate_unique_id(): """Generate a unique identifier for a job.""" @@ -134,7 +134,7 @@ def main(): kubectl_path = get_kubectl_path() run_command(f"{kubectl_path} apply -f {job_name}-job.yaml") job_names.append(job_name) - + if not wait_for_jobs_to_complete(job_names): exit(1) # Exit with an error code if any job failed @@ -191,6 +191,11 @@ def populate_job_template(image_name, model_name, model_type, model_runtime, mod def log_job_info(job_name): """Log information about our Job's pod for debugging.""" + # Describe the job + # command_describe_job = f"kubectl describe job {job_name}" + # job_desc = run_command(command_describe_job) + # print(f"Job Description: \n{job_desc}") + # print("===============================\n") # Find the pod(s) associated with the job command_find_pods = f"kubectl get pods --selector=job-name=docker-build-job-{job_name} -o jsonpath='{{.items[*].metadata.name}}'" pod_names = run_command(command_find_pods) diff --git a/.github/workflows/preset-image-build-1ES.yml b/.github/workflows/preset-image-build-1ES.yml index 1e14ea9fa..edcd3106a 100644 --- a/.github/workflows/preset-image-build-1ES.yml +++ b/.github/workflows/preset-image-build-1ES.yml @@ -23,6 +23,10 @@ on: type: boolean default: false description: "Run all models for build" + force-run-all-public: + type: boolean + default: false + description: "Run all public models for build" force-run-all-phi: type: boolean default: false @@ -56,6 +60,7 @@ jobs: run: | echo "FORCE_RUN_ALL=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.force-run-all == 'true' }}" >> $GITHUB_OUTPUT echo "FORCE_RUN_ALL_PHI=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.force-run-all-phi == 'true' }}" >> $GITHUB_OUTPUT + echo "FORCE_RUN_ALL_PUBLIC=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.force-run-all-public == 'true' }}" >> $GITHUB_OUTPUT # This script should output a JSON array of model names - name: Determine Affected Models @@ -64,6 +69,7 @@ jobs: PR_BRANCH=${{ env.BRANCH_NAME }} \ FORCE_RUN_ALL=${{ steps.set_force_run_all.outputs.FORCE_RUN_ALL }} \ FORCE_RUN_ALL_PHI=${{ steps.set_force_run_all.outputs.FORCE_RUN_ALL_PHI }} \ + FORCE_RUN_ALL_PUBLIC=${{ steps.set_force_run_all.outputs.FORCE_RUN_ALL_PUBLIC }} \ python3 .github/workflows/kind-cluster/determine_models.py - name: Print Determined Models