From a991dedb20ab25de3446dde11b5b4e81b10bbf88 Mon Sep 17 00:00:00 2001 From: Allison Piper Date: Mon, 22 Apr 2024 18:31:00 +0000 Subject: [PATCH] Restrict two-stage to have a single producer only, more UI tweaks. --- .github/workflows/ci-dispatch-two-stage.yml | 5 ++ ci/compute-matrix.py | 66 ++++++++++++++------- ci/matrix.yaml | 4 +- 3 files changed, 49 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci-dispatch-two-stage.yml b/.github/workflows/ci-dispatch-two-stage.yml index dda70fcfa2d..9a7d9f03620 100644 --- a/.github/workflows/ci-dispatch-two-stage.yml +++ b/.github/workflows/ci-dispatch-two-stage.yml @@ -15,6 +15,11 @@ permissions: jobs: producers: + # It is impossible to accumulate output variables across a matrix, and we cannot rely on the results of the dispatch-job workflow to determine success. + # See the note in ci-dispatch-job.yml for more information. + # + # Since we cannot accumulate results from multiple producers, only support a single producer for now. + if: ${{ fromJSON(inputs.producers).length == 1 }} permissions: id-token: write contents: read diff --git a/ci/compute-matrix.py b/ci/compute-matrix.py index 8683502259b..be4a27f2bd5 100755 --- a/ci/compute-matrix.py +++ b/ci/compute-matrix.py @@ -84,13 +84,25 @@ def lookup_os(ctk, host_compiler_name, host_compiler_version): def get_formatted_projected_name(project_name): - return matrix_yaml['formatted_project_names'][project_name] - + if project_name in matrix_yaml['formatted_project_names']: + return matrix_yaml['formatted_project_names'][project_name] + return project_name + +def get_formatted_host_compiler_name(host_compiler): + config_name = host_compiler['name'] + if config_name in matrix_yaml['formatted_host_compiler_names']: + return matrix_yaml['formatted_host_compiler_names'][config_name] + return config_name + +def get_formatted_job_type(job_type): + if job_type in matrix_yaml['formatted_job_types']: + return matrix_yaml['formatted_job_types'][job_type] + # Return with first letter capitalized: + return job_type.capitalize() def is_windows(matrix_job): return matrix_job['os'].startswith('windows') - def validate_matrix_job(matrix_job): for tag in matrix_yaml['required_tags']: if tag not in matrix_job: @@ -137,17 +149,11 @@ def explode_matrix_job(matrix_job): return new_jobs if len(new_jobs) > 0 else None - def generate_dispatch_group_name(matrix_job): project_name = get_formatted_projected_name(matrix_job['projects'][0]) ctk = matrix_job['ctk'] device_compiler = matrix_job['device_compiler'] - host_compiler = matrix_job['host_compiler'] - - if host_compiler['name'] in matrix_yaml['formatted_host_compiler_names']: - host_compiler_name = matrix_yaml['formatted_host_compiler_names'][host_compiler['name']] - else: - host_compiler_name = host_compiler['name'] + host_compiler_name = get_formatted_host_compiler_name(matrix_job['host_compiler']) compiler_info = "" if device_compiler['name'] == 'nvcc': @@ -168,17 +174,21 @@ def generate_dispatch_job_runner_dispatch(matrix_job, job_type): def generate_dispatch_job_name(matrix_job, job_type): - formatted_job_type = matrix_yaml['formatted_job_types'][job_type] + std_ver = matrix_job['std'][0] + cpu_str = matrix_job['cpu'] + gpu_str = (', ' + matrix_job['gpu'].upper()) if job_type in matrix_yaml['gpu_required_job_types'] else "" + cuda_compile_arch = (" sm{" + matrix_job['cmake_cuda_arch'] + "}") if 'cmake_cuda_arch' in matrix_job else "" + cmake_options = matrix_job['cmake_options'] if 'cmake_options' in matrix_job else "" - gpu_str = "" - if job_type in matrix_yaml['gpu_required_job_types']: - gpu_str = "," + matrix_job['gpu'].upper() + host_compiler_name = get_formatted_host_compiler_name(matrix_job['host_compiler']) + host_compiler_info = f"{host_compiler_name}{matrix_job['host_compiler']['version']}" - cuda_compile_arch = (" sm{" + matrix_job['cmake_cuda_arch'] + "}") if 'cmake_cuda_arch' in matrix_job else "" - cmake_options = (" " + matrix_job['cmake_options']) if 'cmake_options' in matrix_job else "" + config_tag = f"C++{std_ver} {host_compiler_info}{cuda_compile_arch}" + + formatted_job_type = get_formatted_job_type(job_type) + + return f"[{config_tag}] {formatted_job_type}({cpu_str}{gpu_str})" - extra_args=cuda_compile_arch + cmake_options - extra_args = f":{extra_args}" if extra_args else "" return "[{}-{} C++{}] {}({}{}){}".format( matrix_job['host_compiler']['name'], @@ -257,15 +267,14 @@ def generate_dispatch_job_json(matrix_job, job_type): # Create a single build producer, and a separate consumer for each test_job_type: def generate_dispatch_build_and_test_json(matrix_job, build_job_type, test_job_types): build_json = generate_dispatch_job_json(matrix_job, build_job_type) - nvrtc_json = generate_dispatch_job_json(matrix_job, "nvrtc") - test_json = {} + test_json = [] for test_job_type in test_job_types: - test_json.update(generate_dispatch_job_json(matrix_job, test_job_type)) + test_json.append(generate_dispatch_job_json(matrix_job, test_job_type)) return { - "producers": [build_json, nvrtc_json], - "consumers": [test_json, test_json] + "producers": [build_json], + "consumers": test_json } @@ -375,6 +384,17 @@ def finalize_workflow_dispatch_groups(workflow_dispatch_groups_orig): print(f"Total jobs: {total_jobs}", file=sys.stderr) + # Check to see if any .two_stage.producers arrays have more than 1 job, which is not supported. See ci-dispatch-two-stage.yml for details. + for group_name, group_json in workflow_dispatch_groups.items(): + if 'two_stage' in group_json: + for two_stage_json in group_json['two_stage']: + num_producers = len(two_stage_json['producers']) + if num_producers > 1: + producer_names = "" + for job in two_stage_json['producers']: + producer_names += f" - {job['name']}\n" + raise Exception(f"ci-dispatch-two-stage.yml currently only supports a single producer. Found {num_producers} producers in '{group_name}':\n{producer_names}") + return workflow_dispatch_groups diff --git a/ci/matrix.yaml b/ci/matrix.yaml index d72b4b52ea8..345767a1010 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -106,9 +106,7 @@ all_job_types: - 'build' - 'test' - 'nvrtc' -formatted_job_types: - 'build': 'Build' - 'test': 'Test' +formatted_job_types: # Default: Capitalize first letter. 'nvrtc': 'NVRTC' formatted_host_compiler_names: