From adf8a5da9d425399d2c29195941b37f098911049 Mon Sep 17 00:00:00 2001 From: Allison Piper Date: Tue, 23 Apr 2024 01:30:25 +0000 Subject: [PATCH] Simplify matrix specification. --- ci/compute-matrix.py | 25 +++++++++------- ci/matrix.yaml | 69 +++++++++++++++++--------------------------- 2 files changed, 41 insertions(+), 53 deletions(-) diff --git a/ci/compute-matrix.py b/ci/compute-matrix.py index ea614ea178a..926d175cb4c 100755 --- a/ci/compute-matrix.py +++ b/ci/compute-matrix.py @@ -19,7 +19,7 @@ "std": [ 17 ], - "projects": [ + "project": [ "libcudacxx", "cub", "thrust" @@ -130,6 +130,8 @@ def fill_defaults_matrix_job(matrix_job): if tag not in matrix_job: matrix_job[tag] = matrix_yaml['default_'+tag] + +def set_derived_tags(matrix_job): if 'os' not in matrix_job: matrix_job['os'] = lookup_os(matrix_job['ctk'], matrix_job['host_compiler']['name'], @@ -142,11 +144,11 @@ def fill_defaults_matrix_job(matrix_job): def explode_matrix_job(matrix_job): new_jobs = [] - for tag in matrix_yaml['explodable_tags']: - if tag in matrix_job and isinstance(matrix_job[tag], list) and len(matrix_job[tag]) > 1: + for tag in matrix_job: + if tag != "job_types" and isinstance(matrix_job[tag], list): for value in matrix_job[tag]: new_job = copy.deepcopy(matrix_job) - new_job[tag] = [value] + new_job[tag] = value exploded = explode_matrix_job(new_job) if exploded: new_jobs.extend(exploded) @@ -159,7 +161,7 @@ def explode_matrix_job(matrix_job): def generate_dispatch_group_name(matrix_job): - project_name = get_formatted_projected_name(matrix_job['projects'][0]) + project_name = get_formatted_projected_name(matrix_job['project']) ctk = matrix_job['ctk'] device_compiler = matrix_job['device_compiler'] host_compiler_name = get_formatted_host_compiler_name(matrix_job['host_compiler']) @@ -176,7 +178,7 @@ def generate_dispatch_group_name(matrix_job): def generate_dispatch_job_name(matrix_job, job_type): - std_str = ("C++" + str(matrix_job['std'][0]) + " ") if 'std' in matrix_job else '' + std_str = ("C++" + str(matrix_job['std']) + " ") if 'std' in matrix_job else '' cpu_str = matrix_job['cpu'] gpu_str = (', ' + matrix_job['gpu'].upper()) if job_type in matrix_yaml['gpu_required_job_types'] else "" cuda_compile_arch = (" sm{" + matrix_job['cmake_cuda_arch'] + "}") if 'cmake_cuda_arch' in matrix_job else "" @@ -224,10 +226,10 @@ def generate_dispatch_job_command(matrix_job, job_type): script_path = "ci/windows" if is_windows(matrix_job) else "ci" script_ext = ".ps1" if is_windows(matrix_job) else ".sh" script_job_type = job_type - script_project = matrix_job['projects'][0] + script_project = matrix_job['project'] script_name = f"{script_path}/{script_job_type}_{script_project}{script_ext}" - std_str = str(matrix_job['std'][0]) if 'std' in matrix_job else '' + std_str = str(matrix_job['std']) if 'std' in matrix_job else '' host_compiler_exe = matrix_job['host_compiler']['exe'] device_compiler_name = matrix_job['device_compiler']['name'] @@ -325,8 +327,10 @@ def matrix_job_to_dispatch_group(matrix_job): merge_dispatch_groups(all_dispatch_groups, dispatch_group) return all_dispatch_groups + set_derived_tags(matrix_job) + # Filter jobs that don't need to rerun: - if matrix_job['projects'][0] not in dirty_projects: + if matrix_job['project'] not in dirty_projects: return {} # We have a fully specified job, start processing. @@ -360,7 +364,8 @@ def natural_sort_key(key): if 'standalone' in group_json: group_json['standalone'] = sorted(group_json['standalone'], key=lambda x: natural_sort_key(x['name'])) if 'two_stage' in group_json: - group_json['two_stage'] = sorted(group_json['two_stage'], key=lambda x: natural_sort_key(x['producers'][0]['name'])) + group_json['two_stage'] = sorted( + group_json['two_stage'], key=lambda x: natural_sort_key(x['producers'][0]['name'])) # Count the total number of jobs: total_jobs = 0 diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 5ec28b12e75..6a293d47a01 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -44,14 +44,11 @@ oneapi: &oneapi { name: 'oneapi', version: '2023.2.0', exe: 'icpc' } required_tags: ['job_types'] # Tags that will be added if not specified: -defaulted_tags: ['ctk', 'cpu', 'gpu', 'host_compiler', 'device_compiler', 'projects', 'os'] +defaulted_tags: ['ctk', 'cpu', 'gpu', 'host_compiler', 'device_compiler', 'project', 'os'] # Tags that may be omitted: optional_tags: ['std', 'cmake_cuda_arch', 'cmake_options'] - # If these tags are lists, they will be exploded into separate jobs -explodable_tags: ['projects', 'std'] - # job_types that have an implied prerequisite 'build' job: build_required_job_types: - 'test' @@ -83,7 +80,7 @@ default_device_compiler: 'nvcc' default_host_compiler: *gcc12 default_cpu: 'amd64' default_gpu: 'v100' -default_projects: &default_projects +default_project: - 'libcudacxx' - 'cub' - 'thrust' @@ -136,44 +133,29 @@ testing_pool_gpus: # Configurations that will run for every PR pull_request: - - {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *gcc6, std: [11, 14] } - - {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *gcc7, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *gcc8, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *gcc9, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *llvm9, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *msvc2017, std: [14, 17] } - - {job_types: ['build'], ctk: *ctk_prev_max, host_compiler: *gcc11, std: [11, 14, 17], cmake_cuda_arch: '90'} - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc7, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc8, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc9, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc10, std: [11, 14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc11, std: [11, 14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20], cmake_cuda_arch: '90'} - - {job_types: ['test'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20], cpu: 'arm64'} - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm9, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm10, std: [11, 14, 17] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm11, std: [11, 14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm12, std: [11, 14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm13, std: [11, 14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm14, std: [11, 14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm15, std: [11, 14, 17, 20] } - - {job_types: ['test'], ctk: *ctk_curr, host_compiler: *llvm16, std: [11, 14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm16, std: [11, 14, 17, 20], cpu: 'arm64'} - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *msvc2019, std: [14, 17] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *msvc2022, std: [14, 17, 20] } - - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *oneapi, std: [11, 14, 17] } - # nvrtc: - - {job_types: ['nvrtc'], projects: ['libcudacxx'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20]} + - {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: [*msvc2017, *gcc6], std: [11, 14] } + - {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: [*gcc7, *gcc8, *gcc9, *llvm9], std: [11, 14, 17] } + - {job_types: ['build'], ctk: *ctk_prev_max, host_compiler: *gcc11, std: [11, 14, 17], cmake_cuda_arch: '60;70;80;90'} + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*gcc7, *gcc8, *gcc9], std: [11, 14, 17] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*gcc10, *gcc11], std: [11, 14, 17, 20] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*llvm9, *llvm10], std: [11, 14, 17] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*llvm11, *llvm12, *llvm13], std: [11, 14, 17, 20] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*llvm14, *llvm15], std: [11, 14, 17, 20] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *msvc2019, std: [14, 17] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *msvc2022, std: [14, 17, 20] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: *oneapi, std: [11, 14, 17] } + - {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*gcc12, *llvm16], std: [11, 14, 17, 20], cpu: 'arm64'} + - {job_types: ['test'], ctk: *ctk_curr, host_compiler: [*gcc12, *llvm16], std: [11, 14, 17, 20], cmake_cuda_arch: '60;70;80;90'} # clang-cuda: - {job_types: ['build'], device_compiler: *llvm-newest, host_compiler: *llvm-newest, std: [17, 20]} + # nvrtc: + - {job_types: ['nvrtc'], project: 'libcudacxx', ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20]} # cccl-infra: - - {projects: ['cccl'], job_types: ['infra'], ctk: *ctk_prev_min, host_compiler: *gcc-oldest} - - {projects: ['cccl'], job_types: ['infra'], ctk: *ctk_prev_min, host_compiler: *llvm-oldest} - - {projects: ['cccl'], job_types: ['infra'], ctk: *ctk_curr, host_compiler: *gcc-newest} - - {projects: ['cccl'], job_types: ['infra'], ctk: *ctk_curr, host_compiler: *llvm-newest} + - {job_types: ['infra'], project: 'cccl', ctk: *ctk_prev_min, host_compiler: [*gcc-oldest, *llvm-oldest]} + - {job_types: ['infra'], project: 'cccl', ctk: *ctk_curr, host_compiler: [*gcc-newest, *llvm-newest]} # verify-codegen: - - {projects: ['libcudacxx'], job_types: ['verify_codegen']} + - { job_types: ['verify_codegen'], project: 'libcudacxx'} # Run each night: nightly: @@ -185,7 +167,8 @@ nightly: - {job_types: ['test'], ctk: *ctk_curr, gpu: 'rtx4090', cmake_cuda_arch: '89', host_compiler: *llvm9, std: [11] } - {job_types: ['test'], ctk: *ctk_curr, gpu: 'h100', cmake_cuda_arch: '90', host_compiler: *gcc12, std: [11, 20] } - {job_types: ['test'], ctk: *ctk_curr, gpu: 'h100', cmake_cuda_arch: '90', host_compiler: *llvm16, std: [17] } - - {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 't4', cmake_cuda_arch: '75', host_compiler: *gcc12, std: [20], projects: ['libcudacxx']} - - {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'rtxa6000', cmake_cuda_arch: '86', host_compiler: *gcc12, std: [20], projects: ['libcudacxx']} - - {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'l4', cmake_cuda_arch: '89', host_compiler: *gcc12, std: [11, 14, 17, 20], projects: ['libcudacxx']} - - {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'h100', cmake_cuda_arch: '90', host_compiler: *gcc12, std: [11, 20], projects: ['libcudacxx']} + # nvrtc: + - {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 't4', cmake_cuda_arch: '75', host_compiler: *gcc12, std: [20], project: ['libcudacxx']} + - {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'rtxa6000', cmake_cuda_arch: '86', host_compiler: *gcc12, std: [20], project: ['libcudacxx']} + - {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'l4', cmake_cuda_arch: '89', host_compiler: *gcc12, std: [11, 14, 17, 20], project: ['libcudacxx']} + - {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'h100', cmake_cuda_arch: '90', host_compiler: *gcc12, std: [11, 20], project: ['libcudacxx']}