Skip to content

Commit

Permalink
Simplify matrix specification.
Browse files Browse the repository at this point in the history
  • Loading branch information
@alliepiper
alliepiper committed Apr 23, 2024
1 parent 9561ae6 commit adf8a5d
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 53 deletions.
25 changes: 15 additions & 10 deletions ci/compute-matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"std": [
17
],
"projects": [
"project": [
"libcudacxx",
"cub",
"thrust"
Expand Down Expand Up @@ -130,6 +130,8 @@ def fill_defaults_matrix_job(matrix_job):
if tag not in matrix_job:
matrix_job[tag] = matrix_yaml['default_'+tag]


def set_derived_tags(matrix_job):
if 'os' not in matrix_job:
matrix_job['os'] = lookup_os(matrix_job['ctk'],
matrix_job['host_compiler']['name'],
Expand All @@ -142,11 +144,11 @@ def fill_defaults_matrix_job(matrix_job):

def explode_matrix_job(matrix_job):
new_jobs = []
for tag in matrix_yaml['explodable_tags']:
if tag in matrix_job and isinstance(matrix_job[tag], list) and len(matrix_job[tag]) > 1:
for tag in matrix_job:
if tag != "job_types" and isinstance(matrix_job[tag], list):
for value in matrix_job[tag]:
new_job = copy.deepcopy(matrix_job)
new_job[tag] = [value]
new_job[tag] = value
exploded = explode_matrix_job(new_job)
if exploded:
new_jobs.extend(exploded)
Expand All @@ -159,7 +161,7 @@ def explode_matrix_job(matrix_job):


def generate_dispatch_group_name(matrix_job):
project_name = get_formatted_projected_name(matrix_job['projects'][0])
project_name = get_formatted_projected_name(matrix_job['project'])
ctk = matrix_job['ctk']
device_compiler = matrix_job['device_compiler']
host_compiler_name = get_formatted_host_compiler_name(matrix_job['host_compiler'])
Expand All @@ -176,7 +178,7 @@ def generate_dispatch_group_name(matrix_job):


def generate_dispatch_job_name(matrix_job, job_type):
std_str = ("C++" + str(matrix_job['std'][0]) + " ") if 'std' in matrix_job else ''
std_str = ("C++" + str(matrix_job['std']) + " ") if 'std' in matrix_job else ''
cpu_str = matrix_job['cpu']
gpu_str = (', ' + matrix_job['gpu'].upper()) if job_type in matrix_yaml['gpu_required_job_types'] else ""
cuda_compile_arch = (" sm{" + matrix_job['cmake_cuda_arch'] + "}") if 'cmake_cuda_arch' in matrix_job else ""
Expand Down Expand Up @@ -224,10 +226,10 @@ def generate_dispatch_job_command(matrix_job, job_type):
script_path = "ci/windows" if is_windows(matrix_job) else "ci"
script_ext = ".ps1" if is_windows(matrix_job) else ".sh"
script_job_type = job_type
script_project = matrix_job['projects'][0]
script_project = matrix_job['project']
script_name = f"{script_path}/{script_job_type}_{script_project}{script_ext}"

std_str = str(matrix_job['std'][0]) if 'std' in matrix_job else ''
std_str = str(matrix_job['std']) if 'std' in matrix_job else ''

host_compiler_exe = matrix_job['host_compiler']['exe']
device_compiler_name = matrix_job['device_compiler']['name']
Expand Down Expand Up @@ -325,8 +327,10 @@ def matrix_job_to_dispatch_group(matrix_job):
merge_dispatch_groups(all_dispatch_groups, dispatch_group)
return all_dispatch_groups

set_derived_tags(matrix_job)

# Filter jobs that don't need to rerun:
if matrix_job['projects'][0] not in dirty_projects:
if matrix_job['project'] not in dirty_projects:
return {}

# We have a fully specified job, start processing.
Expand Down Expand Up @@ -360,7 +364,8 @@ def natural_sort_key(key):
if 'standalone' in group_json:
group_json['standalone'] = sorted(group_json['standalone'], key=lambda x: natural_sort_key(x['name']))
if 'two_stage' in group_json:
group_json['two_stage'] = sorted(group_json['two_stage'], key=lambda x: natural_sort_key(x['producers'][0]['name']))
group_json['two_stage'] = sorted(
group_json['two_stage'], key=lambda x: natural_sort_key(x['producers'][0]['name']))

# Count the total number of jobs:
total_jobs = 0
Expand Down
69 changes: 26 additions & 43 deletions ci/matrix.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,11 @@ oneapi: &oneapi { name: 'oneapi', version: '2023.2.0', exe: 'icpc' }
required_tags: ['job_types']

# Tags that will be added if not specified:
defaulted_tags: ['ctk', 'cpu', 'gpu', 'host_compiler', 'device_compiler', 'projects', 'os']
defaulted_tags: ['ctk', 'cpu', 'gpu', 'host_compiler', 'device_compiler', 'project', 'os']

# Tags that may be omitted:
optional_tags: ['std', 'cmake_cuda_arch', 'cmake_options']

# If these tags are lists, they will be exploded into separate jobs
explodable_tags: ['projects', 'std']

# job_types that have an implied prerequisite 'build' job:
build_required_job_types:
- 'test'
Expand Down Expand Up @@ -83,7 +80,7 @@ default_device_compiler: 'nvcc'
default_host_compiler: *gcc12
default_cpu: 'amd64'
default_gpu: 'v100'
default_projects: &default_projects
default_project:
- 'libcudacxx'
- 'cub'
- 'thrust'
Expand Down Expand Up @@ -136,44 +133,29 @@ testing_pool_gpus:

# Configurations that will run for every PR
pull_request:
- {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *gcc6, std: [11, 14] }
- {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *gcc7, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *gcc8, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *gcc9, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *llvm9, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: *msvc2017, std: [14, 17] }
- {job_types: ['build'], ctk: *ctk_prev_max, host_compiler: *gcc11, std: [11, 14, 17], cmake_cuda_arch: '90'}
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc7, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc8, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc9, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc10, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc11, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20], cmake_cuda_arch: '90'}
- {job_types: ['test'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20], cpu: 'arm64'}
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm9, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm10, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm11, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm12, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm13, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm14, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm15, std: [11, 14, 17, 20] }
- {job_types: ['test'], ctk: *ctk_curr, host_compiler: *llvm16, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *llvm16, std: [11, 14, 17, 20], cpu: 'arm64'}
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *msvc2019, std: [14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *msvc2022, std: [14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *oneapi, std: [11, 14, 17] }
# nvrtc:
- {job_types: ['nvrtc'], projects: ['libcudacxx'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20]}
- {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: [*msvc2017, *gcc6], std: [11, 14] }
- {job_types: ['build'], ctk: *ctk_prev_min, host_compiler: [*gcc7, *gcc8, *gcc9, *llvm9], std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_prev_max, host_compiler: *gcc11, std: [11, 14, 17], cmake_cuda_arch: '60;70;80;90'}
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*gcc7, *gcc8, *gcc9], std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*gcc10, *gcc11], std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*llvm9, *llvm10], std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*llvm11, *llvm12, *llvm13], std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*llvm14, *llvm15], std: [11, 14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *msvc2019, std: [14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *msvc2022, std: [14, 17, 20] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: *oneapi, std: [11, 14, 17] }
- {job_types: ['build'], ctk: *ctk_curr, host_compiler: [*gcc12, *llvm16], std: [11, 14, 17, 20], cpu: 'arm64'}
- {job_types: ['test'], ctk: *ctk_curr, host_compiler: [*gcc12, *llvm16], std: [11, 14, 17, 20], cmake_cuda_arch: '60;70;80;90'}
# clang-cuda:
- {job_types: ['build'], device_compiler: *llvm-newest, host_compiler: *llvm-newest, std: [17, 20]}
# nvrtc:
- {job_types: ['nvrtc'], project: 'libcudacxx', ctk: *ctk_curr, host_compiler: *gcc12, std: [11, 14, 17, 20]}
# cccl-infra:
- {projects: ['cccl'], job_types: ['infra'], ctk: *ctk_prev_min, host_compiler: *gcc-oldest}
- {projects: ['cccl'], job_types: ['infra'], ctk: *ctk_prev_min, host_compiler: *llvm-oldest}
- {projects: ['cccl'], job_types: ['infra'], ctk: *ctk_curr, host_compiler: *gcc-newest}
- {projects: ['cccl'], job_types: ['infra'], ctk: *ctk_curr, host_compiler: *llvm-newest}
- {job_types: ['infra'], project: 'cccl', ctk: *ctk_prev_min, host_compiler: [*gcc-oldest, *llvm-oldest]}
- {job_types: ['infra'], project: 'cccl', ctk: *ctk_curr, host_compiler: [*gcc-newest, *llvm-newest]}
# verify-codegen:
- {projects: ['libcudacxx'], job_types: ['verify_codegen']}
- { job_types: ['verify_codegen'], project: 'libcudacxx'}

# Run each night:
nightly:
Expand All @@ -185,7 +167,8 @@ nightly:
- {job_types: ['test'], ctk: *ctk_curr, gpu: 'rtx4090', cmake_cuda_arch: '89', host_compiler: *llvm9, std: [11] }
- {job_types: ['test'], ctk: *ctk_curr, gpu: 'h100', cmake_cuda_arch: '90', host_compiler: *gcc12, std: [11, 20] }
- {job_types: ['test'], ctk: *ctk_curr, gpu: 'h100', cmake_cuda_arch: '90', host_compiler: *llvm16, std: [17] }
- {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 't4', cmake_cuda_arch: '75', host_compiler: *gcc12, std: [20], projects: ['libcudacxx']}
- {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'rtxa6000', cmake_cuda_arch: '86', host_compiler: *gcc12, std: [20], projects: ['libcudacxx']}
- {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'l4', cmake_cuda_arch: '89', host_compiler: *gcc12, std: [11, 14, 17, 20], projects: ['libcudacxx']}
- {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'h100', cmake_cuda_arch: '90', host_compiler: *gcc12, std: [11, 20], projects: ['libcudacxx']}
# nvrtc:
- {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 't4', cmake_cuda_arch: '75', host_compiler: *gcc12, std: [20], project: ['libcudacxx']}
- {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'rtxa6000', cmake_cuda_arch: '86', host_compiler: *gcc12, std: [20], project: ['libcudacxx']}
- {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'l4', cmake_cuda_arch: '89', host_compiler: *gcc12, std: [11, 14, 17, 20], project: ['libcudacxx']}
- {job_types: ['nvrtc'], ctk: *ctk_curr, gpu: 'h100', cmake_cuda_arch: '90', host_compiler: *gcc12, std: [11, 20], project: ['libcudacxx']}

0 comments on commit adf8a5d

Please sign in to comment.