From 212a9673bd3014292c84b50a512d242230dd8727 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 14:34:39 +0200 Subject: [PATCH 01/10] tools: Autotuner: Prepare Python module structure Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/installer.sh | 2 +- tools/AutoTuner/pyproject.toml | 28 +++++++++++++++++++++++ tools/AutoTuner/requirements-dev.txt | 1 + tools/AutoTuner/src/autotuner/__init__.py | 4 ++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tools/AutoTuner/pyproject.toml create mode 100644 tools/AutoTuner/requirements-dev.txt create mode 100644 tools/AutoTuner/src/autotuner/__init__.py diff --git a/tools/AutoTuner/installer.sh b/tools/AutoTuner/installer.sh index c694715511..7d5f22f5fd 100755 --- a/tools/AutoTuner/installer.sh +++ b/tools/AutoTuner/installer.sh @@ -7,5 +7,5 @@ script_dir="$(dirname "${BASH_SOURCE[0]}")" venv_name="autotuner_env" python3 -m venv "$script_dir/$venv_name" source "$script_dir/$venv_name/bin/activate" -pip3 install -U -r $script_dir/requirements.txt +pip3 install -e "$script_dir" deactivate diff --git a/tools/AutoTuner/pyproject.toml b/tools/AutoTuner/pyproject.toml new file mode 100644 index 0000000000..3261ae831e --- /dev/null +++ b/tools/AutoTuner/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "autotuner" +version = "0.0.1" +description = "This project provides a set of tools for tuning OpenROAD-flow-scripts parameter without user interference." +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD 3-Clause", +] +readme = "README.md" +requires-python = ">= 3.8" +dynamic = ["dependencies", "optional-dependencies"] + +[tool.setuptools.dynamic] +dependencies = { file = ["requirements.txt"] } +optional-dependencies.dev = { file = ["requirements-dev.txt"] } + +[build-system] +requires = ["setuptools", "setuptools_scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src/"] +include = [ + "autotuner*", +] + +[tool.setuptools] +include-package-data = true diff --git a/tools/AutoTuner/requirements-dev.txt b/tools/AutoTuner/requirements-dev.txt new file mode 100644 index 0000000000..7e66a17d49 --- /dev/null +++ b/tools/AutoTuner/requirements-dev.txt @@ -0,0 +1 @@ +black diff --git a/tools/AutoTuner/src/autotuner/__init__.py b/tools/AutoTuner/src/autotuner/__init__.py new file mode 100644 index 0000000000..2703c2c57d --- /dev/null +++ b/tools/AutoTuner/src/autotuner/__init__.py @@ -0,0 +1,4 @@ +""" +AutoTuner module integrating Ray Tune and Vizier framework +for ORFS parameters optimization. +""" From c674f3f1341380d850879f165edfbb7e91a90d54 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 14:36:33 +0200 Subject: [PATCH 02/10] tools: Autotuner: Move part of the code to utils Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/src/autotuner/distributed.py | 604 ++--------------- tools/AutoTuner/src/autotuner/utils.py | 649 +++++++++++++++++++ 2 files changed, 701 insertions(+), 552 deletions(-) create mode 100644 tools/AutoTuner/src/autotuner/utils.py diff --git a/tools/AutoTuner/src/autotuner/distributed.py b/tools/AutoTuner/src/autotuner/distributed.py index 9af8da60a0..c0e54ef45b 100644 --- a/tools/AutoTuner/src/autotuner/distributed.py +++ b/tools/AutoTuner/src/autotuner/distributed.py @@ -3,23 +3,23 @@ Dependencies are documented in pip format at distributed-requirements.txt For both sweep and tune modes: - python3 distributed.py -h + python3 -m autotuner.distributed -h Note: the order of the parameters matter. Arguments --design, --platform and --config are always required and should precede the . AutoTuner: - python3 distributed.py tune -h - python3 distributed.py --design gcd --platform sky130hd \ + python3 -m autotuner.distributed tune -h + python3 -m autotuner.distributed --design gcd --platform sky130hd \ --config ../designs/sky130hd/gcd/autotuner.json \ tune Example: Parameter sweeping: - python3 distributed.py sweep -h + python3 -m autotuner.distributed sweep -h Example: - python3 distributed.py --design gcd --platform sky130hd \ + python3 -m autotuner.distributed --design gcd --platform sky130hd \ --config distributed-sweep-example.json \ sweep """ @@ -27,17 +27,11 @@ import argparse import json import os -import re import sys -import glob -import subprocess import random -from datetime import datetime -from multiprocessing import cpu_count -from subprocess import run from itertools import product from collections import namedtuple -from uuid import uuid4 as uuid +from multiprocessing import cpu_count import numpy as np import torch @@ -55,15 +49,28 @@ from ax.service.ax_client import AxClient -DATE = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") -ORFS_URL = "https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts" -FASTROUTE_TCL = "fastroute.tcl" -CONSTRAINTS_SDC = "constraint.sdc" +from autotuner.utils import ( + openroad, + consumer, + parse_config, + read_config, + read_metrics, + prepare_ray_server, + DATE, + CONSTRAINTS_SDC, + FASTROUTE_TCL, +) + +# Name of the final metric METRIC = "minimum" +# The worst of optimized metric ERROR_METRIC = 9e99 +# Path to the FLOW_HOME directory ORFS_FLOW_DIR = os.path.abspath( os.path.join(os.path.dirname(__file__), "../../../../flow") ) +# URL to ORFS GitHub repository +ORFS_URL = "https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts" class AutoTunerBase(tune.Trainable): @@ -78,9 +85,19 @@ def setup(self, config): # We create the following directory structure: # 1/ 2/ 3/ 4/ 5/ 6/ # ////-DATE// + # Run by Ray in directory specified by `local_dir` repo_dir = os.getcwd() + "/../" * 6 self.repo_dir = os.path.abspath(repo_dir) - self.parameters = parse_config(config, path=os.getcwd()) + self.parameters = parse_config( + config=config, + base_dir=self.repo_dir, + platform=args.platform, + sdc_original=SDC_ORIGINAL, + constraints_sdc=CONSTRAINTS_SDC, + fr_original=FR_ORIGINAL, + fastroute_tcl=FASTROUTE_TCL, + path=os.getcwd(), + ) self.step_ = 0 self.variant = f"variant-{self.__class__.__name__}-{self.trial_id}-or" @@ -88,9 +105,15 @@ def step(self): """ Run step experiment and compute its score. """ - metrics_file = openroad(self.repo_dir, self.parameters, self.variant) + metrics_file = openroad( + args=args, + base_dir=self.repo_dir, + parameters=self.parameters, + flow_variant=self.variant, + install_path=INSTALL_PATH, + ) self.step_ += 1 - score = self.evaluate(self.read_metrics(metrics_file)) + score = self.evaluate(read_metrics(metrics_file)) # Feed the score back to Tune. # return must match 'metric' used in tune.run() return {METRIC: score} @@ -110,46 +133,6 @@ def evaluate(self, metrics): score = score * (self.step_ / 100) ** (-1) + gamma * metrics["num_drc"] return score - @classmethod - def read_metrics(cls, file_name): - """ - Collects metrics to evaluate the user-defined objective function. - """ - with open(file_name) as file: - data = json.load(file) - clk_period = 9999999 - worst_slack = "ERR" - wirelength = "ERR" - num_drc = "ERR" - total_power = "ERR" - core_util = "ERR" - final_util = "ERR" - for stage, value in data.items(): - if stage == "constraints" and len(value["clocks__details"]) > 0: - clk_period = float(value["clocks__details"][0].split()[1]) - if stage == "floorplan" and "design__instance__utilization" in value: - core_util = value["design__instance__utilization"] - if stage == "detailedroute" and "route__drc_errors" in value: - num_drc = value["route__drc_errors"] - if stage == "detailedroute" and "route__wirelength" in value: - wirelength = value["route__wirelength"] - if stage == "finish" and "timing__setup__ws" in value: - worst_slack = value["timing__setup__ws"] - if stage == "finish" and "power__total" in value: - total_power = value["power__total"] - if stage == "finish" and "design__instance__utilization" in value: - final_util = value["design__instance__utilization"] - ret = { - "clk_period": clk_period, - "worst_slack": worst_slack, - "wirelength": wirelength, - "num_drc": num_drc, - "total_power": total_power, - "core_util": core_util, - "final_util": final_util, - } - return ret - class PPAImprov(AutoTunerBase): """ @@ -199,453 +182,6 @@ def evaluate(self, metrics): return score -def read_config(file_name): - """ - Please consider inclusive, exclusive - Most type uses [min, max) - But, Quantization makes the upper bound inclusive. - e.g., qrandint and qlograndint uses [min, max] - step value is used for quantized type (e.g., quniform). Otherwise, write 0. - When min==max, it means the constant value - """ - - def read(path): - # if file path does not exist, return empty string - print(os.path.abspath(path)) - if not os.path.isfile(os.path.abspath(path)): - return "" - with open(os.path.abspath(path), "r") as file: - ret = file.read() - return ret - - def read_sweep(this): - return [*this["minmax"], this["step"]] - - def apply_condition(config, data): - # TODO: tune.sample_from only supports random search algorithm. - # To make conditional parameter for the other algorithms, different - # algorithms should take different methods (will be added) - if args.algorithm != "random": - return config - dp_pad_min = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["minmax"][0] - dp_pad_step = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["step"] - if dp_pad_step == 1: - config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from( - lambda spec: np.random.randint( - dp_pad_min, spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1 - ) - ) - if dp_pad_step > 1: - config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from( - lambda spec: random.randrange( - dp_pad_min, - spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1, - dp_pad_step, - ) - ) - return config - - def read_tune(this): - min_, max_ = this["minmax"] - if min_ == max_: - # Returning a choice of a single element allow pbt algorithm to - # work. pbt does not accept single values as tunable. - return tune.choice([min_, max_]) - if this["type"] == "int": - if this["step"] == 1: - return tune.randint(min_, max_) - return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"]))) - if this["type"] == "float": - if this["step"] == 0: - return tune.uniform(min_, max_) - return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"]))) - return None - - def read_tune_ax(name, this): - """ - Ax format: https://ax.dev/versions/0.3.7/api/service.html - """ - dict_ = dict(name=name) - if "minmax" not in this: - return None - min_, max_ = this["minmax"] - if min_ == max_: - dict_["type"] = "fixed" - dict_["value"] = min_ - elif this["type"] == "int": - if this["step"] == 1: - dict_["type"] = "range" - dict_["bounds"] = [min_, max_] - dict_["value_type"] = "int" - else: - dict_["type"] = "choice" - dict_["values"] = tune.randint(min_, max_, this["step"]) - dict_["value_type"] = "int" - elif this["type"] == "float": - if this["step"] == 1: - dict_["type"] = "choice" - dict_["values"] = tune.choice( - np.ndarray.tolist(np.arange(min_, max_, this["step"])) - ) - dict_["value_type"] = "float" - else: - dict_["type"] = "range" - dict_["bounds"] = [min_, max_] - dict_["value_type"] = "float" - return dict_ - - def read_tune_pbt(name, this): - """ - PBT format: https://docs.ray.io/en/releases-2.9.3/tune/examples/pbt_guide.html - Note that PBT does not support step values. - """ - if "minmax" not in this: - return None - min_, max_ = this["minmax"] - if min_ == max_: - return ray.tune.choice([min_, max_]) - if this["type"] == "int": - return ray.tune.randint(min_, max_) - if this["type"] == "float": - return ray.tune.uniform(min_, max_) - - # Check file exists and whether it is a valid JSON file. - assert os.path.isfile(file_name), f"File {file_name} not found." - try: - with open(file_name) as file: - data = json.load(file) - except json.JSONDecodeError: - raise ValueError(f"Invalid JSON file: {file_name}") - sdc_file = "" - fr_file = "" - if args.mode == "tune" and args.algorithm == "ax": - config = list() - else: - config = dict() - for key, value in data.items(): - if key == "best_result": - continue - if key == "_SDC_FILE_PATH" and value != "": - if sdc_file != "": - print("[WARNING TUN-0004] Overwriting SDC base file.") - sdc_file = read(f"{os.path.dirname(file_name)}/{value}") - continue - if key == "_FR_FILE_PATH" and value != "": - if fr_file != "": - print("[WARNING TUN-0005] Overwriting FastRoute base file.") - fr_file = read(f"{os.path.dirname(file_name)}/{value}") - continue - if not isinstance(value, dict): - # To take care of empty values like _FR_FILE_PATH - if args.mode == "tune" and args.algorithm == "ax": - param_dict = read_tune_ax(key, value) - if param_dict: - config.append(param_dict) - elif args.mode == "tune" and args.algorithm == "pbt": - param_dict = read_tune_pbt(key, value) - if param_dict: - config[key] = param_dict - else: - config[key] = value - elif args.mode == "sweep": - config[key] = read_sweep(value) - elif args.mode == "tune" and args.algorithm == "ax": - config.append(read_tune_ax(key, value)) - elif args.mode == "tune" and args.algorithm == "pbt": - config[key] = read_tune_pbt(key, value) - elif args.mode == "tune": - config[key] = read_tune(value) - if args.mode == "tune": - config = apply_condition(config, data) - return config, sdc_file, fr_file - - -def parse_flow_variables(): - """ - Parse the flow variables from source - - Code: Makefile `vars` target output - - TODO: Tests. - - Output: - - flow_variables: set of flow variables - """ - cur_path = os.path.dirname(os.path.realpath(__file__)) - - # first, generate vars.tcl - makefile_path = os.path.join(cur_path, "../../../../flow/") - initial_path = os.path.abspath(os.getcwd()) - os.chdir(makefile_path) - result = subprocess.run(["make", "vars", f"PLATFORM={args.platform}"]) - if result.returncode != 0: - print(f"[ERROR TUN-0018] Makefile failed with error code {result.returncode}.") - sys.exit(1) - if not os.path.exists("vars.tcl"): - print(f"[ERROR TUN-0019] Makefile did not generate vars.tcl.") - sys.exit(1) - os.chdir(initial_path) - - # for code parsing, you need to parse from both scripts and vars.tcl file. - pattern = r"(?:::)?env\((.*?)\)" - files = glob.glob(os.path.join(cur_path, "../../../../flow/scripts/*.tcl")) - files.append(os.path.join(cur_path, "../../../../flow/vars.tcl")) - variables = set() - for file in files: - with open(file) as fp: - matches = re.findall(pattern, fp.read()) - for match in matches: - for variable in match.split("\n"): - variables.add(variable.strip().upper()) - return variables - - -def parse_config(config, path=os.getcwd()): - """ - Parse configuration received from tune into make variables. - """ - options = "" - sdc = {} - fast_route = {} - flow_variables = parse_flow_variables() - for key, value in config.items(): - # Keys that begin with underscore need special handling. - if key.startswith("_"): - # Variables to be injected into fastroute.tcl - if key.startswith("_FR_"): - fast_route[key.replace("_FR_", "", 1)] = value - # Variables to be injected into constraints.sdc - elif key.startswith("_SDC_"): - sdc[key.replace("_SDC_", "", 1)] = value - # Special substitution cases - elif key == "_PINS_DISTANCE": - options += f' PLACE_PINS_ARGS="-min_distance {value}"' - elif key == "_SYNTH_FLATTEN": - print( - "[WARNING TUN-0013] Non-flatten the designs are not " - "fully supported, ignoring _SYNTH_FLATTEN parameter." - ) - # Default case is VAR=VALUE - else: - # FIXME there is no robust way to get this metainformation from - # ORFS about the variables, so disable this code for now. - - # Sanity check: ignore all flow variables that are not tunable - # if key not in flow_variables: - # print(f"[ERROR TUN-0017] Variable {key} is not tunable.") - # sys.exit(1) - options += f" {key}={value}" - if bool(sdc): - write_sdc(sdc, path) - options += f" SDC_FILE={path}/{CONSTRAINTS_SDC}" - if bool(fast_route): - write_fast_route(fast_route, path) - options += f" FASTROUTE_TCL={path}/{FASTROUTE_TCL}" - return options - - -def write_sdc(variables, path): - """ - Create a SDC file with parameters for current tuning iteration. - """ - # Handle case where the reference file does not exist - if SDC_ORIGINAL == "": - print("[ERROR TUN-0020] No SDC reference file provided.") - sys.exit(1) - new_file = SDC_ORIGINAL - for key, value in variables.items(): - if key == "CLK_PERIOD": - if new_file.find("set clk_period") != -1: - new_file = re.sub( - r"set clk_period .*\n(.*)", f"set clk_period {value}\n\\1", new_file - ) - else: - new_file = re.sub( - r"-period [0-9\.]+ (.*)", f"-period {value} \\1", new_file - ) - new_file = re.sub(r"-waveform [{}\s0-9\.]+[\s|\n]", "", new_file) - elif key == "UNCERTAINTY": - if new_file.find("set uncertainty") != -1: - new_file = re.sub( - r"set uncertainty .*\n(.*)", - f"set uncertainty {value}\n\\1", - new_file, - ) - else: - new_file += f"\nset uncertainty {value}\n" - elif key == "IO_DELAY": - if new_file.find("set io_delay") != -1: - new_file = re.sub( - r"set io_delay .*\n(.*)", f"set io_delay {value}\n\\1", new_file - ) - else: - new_file += f"\nset io_delay {value}\n" - file_name = path + f"/{CONSTRAINTS_SDC}" - with open(file_name, "w") as file: - file.write(new_file) - return file_name - - -def write_fast_route(variables, path): - """ - Create a FastRoute Tcl file with parameters for current tuning iteration. - """ - # Handle case where the reference file does not exist (asap7 doesn't have reference) - if FR_ORIGINAL == "" and args.platform != "asap7": - print("[ERROR TUN-0021] No FastRoute Tcl reference file provided.") - sys.exit(1) - layer_cmd = "set_global_routing_layer_adjustment" - new_file = FR_ORIGINAL - for key, value in variables.items(): - if key.startswith("LAYER_ADJUST"): - layer = key.lstrip("LAYER_ADJUST") - # If there is no suffix (i.e., layer name) apply adjust to all - # layers. - if layer == "": - new_file += "\nset_global_routing_layer_adjustment" - new_file += " $::env(MIN_ROUTING_LAYER)" - new_file += "-$::env(MAX_ROUTING_LAYER)" - new_file += f" {value}" - elif re.search(f"{layer_cmd}.*{layer}", new_file): - new_file = re.sub( - f"({layer_cmd}.*{layer}).*\n(.*)", f"\\1 {value}\n\\2", new_file - ) - else: - new_file += f"\n{layer_cmd} {layer} {value}\n" - elif key == "GR_SEED": - new_file += f"\nset_global_routing_random -seed {value}\n" - file_name = path + f"/{FASTROUTE_TCL}" - with open(file_name, "w") as file: - file.write(new_file) - return file_name - - -def run_command(cmd, timeout=None, stderr_file=None, stdout_file=None, fail_fast=False): - """ - Wrapper for subprocess.run - Allows to run shell command, control print and exceptions. - """ - process = run( - cmd, timeout=timeout, capture_output=True, text=True, check=False, shell=True - ) - if stderr_file is not None and process.stderr != "": - with open(stderr_file, "a") as file: - file.write(f"\n\n{cmd}\n{process.stderr}") - if stdout_file is not None and process.stdout != "": - with open(stdout_file, "a") as file: - file.write(f"\n\n{cmd}\n{process.stdout}") - if args.verbose >= 1: - print(process.stderr) - if args.verbose >= 2: - print(process.stdout) - - if fail_fast and process.returncode != 0: - raise RuntimeError - - -@ray.remote -def openroad_distributed(repo_dir, config, path): - """Simple wrapper to run openroad distributed with Ray.""" - config = parse_config(config) - openroad(repo_dir, config, str(uuid()), path=path) - - -def openroad(base_dir, parameters, flow_variant, path=""): - """ - Run OpenROAD-flow-scripts with a given set of parameters. - """ - # Make sure path ends in a slash, i.e., is a folder - flow_variant = f"{args.experiment}/{flow_variant}" - if path != "": - log_path = f"{path}/{flow_variant}/" - report_path = log_path.replace("logs", "reports") - run_command(f"mkdir -p {log_path}") - run_command(f"mkdir -p {report_path}") - else: - log_path = report_path = os.getcwd() + "/" - - export_command = f"export PATH={INSTALL_PATH}/OpenROAD/bin" - export_command += f":{INSTALL_PATH}/yosys/bin:$PATH" - export_command += " && " - - make_command = export_command - make_command += f"make -C {base_dir}/flow DESIGN_CONFIG=designs/" - make_command += f"{args.platform}/{args.design}/config.mk" - make_command += f" PLATFORM={args.platform}" - make_command += f" FLOW_VARIANT={flow_variant} {parameters}" - make_command += f" EQUIVALENCE_CHECK=0" - make_command += f" NPROC={args.openroad_threads} SHELL=bash" - run_command( - make_command, - timeout=args.timeout, - stderr_file=f"{log_path}error-make-finish.log", - stdout_file=f"{log_path}make-finish-stdout.log", - ) - - metrics_file = os.path.join(report_path, "metrics.json") - metrics_command = export_command - metrics_command += f"{base_dir}/flow/util/genMetrics.py -x" - metrics_command += f" -v {flow_variant}" - metrics_command += f" -d {args.design}" - metrics_command += f" -p {args.platform}" - metrics_command += f" -o {metrics_file}" - run_command( - metrics_command, - stderr_file=f"{log_path}error-metrics.log", - stdout_file=f"{log_path}metrics-stdout.log", - ) - - return metrics_file - - -def clone(path): - """ - Clone base repo in the remote machine. Only used for Kubernetes at GCP. - """ - if args.git_clone: - run_command(f"rm -rf {path}") - if not os.path.isdir(f"{path}/.git"): - git_command = "git clone --depth 1 --recursive --single-branch" - git_command += f" {args.git_clone_args}" - git_command += f" --branch {args.git_orfs_branch}" - git_command += f" {args.git_url} {path}" - run_command(git_command) - - -def build(base, install): - """ - Build OpenROAD, Yosys and other dependencies. - """ - build_command = f'cd "{base}"' - if args.git_clean: - build_command += " && git clean -xdf tools" - build_command += " && git submodule foreach --recursive git clean -xdf" - if ( - args.git_clean - or not os.path.isfile(f"{install}/OpenROAD/bin/openroad") - or not os.path.isfile(f"{install}/yosys/bin/yosys") - ): - build_command += ' && bash -ic "./build_openroad.sh' - # Some GCP machines have 200+ cores. Let's be reasonable... - build_command += f" --local --nice --threads {min(32, cpu_count())}" - if args.git_latest: - build_command += " --latest" - build_command += f' {args.build_args}"' - run_command(build_command) - - -@ray.remote -def setup_repo(base): - """ - Clone ORFS repository and compile binaries. - """ - print(f"[INFO TUN-0000] Remote folder: {base}") - install = f"{base}/tools/install" - if args.server is not None: - clone(base) - build(base, install) - return install - - def parse_arguments(): """ Parse arguments from command line. @@ -789,7 +325,7 @@ def parse_arguments(): type=int, metavar="", default=1, - help="Number of CPUs to request for each tunning job.", + help="Number of CPUs to request for each tuning job.", ) tune_parser.add_argument( "--reference", @@ -966,17 +502,6 @@ def save_best(results): print(f"[INFO TUN-0003] Best parameters written to {new_best_path}") -@ray.remote -def consumer(queue): - """consumer""" - while not queue.empty(): - next_item = queue.get() - name = next_item[1] - print(f"[INFO TUN-0007] Scheduling run for parameter {name}.") - ray.get(openroad_distributed.remote(*next_item)) - print(f"[INFO TUN-0008] Finished run for parameter {name}.") - - def sweep(): """Run sweep of parameters""" if args.server is not None: @@ -985,7 +510,7 @@ def sweep(): # //// repo_dir = os.path.abspath(LOCAL_DIR + "/../" * 4) else: - repo_dir = os.path.abspath("../") + repo_dir = os.path.abspath(os.path.join(ORFS_FLOW_DIR, "..")) print(f"[INFO TUN-0012] Log folder {LOCAL_DIR}.") queue = Queue() parameter_list = list() @@ -1002,8 +527,9 @@ def sweep(): temp = dict() for value in parameter: temp.update(value) - print(temp) - queue.put([repo_dir, temp, LOCAL_DIR]) + queue.put( + [args, repo_dir, temp, LOCAL_DIR, SDC_ORIGINAL, FR_ORIGINAL, INSTALL_PATH] + ) workers = [consumer.remote(queue) for _ in range(args.jobs)] print("[INFO TUN-0009] Waiting for results.") ray.get(workers) @@ -1015,37 +541,11 @@ def sweep(): # Read config and original files before handling where to run in case we # need to upload the files. - config_dict, SDC_ORIGINAL, FR_ORIGINAL = read_config(os.path.abspath(args.config)) + config_dict, SDC_ORIGINAL, FR_ORIGINAL = read_config( + os.path.abspath(args.config), args.mode, getattr(args, "algorithm", None) + ) - # Connect to remote Ray server if any, otherwise will run locally - if args.server is not None: - # At GCP we have a NFS folder that is present for all worker nodes. - # This allows to build required binaries once. We clone, build and - # store intermediate files at LOCAL_DIR. - with open(args.config) as config_file: - LOCAL_DIR = "/shared-data/autotuner" - LOCAL_DIR += f"-orfs-{args.git_orfs_branch}" - if args.git_or_branch != "": - LOCAL_DIR += f"-or-{args.git_or_branch}" - if args.git_latest: - LOCAL_DIR += "-or-latest" - # Connect to ray server before first remote execution. - ray.init(f"ray://{args.server}:{args.port}") - # Remote functions return a task id and are non-blocking. Since we - # need the setup repo before continuing, we call ray.get() to wait - # for its completion. - INSTALL_PATH = ray.get(setup_repo.remote(LOCAL_DIR)) - LOCAL_DIR += f"/flow/logs/{args.platform}/{args.design}" - print("[INFO TUN-0001] NFS setup completed.") - else: - # For local runs, use the same folder as other ORFS utilities. - ORFS_FLOW_DIR = os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../../../flow") - ) - os.chdir(ORFS_FLOW_DIR) - LOCAL_DIR = f"logs/{args.platform}/{args.design}" - LOCAL_DIR = os.path.abspath(LOCAL_DIR) - INSTALL_PATH = os.path.abspath("../tools/install") + LOCAL_DIR, ORFS_FLOW_DIR, INSTALL_PATH = prepare_ray_server(args) if args.mode == "tune": best_params = set_best_params(args.platform, args.design) @@ -1053,7 +553,7 @@ def sweep(): TrainClass = set_training_class(args.eval) # PPAImprov requires a reference file to compute training scores. if args.eval == "ppa-improv": - reference = PPAImprov.read_metrics(args.reference) + reference = read_metrics(args.reference) tune_args = dict( name=args.experiment, diff --git a/tools/AutoTuner/src/autotuner/utils.py b/tools/AutoTuner/src/autotuner/utils.py new file mode 100644 index 0000000000..d82868466e --- /dev/null +++ b/tools/AutoTuner/src/autotuner/utils.py @@ -0,0 +1,649 @@ +import glob +import json +import os +import re +import subprocess +import sys +from multiprocessing import cpu_count +from datetime import datetime +from uuid import uuid4 as uuid +from time import time + +import numpy as np +import ray + +# Default scheme of a SDC constraints file +SDC_TEMPLATE = """ +set clk_name core_clock +set clk_port_name clk +set clk_period 2000 +set clk_io_pct 0.2 + +set clk_port [get_ports $clk_port_name] + +create_clock -name $clk_name -period $clk_period $clk_port + +set non_clock_inputs [lsearch -inline -all -not -exact [all_inputs] $clk_port] + +set_input_delay [expr $clk_period * $clk_io_pct] -clock $clk_name $non_clock_inputs +set_output_delay [expr $clk_period * $clk_io_pct] -clock $clk_name [all_outputs] +""" +# Name of the SDC file with constraints +CONSTRAINTS_SDC = "constraint.sdc" +# Name of the TCL script run before routing +FASTROUTE_TCL = "fastroute.tcl" +DATE = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + + +def write_sdc(variables, path, sdc_original, constraints_sdc): + """ + Create a SDC file with parameters for current tuning iteration. + """ + # Handle case where the reference file does not exist + if sdc_original == "": + print("[ERROR TUN-0020] No SDC reference file provided.") + sys.exit(1) + new_file = sdc_original + for key, value in variables.items(): + if key == "CLK_PERIOD": + if new_file.find("set clk_period") != -1: + new_file = re.sub( + r"set clk_period .*\n(.*)", f"set clk_period {value}\n\\1", new_file + ) + else: + new_file = re.sub( + r"-period [0-9\.]+ (.*)", f"-period {value} \\1", new_file + ) + new_file = re.sub(r"-waveform [{}\s0-9\.]+[\s|\n]", "", new_file) + elif key == "UNCERTAINTY": + if new_file.find("set uncertainty") != -1: + new_file = re.sub( + r"set uncertainty .*\n(.*)", + f"set uncertainty {value}\n\\1", + new_file, + ) + else: + new_file += f"\nset uncertainty {value}\n" + elif key == "IO_DELAY": + if new_file.find("set io_delay") != -1: + new_file = re.sub( + r"set io_delay .*\n(.*)", f"set io_delay {value}\n\\1", new_file + ) + else: + new_file += f"\nset io_delay {value}\n" + else: + print( + f"[WARN TUN-0025] {key} variable not supported in context of SDC files" + ) + continue + file_name = path + f"/{constraints_sdc}" + with open(file_name, "w") as file: + file.write(new_file) + return file_name + + +def write_fast_route(variables, path, platform, fr_original, fastroute_tcl): + """ + Create a FastRoute Tcl file with parameters for current tuning iteration. + """ + # Handle case where the reference file does not exist (asap7 doesn't have reference) + if fr_original == "" and platform != "asap7": + print("[ERROR TUN-0021] No FastRoute Tcl reference file provided.") + sys.exit(1) + layer_cmd = "set_global_routing_layer_adjustment" + new_file = fr_original + for key, value in variables.items(): + if key.startswith("LAYER_ADJUST"): + layer = key.lstrip("LAYER_ADJUST") + # If there is no suffix (i.e., layer name) apply adjust to all + # layers. + if layer == "": + new_file += "\nset_global_routing_layer_adjustment" + new_file += " $::env(MIN_ROUTING_LAYER)" + new_file += "-$::env(MAX_ROUTING_LAYER)" + new_file += f" {value}" + elif re.search(f"{layer_cmd}.*{layer}", new_file): + new_file = re.sub( + f"({layer_cmd}.*{layer}).*\n(.*)", f"\\1 {value}\n\\2", new_file + ) + else: + new_file += f"\n{layer_cmd} {layer} {value}\n" + elif key == "GR_SEED": + new_file += f"\nset_global_routing_random -seed {value}\n" + else: + print( + f"[WARN TUN-0028] {key} variable not supported in context of FastRoute TCL files" + ) + continue + file_name = path + f"/{fastroute_tcl}" + with open(file_name, "w") as file: + file.write(new_file) + return file_name + + +def parse_flow_variables(base_dir, platform): + """ + Parse the flow variables from source + - Code: Makefile `vars` target output + + Output: + - flow_variables: set of flow variables + """ + # TODO: Tests. + # first, generate vars.tcl + makefile_path = os.path.join(base_dir, "flow") + result = subprocess.run( + ["make", "-C", makefile_path, "vars", f"PLATFORM={platform}"], + capture_output=True, + ) + if result.returncode != 0: + print(f"[ERROR TUN-0018] Makefile failed with error code {result.returncode}.") + sys.exit(1) + if not os.path.exists(os.path.join(makefile_path, "vars.tcl")): + print("[ERROR TUN-0019] Makefile did not generate vars.tcl.") + sys.exit(1) + + # for code parsing, you need to parse from both scripts and vars.tcl file. + pattern = r"(?:::)?env\((.*?)\)" + files = glob.glob(os.path.join(makefile_path, "scripts/*.tcl")) + files.append(os.path.join(makefile_path, "vars.tcl")) + variables = set() + for file in files: + with open(file) as fp: + matches = re.findall(pattern, fp.read()) + for match in matches: + for variable in match.split("\n"): + variables.add(variable.strip().upper()) + return variables + + +def parse_config( + config, + base_dir, + platform, + sdc_original, + constraints_sdc, + fr_original, + fastroute_tcl, + path=os.getcwd(), +): + """ + Parse configuration received from tune into make variables. + """ + options = "" + sdc = {} + fast_route = {} + # flow_variables = parse_flow_variables(base_dir, platform) + for key, value in config.items(): + # Keys that begin with underscore need special handling. + if key.startswith("_"): + # Variables to be injected into fastroute.tcl + if key.startswith("_FR_"): + fast_route[key[4:]] = value + # Variables to be injected into constraints.sdc + elif key.startswith("_SDC_"): + sdc[key[5:]] = value + # Special substitution cases + elif key == "_PINS_DISTANCE": + options += f' PLACE_PINS_ARGS="-min_distance {value}"' + elif key == "_SYNTH_FLATTEN": + print( + "[WARNING TUN-0013] Non-flatten the designs are not " + "fully supported, ignoring _SYNTH_FLATTEN parameter." + ) + # Default case is VAR=VALUE + else: + # Sanity check: ignore all flow variables that are not tunable + # if key not in flow_variables: + # print(f"[ERROR TUN-0017] Variable {key} is not tunable.") + # sys.exit(1) + options += f" {key}={value}" + if sdc: + write_sdc(sdc, path, sdc_original, constraints_sdc) + options += f" SDC_FILE={path}/{constraints_sdc}" + if fast_route: + write_fast_route(fast_route, path, platform, fr_original, fastroute_tcl) + options += f" FASTROUTE_TCL={path}/{fastroute_tcl}" + return options + + +def run_command( + args, cmd, timeout=None, stderr_file=None, stdout_file=None, fail_fast=False +): + """ + Wrapper for subprocess.run + Allows to run shell command, control print and exceptions. + """ + process = subprocess.run( + cmd, timeout=timeout, capture_output=True, text=True, check=False, shell=True + ) + if stderr_file is not None and process.stderr != "": + with open(stderr_file, "a") as file: + file.write(f"\n\n{cmd}\n{process.stderr}") + if stdout_file is not None and process.stdout != "": + with open(stdout_file, "a") as file: + file.write(f"\n\n{cmd}\n{process.stdout}") + if args.verbose >= 1: + print(process.stderr) + if args.verbose >= 2: + print(process.stdout) + + if fail_fast and process.returncode != 0: + raise RuntimeError + + +def openroad( + args, + base_dir, + parameters, + flow_variant, + path="", + install_path=None, +): + """ + Run OpenROAD-flow-scripts with a given set of parameters. + """ + # Make sure path ends in a slash, i.e., is a folder + flow_variant = f"{args.experiment}/{flow_variant}" + if path != "": + log_path = f"{path}/{flow_variant}/" + report_path = log_path.replace("logs", "reports") + run_command(args, f"mkdir -p {log_path}") + run_command(args, f"mkdir -p {report_path}") + else: + log_path = report_path = os.getcwd() + "/" + + if install_path is None: + install_path = os.path.join(base_dir, "tools/install") + + export_command = f"export PATH={install_path}/OpenROAD/bin" + export_command += f":{install_path}/yosys/bin:$PATH" + export_command += " && " + + make_command = export_command + make_command += f"make -C {base_dir}/flow DESIGN_CONFIG=designs/" + make_command += f"{args.platform}/{args.design}/config.mk" + make_command += f" PLATFORM={args.platform}" + make_command += f" FLOW_VARIANT={flow_variant} {parameters}" + make_command += " EQUIVALENCE_CHECK=0" + make_command += f" NPROC={args.openroad_threads} SHELL=bash" + run_command( + args, + make_command, + timeout=args.timeout, + stderr_file=f"{log_path}error-make-finish.log", + stdout_file=f"{log_path}make-finish-stdout.log", + ) + + metrics_file = os.path.abspath(os.path.join(report_path, "metrics.json")) + metrics_command = export_command + metrics_command += f"{base_dir}/flow/util/genMetrics.py -x" + metrics_command += f" -v {flow_variant}" + metrics_command += f" -d {args.design}" + metrics_command += f" -p {args.platform}" + metrics_command += f" -o {metrics_file}" + run_command( + args, + metrics_command, + stderr_file=f"{log_path}error-metrics.log", + stdout_file=f"{log_path}metrics-stdout.log", + ) + + return metrics_file + + +def read_metrics(file_name): + """ + Collects metrics to evaluate the user-defined objective function. + """ + with open(file_name) as file: + data = json.load(file) + clk_period = 9999999 + worst_slack = "ERR" + wirelength = "ERR" + num_drc = "ERR" + total_power = "ERR" + core_util = "ERR" + final_util = "ERR" + design_area = "ERR" + die_area = "ERR" + core_area = "ERR" + for stage_name, value in data.items(): + if stage_name == "constraints" and len(value["clocks__details"]) > 0: + clk_period = float(value["clocks__details"][0].split()[1]) + if stage_name == "floorplan" and "design__instance__utilization" in value: + core_util = value["design__instance__utilization"] + if stage_name == "detailedroute" and "route__drc_errors" in value: + num_drc = value["route__drc_errors"] + if stage_name == "detailedroute" and "route__wirelength" in value: + wirelength = value["route__wirelength"] + if stage_name == "finish" and "timing__setup__ws" in value: + worst_slack = value["timing__setup__ws"] + if stage_name == "finish" and "power__total" in value: + total_power = value["power__total"] + if stage_name == "finish" and "design__instance__utilization" in value: + final_util = value["design__instance__utilization"] + if stage_name == "finish" and "design__instance__area" in value: + design_area = value["design__instance__area"] + if stage_name == "finish" and "design__core__area" in value: + core_area = value["design__core__area"] + if stage_name == "finish" and "design__die__area" in value: + die_area = value["design__die__area"] + ret = { + "clk_period": clk_period, + "worst_slack": worst_slack, + "total_power": total_power, + "core_util": core_util, + "final_util": final_util, + "design_area": design_area, + "core_area": core_area, + "die_area": die_area, + "wirelength": wirelength, + "num_drc": num_drc, + } + return ret + + +def read_config(file_name, mode, algorithm): + """ + Please consider inclusive, exclusive + Most type uses [min, max) + But, Quantization makes the upper bound inclusive. + e.g., qrandint and qlograndint uses [min, max] + step value is used for quantized type (e.g., quniform). Otherwise, write 0. + When min==max, it means the constant value + """ + + def read(path): + # if file path does not exist, return empty string + print(os.path.abspath(path)) + if not os.path.isfile(os.path.abspath(path)): + return "" + with open(os.path.abspath(path), "r") as file: + ret = file.read() + return ret + + def read_sweep(this): + return [*this["minmax"], this["step"]] + + def apply_condition(config, data): + from ray import tune + import random + + # TODO: tune.sample_from only supports random search algorithm. + # To make conditional parameter for the other algorithms, different + # algorithms should take different methods (will be added) + if algorithm != "random": + return config + dp_pad_min = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["minmax"][0] + dp_pad_step = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["step"] + if dp_pad_step == 1: + config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from( + lambda spec: np.random.randint( + dp_pad_min, spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1 + ) + ) + if dp_pad_step > 1: + config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from( + lambda spec: random.randrange( + dp_pad_min, + spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1, + dp_pad_step, + ) + ) + return config + + def read_tune(this): + from ray import tune + + min_, max_ = this["minmax"] + if min_ == max_: + # Returning a choice of a single element allow pbt algorithm to + # work. pbt does not accept single values as tunable. + return tune.choice([min_, max_]) + if this["type"] == "int": + if this["step"] == 1: + return tune.randint(min_, max_) + return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"]))) + if this["type"] == "float": + if this["step"] == 0: + return tune.uniform(min_, max_) + return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"]))) + return None + + def read_tune_ax(name, this): + """ + Ax format: https://ax.dev/versions/0.3.7/api/service.html + """ + from ray import tune + + dict_ = dict(name=name) + if "minmax" not in this: + return None + min_, max_ = this["minmax"] + if min_ == max_: + dict_["type"] = "fixed" + dict_["value"] = min_ + elif this["type"] == "int": + if this["step"] == 1: + dict_["type"] = "range" + dict_["bounds"] = [min_, max_] + dict_["value_type"] = "int" + else: + dict_["type"] = "choice" + dict_["values"] = tune.randint(min_, max_, this["step"]) + dict_["value_type"] = "int" + elif this["type"] == "float": + if this["step"] == 1: + dict_["type"] = "choice" + dict_["values"] = tune.choice( + np.ndarray.tolist(np.arange(min_, max_, this["step"])) + ) + dict_["value_type"] = "float" + else: + dict_["type"] = "range" + dict_["bounds"] = [min_, max_] + dict_["value_type"] = "float" + return dict_ + + def read_tune_pbt(name, this): + """ + PBT format: https://docs.ray.io/en/releases-2.9.3/tune/examples/pbt_guide.html + Note that PBT does not support step values. + """ + from ray import tune + + if "minmax" not in this: + return None + min_, max_ = this["minmax"] + if min_ == max_: + return tune.choice([min_, max_]) + if this["type"] == "int": + return tune.randint(min_, max_) + if this["type"] == "float": + return tune.uniform(min_, max_) + + # Check file exists and whether it is a valid JSON file. + assert os.path.isfile(file_name), f"File {file_name} not found." + try: + with open(file_name) as file: + data = json.load(file) + except json.JSONDecodeError: + raise ValueError(f"Invalid JSON file: {file_name}") + sdc_file = "" + fr_file = "" + if mode == "tune" and algorithm == "ax": + config = list() + else: + config = dict() + for key, value in data.items(): + if key == "best_result": + continue + if key == "_SDC_FILE_PATH" and value != "": + if sdc_file != "": + print("[WARNING TUN-0004] Overwriting SDC base file.") + sdc_file = read(f"{os.path.dirname(file_name)}/{value}") + continue + if key == "_FR_FILE_PATH" and value != "": + if fr_file != "": + print("[WARNING TUN-0005] Overwriting FastRoute base file.") + fr_file = read(f"{os.path.dirname(file_name)}/{value}") + continue + if not isinstance(value, dict): + # To take care of empty values like _FR_FILE_PATH + if mode == "tune" and algorithm == "ax": + param_dict = read_tune_ax(key, value) + if param_dict: + config.append(param_dict) + elif mode == "tune" and algorithm == "pbt": + param_dict = read_tune_pbt(key, value) + if param_dict: + config[key] = param_dict + else: + config[key] = value + elif mode == "sweep": + config[key] = read_sweep(value) + elif mode == "tune" and algorithm == "ax": + config.append(read_tune_ax(key, value)) + elif mode == "tune" and algorithm == "pbt": + config[key] = read_tune_pbt(key, value) + elif mode == "tune": + config[key] = read_tune(value) + if mode == "tune": + config = apply_condition(config, data) + return config, sdc_file, fr_file + + +def clone(args, path): + """ + Clone base repo in the remote machine. Only used for Kubernetes at GCP. + """ + if args.git_clone: + run_command(args, f"rm -rf {path}") + if not os.path.isdir(f"{path}/.git"): + git_command = "git clone --depth 1 --recursive --single-branch" + git_command += f" {args.git_clone_args}" + git_command += f" --branch {args.git_orfs_branch}" + git_command += f" {args.git_url} {path}" + run_command(args, git_command) + + +def build(args, base, install): + """ + Build OpenROAD, Yosys and other dependencies. + """ + build_command = f'cd "{base}"' + if args.git_clean: + build_command += " && git clean -xdf tools" + build_command += " && git submodule foreach --recursive git clean -xdf" + if ( + args.git_clean + or not os.path.isfile(f"{install}/OpenROAD/bin/openroad") + or not os.path.isfile(f"{install}/yosys/bin/yosys") + ): + build_command += ' && bash -ic "./build_openroad.sh' + # Some GCP machines have 200+ cores. Let's be reasonable... + build_command += f" --local --nice --threads {min(32, cpu_count())}" + if args.git_latest: + build_command += " --latest" + build_command += f' {args.build_args}"' + run_command(args, build_command) + + +@ray.remote +def setup_repo(args, base): + """ + Clone ORFS repository and compile binaries. + """ + print(f"[INFO TUN-0000] Remote folder: {base}") + install = f"{base}/tools/install" + if args.server is not None: + clone(base) + build(base, install) + return install + + +def prepare_ray_server(args): + """ + Prepares Ray server and returns basic directories. + """ + # Connect to remote Ray server if any, otherwise will run locally + if args.server is not None: + # At GCP we have a NFS folder that is present for all worker nodes. + # This allows to build required binaries once. We clone, build and + # store intermediate files at LOCAL_DIR. + with open(args.config) as config_file: + local_dir = "/shared-data/autotuner" + local_dir += f"-orfs-{args.git_orfs_branch}" + if args.git_or_branch != "": + local_dir += f"-or-{args.git_or_branch}" + if args.git_latest: + local_dir += "-or-latest" + # Connect to ray server before first remote execution. + ray.init(f"ray://{args.server}:{args.port}") + # Remote functions return a task id and are non-blocking. Since we + # need the setup repo before continuing, we call ray.get() to wait + # for its completion. + install_path = ray.get(setup_repo.remote(local_dir)) + orfs_flow_dir = os.path.join(local_dir, "flow") + local_dir += f"/flow/logs/{args.platform}/{args.design}" + print("[INFO TUN-0001] NFS setup completed.") + else: + orfs_dir = getattr(args, "orfs", None) + # For local runs, use the same folder as other ORFS utilities. + orfs_flow_dir = os.path.abspath( + os.path.join(orfs_dir, "flow") + if orfs_dir + else os.path.join(os.path.dirname(__file__), "../../../../flow") + ) + local_dir = f"logs/{args.platform}/{args.design}" + local_dir = os.path.join(orfs_flow_dir, local_dir) + install_path = os.path.abspath(os.path.join(orfs_flow_dir, "../tools/install")) + return local_dir, orfs_flow_dir, install_path + + +@ray.remote +def openroad_distributed( + args, + repo_dir, + config, + path, + sdc_original, + fr_original, + install_path, + variant=None, +): + """Simple wrapper to run openroad distributed with Ray.""" + config = parse_config( + config=config, + base_dir=repo_dir, + platform=args.platform, + sdc_original=sdc_original, + constraints_sdc=CONSTRAINTS_SDC, + fr_original=fr_original, + fastroute_tcl=FASTROUTE_TCL, + ) + if variant is None: + variant = config.replace(" ", "_").replace("=", "_") + t = time() + metric_file = openroad( + args=args, + base_dir=repo_dir, + parameters=config, + flow_variant=f"{uuid()}-{variant}", + path=path, + install_path=install_path, + ) + duration = time() - t + return metric_file, duration + + +@ray.remote +def consumer(queue): + """consumer""" + while not queue.empty(): + next_item = queue.get() + name = next_item[1] + print(f"[INFO TUN-0007] Scheduling run for parameter {name}.") + ray.get(openroad_distributed.remote(*next_item)) + print(f"[INFO TUN-0008] Finished run for parameter {name}.") From ba652d0728ad505765e526e9742ca09320d1655c Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 14:37:21 +0200 Subject: [PATCH 03/10] tools: Autotuner: Update smoke tests to use AutoTuner as module Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/test/smoke_test_algo_eval.py | 42 +++++-------------- .../test/smoke_test_sample_iteration.py | 30 ++++--------- tools/AutoTuner/test/smoke_test_sweep.py | 31 ++++---------- tools/AutoTuner/test/smoke_test_tune.py | 27 +++--------- 4 files changed, 34 insertions(+), 96 deletions(-) diff --git a/tools/AutoTuner/test/smoke_test_algo_eval.py b/tools/AutoTuner/test/smoke_test_algo_eval.py index a695489b48..d5c0a1d4a5 100644 --- a/tools/AutoTuner/test/smoke_test_algo_eval.py +++ b/tools/AutoTuner/test/smoke_test_algo_eval.py @@ -3,9 +3,7 @@ import os cur_dir = os.path.dirname(os.path.abspath(__file__)) -src_dir = os.path.join(cur_dir, "../src/autotuner") orfs_dir = os.path.join(cur_dir, "../../../flow") -os.chdir(src_dir) class BaseAlgoEvalSmokeTest(unittest.TestCase): @@ -22,7 +20,7 @@ def setUp(self): _eval = ["default", "ppa-improv"] self.matrix = [(a, e) for a in _algo for e in _eval] self.commands = [ - f"python3 distributed.py" + f"python3 -m autotuner.distributed" f" --design {self.design}" f" --platform {self.platform}" f" --experiment {self.experiment}" @@ -34,23 +32,18 @@ def setUp(self): ] def make_base(self): - os.chdir(orfs_dir) commands = [ - f"make DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk clean_all", - f"make DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk EQUIVALENCE_CHECK=0", - f"make DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk update_metadata_autotuner", + f"make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk clean_all", + f"make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk EQUIVALENCE_CHECK=0", + f"make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk update_metadata_autotuner", ] for command in commands: out = subprocess.run(command, shell=True, check=True) self.assertTrue(out.returncode == 0) - os.chdir(src_dir) - - -class ASAP7AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest): - platform = "asap7" - design = "gcd" def test_algo_eval(self): + if not (self.platform and self.design): + raise unittest.SkipTest("Platform and design have to be defined") # Run `make` to get baseline metrics (metadata-base-ok.json) self.make_base() for command in self.commands: @@ -60,33 +53,20 @@ def test_algo_eval(self): self.assertTrue(successful) +class ASAP7AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest): + platform = "asap7" + design = "gcd" + + class IHPSG13G2AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest): platform = "ihp-sg13g2" design = "gcd" - def test_algo_eval(self): - # Run `make` to get baseline metrics (metadata-base-ok.json) - self.make_base() - for command in self.commands: - print(command) - out = subprocess.run(command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - class SKY130HDAlgoEvalSmokeTest(BaseAlgoEvalSmokeTest): platform = "sky130hd" design = "gcd" - def test_algo_eval(self): - # Run `make` to get baseline metrics (metadata-base-ok.json) - self.make_base() - for command in self.commands: - print(command) - out = subprocess.run(command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - if __name__ == "__main__": unittest.main() diff --git a/tools/AutoTuner/test/smoke_test_sample_iteration.py b/tools/AutoTuner/test/smoke_test_sample_iteration.py index f49c22a088..2991eed1aa 100644 --- a/tools/AutoTuner/test/smoke_test_sample_iteration.py +++ b/tools/AutoTuner/test/smoke_test_sample_iteration.py @@ -3,8 +3,6 @@ import os cur_dir = os.path.dirname(os.path.abspath(__file__)) -src_dir = os.path.join(cur_dir, "../src/autotuner") -os.chdir(src_dir) class BaseSampleIterationSmokeTest(unittest.TestCase): @@ -19,7 +17,7 @@ def setUp(self): self.experiment = f"smoke-test-sample-iteration-{self.platform}" self.matrix = [(5, 1), (1, 5), (2, 2), (1, 1)] self.commands = [ - f"python3 distributed.py" + f"python3 -m autotuner.distributed" f" --design {self.design}" f" --platform {self.platform}" f" --experiment {self.experiment}" @@ -28,12 +26,9 @@ def setUp(self): for s, i in self.matrix ] - -class ASAP7SampleIterationSmokeTest(BaseSampleIterationSmokeTest): - platform = "asap7" - design = "gcd" - def test_sample_iteration(self): + if not (self.platform and self.design): + raise unittest.SkipTest("Platform and design have to be defined") for command in self.commands: print(command) out = subprocess.run(command, shell=True, check=True) @@ -41,29 +36,20 @@ def test_sample_iteration(self): self.assertTrue(successful) +class ASAP7SampleIterationSmokeTest(BaseSampleIterationSmokeTest): + platform = "asap7" + design = "gcd" + + class SKY130HDSampleIterationSmokeTest(BaseSampleIterationSmokeTest): platform = "sky130hd" design = "gcd" - def test_sample_iteration(self): - for command in self.commands: - print(command) - out = subprocess.run(command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - class IHPSG13G2SampleIterationSmokeTest(BaseSampleIterationSmokeTest): platform = "ihp-sg13g2" design = "gcd" - def test_sample_iteration(self): - for command in self.commands: - print(command) - out = subprocess.run(command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - if __name__ == "__main__": unittest.main() diff --git a/tools/AutoTuner/test/smoke_test_sweep.py b/tools/AutoTuner/test/smoke_test_sweep.py index 7a1b013911..209d2828d2 100644 --- a/tools/AutoTuner/test/smoke_test_sweep.py +++ b/tools/AutoTuner/test/smoke_test_sweep.py @@ -4,8 +4,6 @@ import json cur_dir = os.path.dirname(os.path.abspath(__file__)) -src_dir = os.path.join(cur_dir, "../src/autotuner") -os.chdir(src_dir) class BaseSweepSmokeTest(unittest.TestCase): @@ -13,7 +11,9 @@ class BaseSweepSmokeTest(unittest.TestCase): design = "" def setUp(self): - self.config = "distributed-sweep-example.json" + self.config = os.path.abspath( + os.path.join(cur_dir, "../src/autotuner/distributed-sweep-example.json") + ) # make sure this json only has 1 key called "CTS_CLUSTER_SIZE" and 4 possible values with open(self.config) as f: contents = json.load(f) @@ -31,7 +31,7 @@ def setUp(self): self.jobs = 4 if core >= 4 else core self.experiment = f"smoke-test-sweep-{self.platform}" self.command = ( - "python3 distributed.py" + "python3 -m autotuner.distributed" f" --design {self.design}" f" --platform {self.platform}" f" --experiment {self.experiment}" @@ -41,40 +41,27 @@ def setUp(self): ) def test_sweep(self): - raise NotImplementedError( - "This method needs to be implemented in the derivative classes." - ) + if not (self.platform and self.design): + raise unittest.SkipTest("Platform and design have to be defined") + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) class ASAP7SweepSmokeTest(BaseSweepSmokeTest): platform = "asap7" design = "gcd" - def test_sweep(self): - out = subprocess.run(self.command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - class SKY130HDSweepSmokeTest(BaseSweepSmokeTest): platform = "sky130hd" design = "gcd" - def test_sweep(self): - out = subprocess.run(self.command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - class IHPSG13G2SweepSmokeTest(BaseSweepSmokeTest): platform = "ihp-sg13g2" design = "gcd" - def test_sweep(self): - out = subprocess.run(self.command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - if __name__ == "__main__": unittest.main() diff --git a/tools/AutoTuner/test/smoke_test_tune.py b/tools/AutoTuner/test/smoke_test_tune.py index 9710416745..4ddeafa0cc 100644 --- a/tools/AutoTuner/test/smoke_test_tune.py +++ b/tools/AutoTuner/test/smoke_test_tune.py @@ -3,8 +3,6 @@ import os cur_dir = os.path.dirname(os.path.abspath(__file__)) -src_dir = os.path.join(cur_dir, "../src/autotuner") -os.chdir(src_dir) class BaseTuneSmokeTest(unittest.TestCase): @@ -18,7 +16,7 @@ def setUp(self): ) self.experiment = f"smoke-test-tune-{self.platform}" self.command = ( - "python3 distributed.py" + "python3 -m autotuner.distributed" f" --design {self.design}" f" --platform {self.platform}" f" --experiment {self.experiment}" @@ -27,40 +25,27 @@ def setUp(self): ) def test_tune(self): - raise NotImplementedError( - "This method needs to be implemented in the derivative classes." - ) + if not (self.platform and self.design): + raise unittest.SkipTest("Platform and design have to be defined") + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) class ASAP7TuneSmokeTest(BaseTuneSmokeTest): platform = "asap7" design = "gcd" - def test_tune(self): - out = subprocess.run(self.command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - class SKY130HDTuneSmokeTest(BaseTuneSmokeTest): platform = "sky130hd" design = "gcd" - def test_tune(self): - out = subprocess.run(self.command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - class IHPSG13G2TuneSmokeTest(BaseTuneSmokeTest): platform = "ihp-sg13g2" design = "gcd" - def test_tune(self): - out = subprocess.run(self.command, shell=True, check=True) - successful = out.returncode == 0 - self.assertTrue(successful) - if __name__ == "__main__": unittest.main() From fe29652d7d026b0794ab00f6defcd1d38349928c Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 14:52:03 +0200 Subject: [PATCH 04/10] docs: user: Update installation and running commands for AutoTuner Signed-off-by: Eryk Szpotanski --- docs/user/InstructionsForAutoTuner.md | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/user/InstructionsForAutoTuner.md b/docs/user/InstructionsForAutoTuner.md index 0344379a06..b3bf6826ab 100644 --- a/docs/user/InstructionsForAutoTuner.md +++ b/docs/user/InstructionsForAutoTuner.md @@ -27,10 +27,6 @@ We have provided two convenience scripts, `./installer.sh` and `./setup.sh` that works in Python3.8 for installation and configuration of AutoTuner, as shown below: -```{note} -Make sure you run the following commands in the ORFS root directory. -``` - ```shell # Install prerequisites ./tools/AutoTuner/installer.sh @@ -104,7 +100,7 @@ For Global Routing parameters that are set on `fastroute.tcl` you can use: ### General Information -The `distributed.py` script located in `./tools/AutoTuner/src/autotuner` uses [Ray's](https://docs.ray.io/en/latest/index.html) job scheduling and management to +The `autotuner.distributed` module uses [Ray's](https://docs.ray.io/en/latest/index.html) job scheduling and management to fully utilize available hardware resources from a single server configuration, on-premise or over the cloud with multiple CPUs. @@ -115,35 +111,37 @@ The two modes of operation: The `sweep` mode is useful when we want to isolate or test a single or very few parameters. On the other hand, `tune` is more suitable for finding the best combination of a complex and large number of flow -parameters. Both modes rely on user-specified search space that is -defined by a `.json` file, they use the same syntax and format, -though some features may not be available for sweeping. +parameters. ```{note} The order of the parameters matter. Arguments `--design`, `--platform` and `--config` are always required and should precede *mode*. ``` +```{note} +The following commands should be run from `./tools/AutoTuner`. +``` + #### Tune only -* AutoTuner: `python3 distributed.py tune -h` +* AutoTuner: `python3 -m autotuner.distributed tune -h` Example: ```shell -python3 distributed.py --design gcd --platform sky130hd \ - --config ../../../../flow/designs/sky130hd/gcd/autotuner.json \ +python3 -m autotuner.distributed --design gcd --platform sky130hd \ + --config ../../flow/designs/sky130hd/gcd/autotuner.json \ tune --samples 5 ``` #### Sweep only -* Parameter sweeping: `python3 distributed.py sweep -h` +* Parameter sweeping: `python3 -m autotuner.distributed sweep -h` Example: ```shell -python3 distributed.py --design gcd --platform sky130hd \ - --config distributed-sweep-example.json \ +python3 -m autotuner.distributed --design gcd --platform sky130hd \ + --config src/autotuner/distributed-sweep-example.json \ sweep ``` From 23bb08c891ffed21e6f066876041d04c60b684c5 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Fri, 11 Oct 2024 15:11:12 +0200 Subject: [PATCH 05/10] flow: scripts: detail_route: Report metrics Signed-off-by: Eryk Szpotanski --- flow/scripts/detail_route.tcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flow/scripts/detail_route.tcl b/flow/scripts/detail_route.tcl index acd6377fa0..771e219b1e 100644 --- a/flow/scripts/detail_route.tcl +++ b/flow/scripts/detail_route.tcl @@ -68,6 +68,8 @@ if { [env_var_exists_and_non_empty POST_DETAIL_ROUTE_TCL] } { check_antennas -report_file $env(REPORTS_DIR)/drt_antennas.log +report_metrics 5 "detailed route" + if {![design_is_routed]} { error "Design has unrouted nets." } From 07936bb7b041ada6551a4f1daf3f59ee0e354f8a Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:27:56 +0200 Subject: [PATCH 06/10] tools: Autotuner: Add dependencies for Vizier Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/installer.sh | 13 ++++++++++++- tools/AutoTuner/pyproject.toml | 2 ++ tools/AutoTuner/requirements-ray.txt | 11 +++++++++++ tools/AutoTuner/requirements-vizier.txt | 3 +++ tools/AutoTuner/requirements.txt | 12 +----------- 5 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 tools/AutoTuner/requirements-ray.txt create mode 100644 tools/AutoTuner/requirements-vizier.txt diff --git a/tools/AutoTuner/installer.sh b/tools/AutoTuner/installer.sh index 7d5f22f5fd..27cfbd77fc 100755 --- a/tools/AutoTuner/installer.sh +++ b/tools/AutoTuner/installer.sh @@ -3,9 +3,20 @@ # Get the directory where the script is located script_dir="$(dirname "${BASH_SOURCE[0]}")" +dependencies="" +if [[ "$#" -eq 0 ]]; then + echo "Installing dependencies for Ray Tune and Vizier" + dependencies="ray,vizier" +elif [[ "$#" -ne 1 ]] || ([[ "$1" != "ray" ]] && [[ "$1" != "vizier" ]]); then + echo "Please specify whether 'ray' or 'vizier' dependencies should be installed" >&2 + exit 1 +else + dependencies="$1" +fi + # Define the virtual environment name venv_name="autotuner_env" python3 -m venv "$script_dir/$venv_name" source "$script_dir/$venv_name/bin/activate" -pip3 install -e "$script_dir" +pip3 install -e "$script_dir[$dependencies]" deactivate diff --git a/tools/AutoTuner/pyproject.toml b/tools/AutoTuner/pyproject.toml index 3261ae831e..937803cbae 100644 --- a/tools/AutoTuner/pyproject.toml +++ b/tools/AutoTuner/pyproject.toml @@ -12,6 +12,8 @@ dynamic = ["dependencies", "optional-dependencies"] [tool.setuptools.dynamic] dependencies = { file = ["requirements.txt"] } +optional-dependencies.ray = { file = ["requirements-ray.txt"] } +optional-dependencies.vizier = { file = ["requirements-vizier.txt"] } optional-dependencies.dev = { file = ["requirements-dev.txt"] } [build-system] diff --git a/tools/AutoTuner/requirements-ray.txt b/tools/AutoTuner/requirements-ray.txt new file mode 100644 index 0000000000..5a09397ad0 --- /dev/null +++ b/tools/AutoTuner/requirements-ray.txt @@ -0,0 +1,11 @@ +ray[tune]==2.9.3 +ax-platform>=0.3.3,<=0.3.7 +hyperopt==0.2.7 +optuna==3.6.0 +pandas>=2.0,<=2.2.1 +bayesian-optimization==1.4.0 +colorama==0.4.6 +tensorboard>=2.14.0,<=2.16.2 +protobuf==3.20.3 +SQLAlchemy==1.4.17 +urllib3<=1.26.15 diff --git a/tools/AutoTuner/requirements-vizier.txt b/tools/AutoTuner/requirements-vizier.txt new file mode 100644 index 0000000000..0222aeddb4 --- /dev/null +++ b/tools/AutoTuner/requirements-vizier.txt @@ -0,0 +1,3 @@ +jax<=0.4.33 +google-vizier[jax] +tqdm diff --git a/tools/AutoTuner/requirements.txt b/tools/AutoTuner/requirements.txt index 5bf65305cc..e598feeda0 100644 --- a/tools/AutoTuner/requirements.txt +++ b/tools/AutoTuner/requirements.txt @@ -1,11 +1 @@ -ray[default,tune]==2.9.3 -ax-platform>=0.3.3,<=0.3.7 -hyperopt==0.2.7 -optuna==3.6.0 -pandas>=2.0,<=2.2.1 -bayesian-optimization==1.4.0 -colorama==0.4.6 -tensorboard>=2.14.0,<=2.16.2 -protobuf==3.20.3 -SQLAlchemy==1.4.17 -urllib3<=1.26.15 +ray[default]==2.9.3 From 75c72a28c668d8158427ecf7f2cd10f545bd6d43 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:37:07 +0200 Subject: [PATCH 07/10] tools: Autotuner: Implement Vizier support Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/src/autotuner/distributed.py | 127 +---- tools/AutoTuner/src/autotuner/utils.py | 171 +++++++ tools/AutoTuner/src/autotuner/vizier.py | 497 +++++++++++++++++++ 3 files changed, 670 insertions(+), 125 deletions(-) create mode 100644 tools/AutoTuner/src/autotuner/vizier.py diff --git a/tools/AutoTuner/src/autotuner/distributed.py b/tools/AutoTuner/src/autotuner/distributed.py index c0e54ef45b..7aae0ba0f7 100644 --- a/tools/AutoTuner/src/autotuner/distributed.py +++ b/tools/AutoTuner/src/autotuner/distributed.py @@ -31,7 +31,6 @@ import random from itertools import product from collections import namedtuple -from multiprocessing import cpu_count import numpy as np import torch @@ -50,6 +49,7 @@ from ax.service.ax_client import AxClient from autotuner.utils import ( + add_common_args, openroad, consumer, parse_config, @@ -69,8 +69,6 @@ ORFS_FLOW_DIR = os.path.abspath( os.path.join(os.path.dirname(__file__), "../../../../flow") ) -# URL to ORFS GitHub repository -ORFS_URL = "https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts" class AutoTunerBase(tune.Trainable): @@ -194,30 +192,9 @@ def parse_arguments(): tune_parser = subparsers.add_parser("tune") _ = subparsers.add_parser("sweep") - # DUT - parser.add_argument( - "--design", - type=str, - metavar="", - required=True, - help="Name of the design for Autotuning.", - ) - parser.add_argument( - "--platform", - type=str, - metavar="", - required=True, - help="Name of the platform for Autotuning.", - ) + add_common_args(parser) # Experiment Setup - parser.add_argument( - "--config", - type=str, - metavar="", - required=True, - help="Configuration file that sets which knobs to use for Autotuning.", - ) parser.add_argument( "--experiment", type=str, @@ -226,71 +203,10 @@ def parse_arguments(): help="Experiment name. This parameter is used to prefix the" " FLOW_VARIANT and to set the Ray log destination.", ) - parser.add_argument( - "--timeout", - type=float, - metavar="", - default=None, - help="Time limit (in hours) for each trial run. Default is no limit.", - ) tune_parser.add_argument( "--resume", action="store_true", help="Resume previous run." ) - # Setup - parser.add_argument( - "--git_clean", - action="store_true", - help="Clean binaries and build files." - " WARNING: may lose previous data." - " Use carefully.", - ) - parser.add_argument( - "--git_clone", - action="store_true", - help="Force new git clone." - " WARNING: may lose previous data." - " Use carefully.", - ) - parser.add_argument( - "--git_clone_args", - type=str, - metavar="", - default="", - help="Additional git clone arguments.", - ) - parser.add_argument( - "--git_latest", action="store_true", help="Use latest version of OpenROAD app." - ) - parser.add_argument( - "--git_or_branch", - type=str, - metavar="", - default="", - help="OpenROAD app branch to use.", - ) - parser.add_argument( - "--git_orfs_branch", - type=str, - metavar="", - default="master", - help="OpenROAD-flow-scripts branch to use.", - ) - parser.add_argument( - "--git_url", - type=str, - metavar="", - default=ORFS_URL, - help="OpenROAD-flow-scripts repo URL to use.", - ) - parser.add_argument( - "--build_args", - type=str, - metavar="", - default="", - help="Additional arguments given to ./build_openroad.sh.", - ) - # ML tune_parser.add_argument( "--algorithm", @@ -349,45 +265,6 @@ def parse_arguments(): help="Random seed. (0 means no seed.)", ) - # Workload - parser.add_argument( - "--jobs", - type=int, - metavar="", - default=int(np.floor(cpu_count() / 2)), - help="Max number of concurrent jobs.", - ) - parser.add_argument( - "--openroad_threads", - type=int, - metavar="", - default=16, - help="Max number of threads openroad can use.", - ) - parser.add_argument( - "--server", - type=str, - metavar="", - default=None, - help="The address of Ray server to connect.", - ) - parser.add_argument( - "--port", - type=int, - metavar="", - default=10001, - help="The port of Ray server to connect.", - ) - - parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="Verbosity level.\n\t0: only print Ray status\n\t1: also print" - " training stderr\n\t2: also print training stdout.", - ) - arguments = parser.parse_args() if arguments.mode == "tune": arguments.algorithm = arguments.algorithm.lower() diff --git a/tools/AutoTuner/src/autotuner/utils.py b/tools/AutoTuner/src/autotuner/utils.py index d82868466e..e1e46719dc 100644 --- a/tools/AutoTuner/src/autotuner/utils.py +++ b/tools/AutoTuner/src/autotuner/utils.py @@ -1,3 +1,4 @@ +import argparse import glob import json import os @@ -32,6 +33,8 @@ CONSTRAINTS_SDC = "constraint.sdc" # Name of the TCL script run before routing FASTROUTE_TCL = "fastroute.tcl" +# URL to ORFS GitHub repository +ORFS_URL = "https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts" DATE = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") @@ -292,6 +295,27 @@ def openroad( return metrics_file +STAGES = list( + enumerate( + [ + "synth", + "floorplan", + "floorplan_io", + "floorplan_tdms", + "floorplan_macro", + "floorplan_tap", + "floorplan_pdn", + "globalplace", + "detailedplace", + "cts", + "globalroute", + "detailedroute", + "finish", + ] + ) +) + + def read_metrics(file_name): """ Collects metrics to evaluate the user-defined objective function. @@ -308,6 +332,7 @@ def read_metrics(file_name): design_area = "ERR" die_area = "ERR" core_area = "ERR" + last_stage = -1 for stage_name, value in data.items(): if stage_name == "constraints" and len(value["clocks__details"]) > 0: clk_period = float(value["clocks__details"][0].split()[1]) @@ -329,6 +354,10 @@ def read_metrics(file_name): core_area = value["design__core__area"] if stage_name == "finish" and "design__die__area" in value: die_area = value["design__die__area"] + for i, stage_name in reversed(STAGES): + if stage_name in data and [d for d in data[stage_name].values() if d != "ERR"]: + last_stage = i + break ret = { "clk_period": clk_period, "worst_slack": worst_slack, @@ -463,6 +492,20 @@ def read_tune_pbt(name, this): if this["type"] == "float": return tune.uniform(min_, max_) + def read_vizier(this): + dict_ = {} + min_, max_ = this["minmax"] + dict_["value"] = (min_, max_) + if "scale_type" in this: + dict_["scale_type"] = this["scale_type"] + if min_ == max_: + dict_["type"] = "fixed" + elif this["type"] == "int": + dict_["type"] = "int" + elif this["type"] == "float": + dict_["type"] = "float" + return dict_ + # Check file exists and whether it is a valid JSON file. assert os.path.isfile(file_name), f"File {file_name} not found." try: @@ -509,6 +552,8 @@ def read_tune_pbt(name, this): config[key] = read_tune_pbt(key, value) elif mode == "tune": config[key] = read_tune(value) + elif mode == "vizier": + config[key] = read_vizier(value) if mode == "tune": config = apply_condition(config, data) return config, sdc_file, fr_file @@ -647,3 +692,129 @@ def consumer(queue): print(f"[INFO TUN-0007] Scheduling run for parameter {name}.") ray.get(openroad_distributed.remote(*next_item)) print(f"[INFO TUN-0008] Finished run for parameter {name}.") + + +def add_common_args(parser: argparse.ArgumentParser): + # DUT + parser.add_argument( + "--design", + type=str, + metavar="", + required=True, + help="Name of the design for Autotuning.", + ) + parser.add_argument( + "--platform", + type=str, + metavar="", + required=True, + help="Name of the platform for Autotuning.", + ) + # Experiment Setup + parser.add_argument( + "--config", + type=str, + metavar="", + required=True, + help="Configuration file that sets which knobs to use for Autotuning.", + ) + parser.add_argument( + "--timeout", + type=float, + metavar="", + default=None, + help="Time limit (in hours) for each trial run. Default is no limit.", + ) + # Workload + parser.add_argument( + "--openroad_threads", + type=int, + metavar="", + default=16, + help="Max number of threads openroad can use.", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Verbosity level.\n\t0: only print status\n\t1: also print" + " training stderr\n\t2: also print training stdout.", + ) + + # Setup + parser.add_argument( + "--git_clean", + action="store_true", + help="Clean binaries and build files." + " WARNING: may lose previous data." + " Use carefully.", + ) + parser.add_argument( + "--git_clone", + action="store_true", + help="Force new git clone." + " WARNING: may lose previous data." + " Use carefully.", + ) + parser.add_argument( + "--git_clone_args", + type=str, + metavar="", + default="", + help="Additional git clone arguments.", + ) + parser.add_argument( + "--git_latest", action="store_true", help="Use latest version of OpenROAD app." + ) + parser.add_argument( + "--git_or_branch", + type=str, + metavar="", + default="", + help="OpenROAD app branch to use.", + ) + parser.add_argument( + "--git_orfs_branch", + type=str, + metavar="", + default="master", + help="OpenROAD-flow-scripts branch to use.", + ) + parser.add_argument( + "--git_url", + type=str, + metavar="", + default=ORFS_URL, + help="OpenROAD-flow-scripts repo URL to use.", + ) + parser.add_argument( + "--build_args", + type=str, + metavar="", + default="", + help="Additional arguments given to ./build_openroad.sh.", + ) + + # Workload + parser.add_argument( + "--jobs", + type=int, + metavar="", + default=int(np.floor(cpu_count() / 2)), + help="Max number of concurrent jobs.", + ) + parser.add_argument( + "--server", + type=str, + metavar="", + default=None, + help="The address of Ray server to connect.", + ) + parser.add_argument( + "--port", + type=int, + metavar="", + default=10001, + help="The port of Ray server to connect.", + ) diff --git a/tools/AutoTuner/src/autotuner/vizier.py b/tools/AutoTuner/src/autotuner/vizier.py new file mode 100644 index 0000000000..225df742ed --- /dev/null +++ b/tools/AutoTuner/src/autotuner/vizier.py @@ -0,0 +1,497 @@ +import argparse +import json +import sys +import traceback +from pathlib import Path +from typing import Dict + +import ray +from tqdm import tqdm +from vizier import service +from vizier.service import clients, servers +from vizier.service import pyvizier as vz + +from autotuner.utils import ( + DATE, + add_common_args, + openroad_distributed, + read_config, + read_metrics, + prepare_ray_server, +) + +# Path to the ORFS base directory +ORFS = list(Path(__file__).absolute().parents)[4] +# Maps metrics to a goal (min or max) +METRIC_TO_GOAL = { + "worst_slack": vz.ObjectiveMetricGoal.MAXIMIZE, + "clk_period-worst_slack": vz.ObjectiveMetricGoal.MINIMIZE, + "total_power": vz.ObjectiveMetricGoal.MINIMIZE, + "core_util": vz.ObjectiveMetricGoal.MAXIMIZE, + "final_util": vz.ObjectiveMetricGoal.MAXIMIZE, + "design_area": vz.ObjectiveMetricGoal.MINIMIZE, + "core_area": vz.ObjectiveMetricGoal.MINIMIZE, + "die_area": vz.ObjectiveMetricGoal.MINIMIZE, + "last_successful_stage": vz.ObjectiveMetricGoal.MAXIMIZE, +} +# Maps goal to a worst value +GOAL_TO_VALUE = { + vz.ObjectiveMetricGoal.MINIMIZE: float("inf"), + vz.ObjectiveMetricGoal.MAXIMIZE: float("-inf"), +} +# Maps string to Vizier ScaleType +MAP_SCALE_TYPE = { + "linear": vz.ScaleType.LINEAR, + "log": vz.ScaleType.LOG, + "rlog": vz.ScaleType.REVERSE_LOG, +} + + +def evaluate(args: argparse.Namespace, metric_file: str) -> Dict[str, float]: + """ + Runs ORFS and calculates metrics. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + metric_file : str + Path to the file with metrics + + Returns + ------- + Dict[str, float] + Dictionary with metrics + """ + try: + metrics = read_metrics(metric_file) + # Calculate difference of clock period and worst slack + if metrics["clk_period"] != 9999999 and metrics["worst_slack"] != "ERR": + metrics["clk_period-worst_slack"] = ( + metrics["clk_period"] - metrics["worst_slack"] + ) + else: + metrics["clk_period-worst_slack"] = "ERR" + + # Copy and normalize metrics + results = {} + for metric in args.use_metrics: + value = metrics[metric] + results[metric] = ( + float(value) + if value != "ERR" + else GOAL_TO_VALUE[METRIC_TO_GOAL[metric]] + ) + if results["last_successful_stage"] <= 6 and results["core_util"] < float( + "inf" + ): + # Invert core util, as for smaller values design should be easier to built + results["core_util"] *= -1 + return results + except Exception as ex: + print( + f"[ERROR TUN-0023] Exception during metrics processing {args.design}: {ex}", + file=sys.stderr, + ) + print("\n".join(traceback.format_tb(ex.__traceback__)), file=sys.stderr) + results = {} + for metric, goal in args.use_metrics: + results[metric] = GOAL_TO_VALUE[METRIC_TO_GOAL[metric]] + return results + + +@ray.remote +def parallel_evaluate( + args: argparse.Namespace, + suggestion: Dict, + i: int, + s: int, + install_path: Path, +) -> Dict: + """ + Wrapper for evaluate, run in thread pool. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + suggestion : Dict + i : int + Number of iteration + s : int + Number of suggestion + install_path : Path + Path to the install directory with ORFS binaries + + Returns + ------- + Dict + Results of evaluation with additional data + """ + variant = f"variant-{i}-{s}" + metric_file, duration = ray.get( + openroad_distributed.remote( + args=args, + repo_dir=str(args.orfs), + config=suggestion, + path=f"logs/{args.platform}/{args.design}", + sdc_original=args.sdc_file, + fr_original=args.fr_file, + install_path=str(install_path), + variant=variant, + ) + ) + objective = evaluate(args, metric_file) + return { + "iterations": i, + "suggestion": s, + "params": suggestion, + "evaluation": objective, + "variant": variant, + "duration": duration, + } + + +def register_param( + args: argparse.Namespace, problem: vz.ProblemStatement, name: str, conf: Dict +): + """ + Registers parameters in Vizier problem statement. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + problem : vz.ProblemStatement + Vizier problem statement + name : str + Name of the parameter + conf : Dict + Parameter config + """ + if conf["type"] == "fixed": + problem.search_space.root.add_discrete_param( + name, + feasible_values=[conf["value"][0]], + ) + else: + map_func = { + "float": problem.search_space.root.add_float_param, + "int": problem.search_space.root.add_int_param, + } + map_func[conf.get("type", "float")]( + name, + min_value=conf["value"][0], + max_value=conf["value"][1], + scale_type=MAP_SCALE_TYPE[conf.get("scale_type", "linear")], + ) + + +def cast_params(params: Dict, config: Dict) -> Dict: + """ + Cast params to integer according to configuration. + + Parameters + ---------- + params : Dict + Dictionary with suggested parameters + config : Dict + Provided configuration with types + + Returns + ------- + Dict + Updated parameters + """ + for key, value in params.items(): + if config[key]["type"] == "int": + params[key] = int(value) + return params + + +def main( + args: argparse.Namespace, + config: Dict, + install_path: Path, + server_endpoint: str = None, +) -> Dict: + """ + Converts config to Vizier problem definition and runs optimization. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + config : Dict + Optimization configuration + install_path : Path + Path to the folder with installed ORFS binaries + server_endpoint : str + URL pointing to Vizier server + + Returns + ------- + Dict + Results of optimization, containing 'config', 'population' + and found 'optimals' + """ + results = {"config": config, "populations": [], "optimals": []} + + problem = vz.ProblemStatement() + for key, value in config.items(): + if isinstance(value, Dict): + register_param(args, problem, key, value) + for metric in args.use_metrics: + problem.metric_information.append( + vz.MetricInformation(metric, goal=METRIC_TO_GOAL[metric]) + ) + + study_config = vz.StudyConfig.from_problem(problem) + study_config.algorithm = args.algorithm + + # Vizier Client setup + if server_endpoint: + clients.environment_variables.server_endpoint = server_endpoint + study_client = clients.Study.from_study_config( + study_config, owner="owner", study_id=f"{args.experiment}-{args.design}" + ) + + state = study_client.materialize_state() + start_iteration = 0 + # Check if experiment should be continued + if state == vz.StudyState.COMPLETED or state == vz.StudyState.ABORTED: + trials = list(study_client.trials().get()) + last_iteration = max( + map(lambda x: int(x.metadata.get("iteration", -1)), trials) + ) + start_iteration = last_iteration + 1 + if start_iteration <= args.iterations - 1: + print(f"[WARN TUN-0026] Trying to restart experiment (previously {state})") + study_client.set_state(vz.StudyState.ACTIVE) + + # Run iterations + for i, s in zip( + range(start_iteration, args.iterations), args.suggestions[start_iteration:] + ): + try: + suggestions = study_client.suggest(count=s) + unfinished = [ + parallel_evaluate.remote( + args, + cast_params(suggestion.parameters, config), + i, + s_i, + install_path, + ) + for s_i, suggestion in enumerate(suggestions) + ] + # Setup tqdm + print("\n") # Prepare space for additional info + tqdm_population = tqdm(total=s) + tqdm_population.set_description(f"Iteration {i + 1}/{args.iterations}") + while unfinished: + finished, unfinished = ray.wait(unfinished, num_returns=1) + sample = ray.get(finished)[0] + print(sample) + results["populations"].append(sample) + final_measurement = vz.Measurement(sample["evaluation"]) + process_suggestion = suggestions[sample["suggestion"]] + process_suggestion.update_metadata( + vz.Metadata( + { + "variant": sample["variant"], + "duration": str(sample["duration"]), + "iteration": str(i), + "suggestion": str(s), + } + ) + ) + # Display suggestion's parameters and evaluations + tqdm_population.display( + f"[INFO TUN-0024] Params: {process_suggestion.parameters}\n" + f"Evaluation: {sample['evaluation']}\n", + -1, + ) + tqdm_population.update(1) + suggestions[sample["suggestion"]].complete(final_measurement) + except KeyboardInterrupt as ex: + study_client.set_state(vz.StudyState.ABORTED) + raise ex + + study_client.set_state(vz.StudyState.COMPLETED) + + for optimal_trial in study_client.optimal_trials(): + trial = optimal_trial.materialize() + print( + f"[INFO TUN-0027] Found params: {trial.parameters.as_dict()}\nMetrics: {trial.final_measurement.metrics}" + ) + results["optimals"].append( + { + "params": trial.parameters.as_dict(), + "evaluation": { + k: v.value for k, v in trial.final_measurement.metrics.items() + }, + "variant": f"{args.experiment}/{trial.metadata.get_or_error('variant')}", + "time": float(trial.metadata.get_or_error("duration")), + } + ) + return results + + +def initialize_parser() -> argparse.ArgumentParser: + """ + Creates parser with required arguments. + + Returns + ------- + argparse.ArgumentParser + Preared parser + """ + parser = argparse.ArgumentParser() + add_common_args(parser) + parser.add_argument( + "--experiment", + type=str, + metavar="", + default=f"test-{DATE}", + help="Experiment name. This parameter is used to prefix the" + " FLOW_VARIANT and as the Vizier study ID.", + ) + parser.add_argument( + "--orfs", + type=Path, + metavar="", + default=ORFS, + help="Path to the OpenROAD-flow-scripts repository", + ) + parser.add_argument( + "--results", + type=Path, + metavar="", + default="results.json", + help="Path where JSON file with results will be saved", + ) + parser.add_argument( + "-a", + "--algorithm", + type=str, + choices=[ + "GAUSSIAN_PROCESS_BANDIT", + "RANDOM_SEARCH", + "QUASI_RANDOM_SEARCH", + "GRID_SEARCH", + "SHUFFLED_GRID_SEARCH", + "NSGA2", + ], + help="Algorithm for the optimization engine", + default="NSGA2", + ) + available_metrics = list(METRIC_TO_GOAL.keys()) + parser.add_argument( + "-m", + "--use-metrics", + nargs="+", + choices=available_metrics, + default=available_metrics, + help="Metrics to optimize", + ) + parser.add_argument( + "-i", + "--iterations", + type=int, + metavar="", + help="Max iteration count for the optimization engine", + default=2, + ) + parser.add_argument( + "-s", + "--suggestions", + type=int, + metavar="", + nargs="+", + help="Suggestion count per iteration of the optimization engine", + default=[5], + ) + vizier_server_args = parser.add_mutually_exclusive_group() + vizier_server_args.add_argument( + "--vz-use-existing-server", + type=str, + metavar="", + help="Address of the running Vizier server", + default=None, + ) + start_server_args = vizier_server_args.add_argument_group("Local server") + start_server_args.add_argument( + "--vz-server-host", + type=str, + metavar="", + help="Spawn Vizier server with given host", + default=None, + ) + start_server_args.add_argument( + "--vz-server-db", + type=str, + metavar="", + help="Path to the Vizier server's database", + default=None, + ) + return parser + + +def run_vizier(): + """ + Entrypoint for Vizier optimization. + + Parses arguments and config, prepares Vizier server, + runs optimization and saves the results. + """ + parser = initialize_parser() + args = parser.parse_args() + + if args.algorithm == "GAUSSIAN_PROCESS_BANDIT" and any( + s > 1 for s in args.suggestions + ): + print( + "[ERROR TUN-0022] GAUSSIAN_PROCESS_BANDIT does not support " + "batch operation, please set suggestions to 1", + file=sys.stderr, + ) + exit(1) + + args.results = args.results.absolute() + args.mode = "vizier" + args.suggestions += [ + args.suggestions[-1] for _ in range(args.iterations - len(args.suggestions)) + ] + + config, sdc_file, fr_file = read_config(args.config, "vizier", args.algorithm) + args.sdc_file = sdc_file + args.fr_file = fr_file + + local_dir, orfs_flow_dir, install_path = prepare_ray_server(args) + args.orfs = Path(orfs_flow_dir).parent + + server_endpoint = None + if args.vz_server_host: + # Start Vizier server + server_database = ( + args.vz_server_db if args.vz_server_db else service.SQL_LOCAL_URL + ) + server = servers.DefaultVizierServer( + host=args.vz_server_host, + database_url=server_database, + ) + print(f"[INFO TUN-0020] Started Vizier Server at: {server.endpoint}") + print(f"[INFO TUN-0021] SQL database file located at: {server._database_url}") + server_endpoint = server.endpoint + if args.vz_use_existing_server: + server_endpoint = args.vz_use_existing_server + + results = main(args, config, install_path, server_endpoint) + with args.results.open("w") as fd: + json.dump(results, fd) + print(f"[INFO TUN-0002] Results saved to {args.results}") + + +if __name__ == "__main__": + run_vizier() From 16c0f8ab0320c4193262f2c134322b511817a9ba Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:43:05 +0200 Subject: [PATCH 08/10] tools: Autotuner: Add --to-stage argument Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/src/autotuner/distributed.py | 5 ++- tools/AutoTuner/src/autotuner/utils.py | 44 +++++++++++++++----- tools/AutoTuner/src/autotuner/vizier.py | 2 +- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/tools/AutoTuner/src/autotuner/distributed.py b/tools/AutoTuner/src/autotuner/distributed.py index 7aae0ba0f7..99dea45f51 100644 --- a/tools/AutoTuner/src/autotuner/distributed.py +++ b/tools/AutoTuner/src/autotuner/distributed.py @@ -109,9 +109,10 @@ def step(self): parameters=self.parameters, flow_variant=self.variant, install_path=INSTALL_PATH, + stage=args.to_stage, ) self.step_ += 1 - score = self.evaluate(read_metrics(metrics_file)) + score = self.evaluate(read_metrics(metrics_file, args.to_stage)) # Feed the score back to Tune. # return must match 'metric' used in tune.run() return {METRIC: score} @@ -430,7 +431,7 @@ def sweep(): TrainClass = set_training_class(args.eval) # PPAImprov requires a reference file to compute training scores. if args.eval == "ppa-improv": - reference = read_metrics(args.reference) + reference = read_metrics(args.reference, args.to_stage) tune_args = dict( name=args.experiment, diff --git a/tools/AutoTuner/src/autotuner/utils.py b/tools/AutoTuner/src/autotuner/utils.py index e1e46719dc..34af249190 100644 --- a/tools/AutoTuner/src/autotuner/utils.py +++ b/tools/AutoTuner/src/autotuner/utils.py @@ -29,6 +29,12 @@ set_input_delay [expr $clk_period * $clk_io_pct] -clock $clk_name $non_clock_inputs set_output_delay [expr $clk_period * $clk_io_pct] -clock $clk_name [all_outputs] """ +# Maps ORFS stage to a name of produced metrics +STAGE_TO_METRICS = { + "route": "detailedroute", + "place": "detailedplace", + "final": "finish", +} # Name of the SDC file with constraints CONSTRAINTS_SDC = "constraint.sdc" # Name of the TCL script run before routing @@ -242,6 +248,7 @@ def openroad( flow_variant, path="", install_path=None, + stage="", ): """ Run OpenROAD-flow-scripts with a given set of parameters. @@ -265,7 +272,7 @@ def openroad( make_command = export_command make_command += f"make -C {base_dir}/flow DESIGN_CONFIG=designs/" - make_command += f"{args.platform}/{args.design}/config.mk" + make_command += f"{args.platform}/{args.design}/config.mk {stage}" make_command += f" PLATFORM={args.platform}" make_command += f" FLOW_VARIANT={flow_variant} {parameters}" make_command += " EQUIVALENCE_CHECK=0" @@ -316,10 +323,11 @@ def openroad( ) -def read_metrics(file_name): +def read_metrics(file_name, stage=""): """ Collects metrics to evaluate the user-defined objective function. """ + metric_name = STAGE_TO_METRICS.get(stage if stage else "final", stage) with open(file_name) as file: data = json.load(file) clk_period = 9999999 @@ -342,17 +350,17 @@ def read_metrics(file_name): num_drc = value["route__drc_errors"] if stage_name == "detailedroute" and "route__wirelength" in value: wirelength = value["route__wirelength"] - if stage_name == "finish" and "timing__setup__ws" in value: + if stage_name == metric_name and "timing__setup__ws" in value: worst_slack = value["timing__setup__ws"] - if stage_name == "finish" and "power__total" in value: + if stage_name == metric_name and "power__total" in value: total_power = value["power__total"] - if stage_name == "finish" and "design__instance__utilization" in value: + if stage_name == metric_name and "design__instance__utilization" in value: final_util = value["design__instance__utilization"] - if stage_name == "finish" and "design__instance__area" in value: + if stage_name == metric_name and "design__instance__area" in value: design_area = value["design__instance__area"] - if stage_name == "finish" and "design__core__area" in value: + if stage_name == metric_name and "design__core__area" in value: core_area = value["design__core__area"] - if stage_name == "finish" and "design__die__area" in value: + if stage_name == metric_name and "design__die__area" in value: die_area = value["design__die__area"] for i, stage_name in reversed(STAGES): if stage_name in data and [d for d in data[stage_name].values() if d != "ERR"]: @@ -367,9 +375,15 @@ def read_metrics(file_name): "design_area": design_area, "core_area": core_area, "die_area": die_area, - "wirelength": wirelength, - "num_drc": num_drc, - } + "last_successful_stage": last_stage, + } | ( + { + "wirelength": wirelength, + "num_drc": num_drc, + } + if metric_name in ("detailedroute", "finish") + else {} + ) return ret @@ -678,6 +692,7 @@ def openroad_distributed( flow_variant=f"{uuid()}-{variant}", path=path, install_path=install_path, + stage=args.to_stage, ) duration = time() - t return metric_file, duration @@ -718,6 +733,13 @@ def add_common_args(parser: argparse.ArgumentParser): required=True, help="Configuration file that sets which knobs to use for Autotuning.", ) + parser.add_argument( + "--to-stage", + type=str, + choices=("floorplan", "place", "cts", "route", "finish"), + default="", + help="Run ORFS only to the given stage (inclusive)", + ) parser.add_argument( "--timeout", type=float, diff --git a/tools/AutoTuner/src/autotuner/vizier.py b/tools/AutoTuner/src/autotuner/vizier.py index 225df742ed..56edd3f1a1 100644 --- a/tools/AutoTuner/src/autotuner/vizier.py +++ b/tools/AutoTuner/src/autotuner/vizier.py @@ -64,7 +64,7 @@ def evaluate(args: argparse.Namespace, metric_file: str) -> Dict[str, float]: Dictionary with metrics """ try: - metrics = read_metrics(metric_file) + metrics = read_metrics(metric_file, stage=args.to_stage) # Calculate difference of clock period and worst slack if metrics["clk_period"] != 9999999 and metrics["worst_slack"] != "ERR": metrics["clk_period-worst_slack"] = ( From 20aba1ab8499736f945d1db7d220c39a777d373f Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:44:15 +0200 Subject: [PATCH 09/10] tools: Autotuner: Add smoke test for Vizier Signed-off-by: Eryk Szpotanski --- flow/test/test_helper.sh | 3 ++ tools/AutoTuner/test/smoke_test_vizier.py | 52 +++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tools/AutoTuner/test/smoke_test_vizier.py diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh index 9b2409d114..b0c0338c97 100755 --- a/flow/test/test_helper.sh +++ b/flow/test/test_helper.sh @@ -98,6 +98,9 @@ if [[ -n "${RUN_AUTOTUNER+x}" ]] && [[ ${RUN_AUTOTUNER} -eq 1 ]]; then echo "Running Autotuner smoke tests for --sample and --iteration." python3 -m unittest tools.AutoTuner.test.smoke_test_sample_iteration.${PLATFORM}SampleIterationSmokeTest.test_sample_iteration + echo "Running Autotuner smoke Vizier test" + python3 -m unittest tools.AutoTuner.test.smoke_test_vizier.${PLATFORM}VizierSmokeTest.test_vizier + if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN" == "gcd" ]; then echo "Running Autotuner ref file test (only once)" python3 -m unittest tools.AutoTuner.test.ref_file_check.RefFileCheck.test_files diff --git a/tools/AutoTuner/test/smoke_test_vizier.py b/tools/AutoTuner/test/smoke_test_vizier.py new file mode 100644 index 0000000000..648d2229d4 --- /dev/null +++ b/tools/AutoTuner/test/smoke_test_vizier.py @@ -0,0 +1,52 @@ +import unittest +import subprocess +import os +from datetime import datetime + +cur_dir = os.path.dirname(os.path.abspath(__file__)) + + +class BaseVizierSmokeTest(unittest.TestCase): + platform = "" + design = "" + + def setUp(self): + self.config = os.path.join( + cur_dir, + f"../../../flow/designs/{self.platform}/{self.design}/autotuner.json", + ) + self.experiment = f"smoke-test-tune-{self.platform}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}" + self.command = ( + "python3 -m autotuner.vizier" + f" --design {self.design}" + f" --platform {self.platform}" + f" --experiment {self.experiment}" + f" --config {self.config}" + f" --iteration 1 --suggestions 1" + ) + + def test_vizier(self): + if not (self.platform and self.design): + raise unittest.SkipTest("Platform and design have to be defined") + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + +class ASAP7VizierSmokeTest(BaseVizierSmokeTest): + platform = "asap7" + design = "gcd" + + +class SKY130HDVizierSmokeTest(BaseVizierSmokeTest): + platform = "sky130hd" + design = "gcd" + + +class IHPSG13G2VizierSmokeTest(BaseVizierSmokeTest): + platform = "ihp-sg13g2" + design = "gcd" + + +if __name__ == "__main__": + unittest.main() From 5bbabb7ea2e082486139efb0ae571f8631812c57 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:46:26 +0200 Subject: [PATCH 10/10] docs: user: Add information about Vizier in AutoTuner Signed-off-by: Eryk Szpotanski --- docs/user/InstructionsForAutoTuner.md | 86 +++++++++++++++++++++------ 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/docs/user/InstructionsForAutoTuner.md b/docs/user/InstructionsForAutoTuner.md index b3bf6826ab..86972cd919 100644 --- a/docs/user/InstructionsForAutoTuner.md +++ b/docs/user/InstructionsForAutoTuner.md @@ -5,20 +5,29 @@ AutoTuner provides a generic interface where users can define parameter configur This enables AutoTuner to easily support various tools and flows. AutoTuner also utilizes [METRICS2.1](https://github.com/ieee-ceda-datc/datc-rdf-Metrics4ML) to capture PPA of individual search trials. With the abundant features of METRICS2.1, users can explore various reward functions that steer the flow autotuning to different PPA goals. -AutoTuner provides two main functionalities as follows. -* Automatic hyperparameter tuning framework for OpenROAD-flow-script (ORFS) -* Parametric sweeping experiments for ORFS +AutoTuner provides three main functionalities as follows. +* [Ray] Automatic hyperparameter tuning framework for OpenROAD-flow-script (ORFS) +* [Ray] Parametric sweeping experiments for ORFS +* [Vizier] Multi-objective optimization of ORFS parameters AutoTuner contains top-level Python script for ORFS, each of which implements a different search algorithm. Current supported search algorithms are as follows. -* Random/Grid Search -* Population Based Training ([PBT](https://www.deepmind.com/blog/population-based-training-of-neural-networks)) -* Tree Parzen Estimator ([HyperOpt](https://hyperopt.github.io/hyperopt)) -* Bayesian + Multi-Armed Bandit ([AxSearch](https://ax.dev/)) -* Tree Parzen Estimator + Covariance Matrix Adaptation Evolution Strategy ([Optuna](https://optuna.org/)) -* Evolutionary Algorithm ([Nevergrad](https://github.com/facebookresearch/nevergrad)) +* Ray (Single-objective optimization) + * Random/Grid Search + * Population Based Training ([PBT](https://www.deepmind.com/blog/population-based-training-of-neural-networks)) + * Tree Parzen Estimator ([HyperOpt](https://hyperopt.github.io/hyperopt)) + * Bayesian + Multi-Armed Bandit ([AxSearch](https://ax.dev/docs/bayesopt.html)) + * Tree Parzen Estimator + Covariance Matrix Adaptation Evolution Strategy ([Optuna](https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.TPESampler.html)) + * Evolutionary Algorithm ([Nevergrad](https://github.com/facebookresearch/nevergrad)) +* Vizier (Multi-objective optimization) + * Random/Grid/Shuffled Search + * Quasi Random Search ([quasi-random](https://developers.google.com/machine-learning/guides/deep-learning-tuning-playbook/quasi-random-search)) + * Gaussian Process Bandit ([GP-Bandit](https://acsweb.ucsd.edu/~shshekha/GPBandits.html)) + * Non-dominated Sorting Genetic Algorithm II ([NSGA-II](https://ieeexplore.ieee.org/document/996017)) -User-defined coefficient values (`coeff_perform`, `coeff_power`, `coeff_area`) of three objectives to set the direction of tuning are written in the script. Each coefficient is expressed as a global variable at the `get_ppa` function in `PPAImprov` class in the script (`coeff_perform`, `coeff_power`, `coeff_area`). Efforts to optimize each of the objectives are proportional to the specified coefficients. +For Ray algorithms, optimized function can be adjusted with user-defined coefficient values (`coeff_perform`, `coeff_power`, `coeff_area`) for three objectives to set the direction of tuning. They are defined in the [distributed.py sricpt](../../tools/AutoTuner/src/autotuner/distributed.py) in `get_ppa` method of `PPAImprov` class. Efforts to optimize each of the objectives are proportional to the specified coefficients. + +Using Vizier algorithms, used can choose which metrics should be optimized with `--use-metrics` argument. ## Setting up AutoTuner @@ -28,8 +37,10 @@ that works in Python3.8 for installation and configuration of AutoTuner, as shown below: ```shell -# Install prerequisites +# Install prerequisites for both Ray Tune and Vizier ./tools/AutoTuner/installer.sh +# Or install prerequisites for `ray` or `vizier` +./tools/AutoTuner/installer.sh vizier # Start virtual environment ./tools/AutoTuner/setup.sh @@ -50,7 +61,8 @@ Alternatively, here is a minimal example to get started: 1.0, 3.7439 ], - "step": 0 + "step": 0, + "scale": "log" }, "CORE_MARGIN": { "type": "int", @@ -67,6 +79,7 @@ Alternatively, here is a minimal example to get started: * `"type"`: Parameter type ("float" or "int") for sweeping/tuning * `"minmax"`: Min-to-max range for sweeping/tuning. The unit follows the default value of each technology std cell library. * `"step"`: Parameter step within the minmax range. Step 0 for type "float" means continuous step for sweeping/tuning. Step 0 for type "int" means the constant parameter. +* `"scale"`: Vizier-specific parameter setting [scaling type](https://oss-vizier.readthedocs.io/en/latest/guides/user/search_spaces.html#scaling), allowed values: `linear`, `log` and `rlog`. ## Tunable / sweepable parameters @@ -118,13 +131,21 @@ The order of the parameters matter. Arguments `--design`, `--platform` and `--config` are always required and should precede *mode*. ``` +The `autotuner.vizier` module integrates OpenROAD flow into the Vizier optimizer. +It is used for multi-objective optimization with an additional features improving chance of finding valid parameters. +Moreover, various algorithms are available for tuning parameters. + +Each mode relies on user-specified search space that is +defined by a `.json` file, they use the same syntax and format, +though some features may not be available for sweeping. + ```{note} The following commands should be run from `./tools/AutoTuner`. ``` #### Tune only -* AutoTuner: `python3 -m autotuner.distributed tune -h` +* Ray-based AutoTuner: `python3 -m autotuner.distributed tune -h` Example: @@ -145,19 +166,37 @@ python3 -m autotuner.distributed --design gcd --platform sky130hd \ sweep ``` +#### Multi-object optimization + +* Vizier-based AutoTuner: `python3 -m autotuner.vizier -h` + +Example: + +```shell +python3 -m autotuner.vizier --design gcd --platform sky130hd \ + --config ../../flow/designs/sky130hd/gcd/autotuner.json +``` ### Google Cloud Platform (GCP) distribution with Ray GCP Setup Tutorial coming soon. -### List of input arguments +### List of common input arguments | Argument | Description | |-------------------------------|-------------------------------------------------------------------------------------------------------| | `--design` | Name of the design for Autotuning. | | `--platform` | Name of the platform for Autotuning. | | `--config` | Configuration file that sets which knobs to use for Autotuning. | | `--experiment` | Experiment name. This parameter is used to prefix the FLOW_VARIANT and to set the Ray log destination.| +| `--algorithm` | Search algorithm to use for Autotuning. | +| `--openroad_threads` | Max number of threads usable. | +| `--to-stage` | The last stage to be built during optimization. | +| `-v` or `--verbose` | Verbosity Level. [0: Only ray status, 1: print stderr, 2: print stdout on top of what is in level 0 and 1. ] | +| | | +### List of Ray-specific input arguments +| Argument | Description | +|-------------------------------|-------------------------------------------------------------------------------------------------------| | `--resume` | Resume previous run. | | `--git_clean` | Clean binaries and build files. **WARNING**: may lose previous data. | | `--git_clone` | Force new git clone. **WARNING**: may lose previous data. | @@ -176,12 +215,22 @@ GCP Setup Tutorial coming soon. | `--perturbation` | Perturbation interval for PopulationBasedTraining | | `--seed` | Random seed. | | `--jobs` | Max number of concurrent jobs. | -| `--openroad_threads` | Max number of threads usable. | | `--server` | The address of Ray server to connect. | | `--port` | The port of Ray server to connect. | -| `-v` or `--verbose` | Verbosity Level. [0: Only ray status, 1: print stderr, 2: print stdout on top of what is in level 0 and 1. ] | -| | | -### GUI + +### List of Vizier-specific input arguments +| Argument | Description | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--orfs` | Path to the OpenROAD-flow-scripts repository | +| `--results` | Path where JSON file with results will be saved | +| `-a` or `--algorithm` | Algorithm for the optimization engine, one of GAUSSIAN_PROCESS_BANDIT, RANDOM_SEARCH, QUASI_RANDOM_SEARCH, GRID_SEARCH, SHUFFLED_GRID_SEARCH, NSGA2 | +| `-m` or `--use-metrics` | Metrics to optimize, list of worst_slack, clk_period-worst_slack, total_power, core_util, final_util, design_area, core_area, die_area, last_successful_stage | +| `-i` or `--iterations` | Max iteration count for the optimization engine | +| `-s` or `--suggestions` | Suggestion count per iteration of the optimization engine | +| `-w` or `--workers` | Number of parallel workers | +| `--use-existing-server` | Address of the running Vizier server | + +### GUI for optimizations with Ray Tune Basically, progress is displayed at the terminal where you run, and when all runs are finished, the results are displayed. You could find the "Best config found" on the screen. @@ -207,6 +256,7 @@ Assuming the virtual environment is setup at `./tools/AutoTuner/autotuner_env`: ./tools/AutoTuner/setup.sh python3 ./tools/AutoTuner/test/smoke_test_sweep.py python3 ./tools/AutoTuner/test/smoke_test_tune.py +python3 ./tools/AutoTuner/test/smoke_test_vizier.py python3 ./tools/AutoTuner/test/smoke_test_sample_iteration.py ```