diff --git a/src/hammer-tech/filters.py b/src/hammer-tech/filters.py index f5ce3f0bb..9299c1181 100644 --- a/src/hammer-tech/filters.py +++ b/src/hammer-tech/filters.py @@ -93,6 +93,28 @@ def paths_func(lib: "Library") -> List[str]: return LibraryFilter.new("timing_lib", "CCS/NLDM timing lib (ASCII .lib)", paths_func=paths_func, is_file=True) + def timing_lib_nldm_filter(self, vts: Optional[List[str]] = None) -> LibraryFilter: + """ + Select ASCII .lib timing libraries. Only chooses NLDM. Only use this + for pipe-cleaning/exploration. Additionally, specify which vts you + would like to filter for as well (for extra speedup) + """ + def paths_func(lib: "Library") -> List[str]: + if len(vts) > 0: + has_vt = False + for vt in vts: + for provided in lib.provides: + has_vt = has_vt or (provided.vt == vt) + if not(has_vt): + return [] + if lib.nldm_liberty_file is not None: + return [lib.nldm_liberty_file] + return [] + + return LibraryFilter.new("timing_lib_nldm", + "NLDM timing lib (liberty ASCII .lib)", + paths_func=paths_func, is_file=True) + @property def timing_lib_with_ecsm_filter(self) -> LibraryFilter: """ @@ -167,6 +189,29 @@ def sort_func(lib: "Library"): return LibraryFilter.new("lef", "LEF physical design layout library", is_file=True, filter_func=filter_func, paths_func=paths_func, sort_func=sort_func) + @property + def tech_lef_filter(self) -> LibraryFilter: + """ + Select tech-LEF files for physical layout. + """ + + def filter_func(lib: "Library") -> bool: + return (lib.lef_file is not None) and \ + (lib.provides is not None) and \ + (len(list(filter(lambda p: p.lib_type == "technology", + lib.provides))) > 0) + + def paths_func(lib: "Library") -> List[str]: + assert lib.lef_file is not None + return [lib.lef_file] + + def sort_func(lib: "Library"): + return 0 + + return LibraryFilter.new("lef", "LEF physical design layout library", + is_file=True, filter_func=filter_func, + paths_func=paths_func, sort_func=sort_func) + @property def verilog_sim_filter(self) -> LibraryFilter: """ diff --git a/src/hammer-vlsi/defaults.yml b/src/hammer-vlsi/defaults.yml index c76975c6a..6ea92721b 100644 --- a/src/hammer-vlsi/defaults.yml +++ b/src/hammer-vlsi/defaults.yml @@ -83,6 +83,36 @@ vlsi.core: # Maximum threads to use in a CAD tool invocation. max_threads: 1 +#---------------------------------------------------------------------------- +# JOB SUBMISSION +#---------------------------------------------------------------------------- +vlsi.submit: + # - The submit command to use. "none", "local", or null will run on the + # current host. See hammer_submit_command.py for other options. + # type: str + command: "local" + # type: List[Dict[str, Dict[str, Any]]] + # The list substitutes settings in order of appearance, and the first + # Dict key is the command. The second dict key is the name of that + # command's setting, followed by whatever type it takes. + settings: [] + + # how many output lines to buffer in memory. "" for unlimited + max_output_lines: null + # how many error lines to buffer in memory. "" for unlimited + max_error_lines: null + # if exit code != 0, or len(error_lines) > 0, raise exception immediately + # use "true" or "false" + abort_on_error: false + + # take a guess that any time these show up, its an error. individual tools + # can override this assumption. use any string that is valid in python regex + error_rgxs: ["ERROR:", "error:", "Error:"] + error_ignore_rgxs: [] + +#---------------------------------------------------------------------------- +# Technology +#---------------------------------------------------------------------------- # TODO ucb-bar/hammer#317 move these to technology.core (discussion to be had) vlsi.technology: # Placement site for macros. (Optional[str]) @@ -129,12 +159,48 @@ vlsi.technology: # If this is not specified, then the tarballs will be extracted to obj//extracted/. extracted_tarballs_dir: null +technology.pcb: + # Desired pad opening diameter (in post-shrink um) + # Typically this should be defined in the technology plugin, assuming everyone will use the same pad and board strategy + # type: Decimal + bump_pad_opening_diameter: null + + # Desired pad metal diameter (in post-shrink um) + # Typically this should be defined in the technology plugin, assuming everyone will use the same pad and board strategy + # type: Decimal + bump_pad_metal_diameter: null + + +technology.core: + # Key name for the technology stackup + # This should exist in the stackups list in the tech json + # type: str + stackup: null + # This should specify the TOPMOST metal layer the standard cells use for power rails + # Note that this is not usually stackup specific; it is based on the std cell library + # type: str + std_cell_rail_layer: null + # This is used to provide a reference master for generating standard cell rails + # Can be a wildcard/glob + # type: str + tap_cell_rail_reference: null + +#---------------------------------------------------------------------------- +# Vendor common settings +#---------------------------------------------------------------------------- cadence: # Path to the folder with defaults.yml for common Cadence settings. common_path: "${vlsi.builtins.hammer_vlsi_path}/common/cadence" common_path_meta: subst -# General VLSI inputs. +mentor: + # Path to the folder with defaults.yml for common Mentor settings. + common_path: "${vlsi.builtins.hammer_vlsi_path}/common/mentor" + common_path_meta: subst + +#---------------------------------------------------------------------------- +# DESIGN SPECIFIC SETTINGS +#---------------------------------------------------------------------------- # These will vary per run of hammer-vlsi. vlsi.inputs: # Supply voltages and names @@ -200,6 +266,12 @@ vlsi.inputs: # netlist (str) - path to the netlist file ilms: [] + # only use nldm libs (instead of ccs. use for pipecleaning flow only) + use_nldm_libs: False + # which subset of stdcell lib vts to use (use for pipecleaning flow only) + # use the values in the 'library{provides}{vt}' section of the tech.json + use_lib_vts: [] + # Multi-mode multi-corner setups, overrides supplies # MMMC struct members: # name (str) - name of the corner. @@ -227,9 +299,11 @@ vlsi.inputs: # Clocks with no group specified will all be placed in separate groups and thus marked as asynchronous to each other and all other groups. clocks: [] - # Default output pin load capacitance. - # Default: 1pF - default_output_load: 1 + # these are generic io constraints, tune for your process + default_output_load: "1pF" + default_max_fanout: 80 + default_max_transition: "100ps" + default_max_clock_transition: "80ps" # List of output load constraints. # Each item in the list should be a struct with the following members: @@ -449,14 +523,9 @@ vlsi.inputs: # Requires the semi_auto pin generation mode. assignments: [] -vlsi.submit: - # The submit command to use. "none", "local", or null will run on the current host. See hammer_submit_command.py for other options. - command: "local" - # type: List[Dict[str, Dict[str, Any]]] - # The list substitutes settings in order of appearance, and the first Dict key is the command. - # The second dict key is the name of that command's setting, followed by whatever type it takes. - settings: [] - +#---------------------------------------------------------------------------- +# SYN +#---------------------------------------------------------------------------- # Specific inputs for the synthesis tool. # These inputs are the generic inputs; specific tools ("CAD junk") may require # additional inputs. @@ -470,6 +539,9 @@ synthesis.inputs: # Set to null to not specify from the JSON. top_module: null + # if true, don't try loading lef/def info, which is faster but less accurate + is_physical: False + # Syntheis tool settings synthesis: # Clock gating mode. @@ -482,11 +554,24 @@ synthesis: # inherit settings from vlsi.submit but allow us to override them synthesis.submit: - command: "${vlsi.submit.command}" - command_meta: lazysubst - settings: "vlsi.submit.settings" - settings_meta: lazycrossref - + command: "${vlsi.submit.command}" + settings: "vlsi.submit.settings" + max_output_lines: "${vlsi.submit.max_output_lines}" + max_error_lines: "${vlsi.submit.max_error_lines}" + abort_on_error: "${vlsi.submit.abort_on_error}" + error_rgxs: "vlsi.submit.error_rgxs" + error_ignore_rgxs: "vlsi.submit.error_ignore_rgxs" + command_meta: lazysubst + settings_meta: lazytrycrossref + max_output_lines_meta: lazytrysubst + max_error_lines_meta: lazytrysubst + abort_on_error_meta: lazytrysubst + error_rgxs_meta: lazytrycrossref + error_ignore_rgxs_meta: lazytrycrossref + +#---------------------------------------------------------------------------- +# Place-and-route settings +#---------------------------------------------------------------------------- # Specific inputs for the place and route tool. # These inputs are the generic inputs; specific tools ("CAD junk") may require # additional inputs. @@ -537,16 +622,7 @@ par.inputs: # type List[str] physical_only_cells_list: [] -# Place-and-route settings par: - # Submission command settings - # inherit settings from vlsi.submit but allow us to override them - submit: - command: "${vlsi.submit.command}" - command_meta: lazysubst - settings: "vlsi.submit.settings" - settings_meta: lazycrossref - # Power straps configuration. # Valid options are: # - empty - Specify no power straps @@ -609,12 +685,25 @@ par: # type: List[str] strap_layers: [] -mentor: - # Path to the folder with defaults.yml for common Mentor settings. - common_path: "${vlsi.builtins.hammer_vlsi_path}/common/mentor" - common_path_meta: subst - +par.submit: + command: "${vlsi.submit.command}" + settings: "vlsi.submit.settings" + max_output_lines: "${vlsi.submit.max_output_lines}" + max_error_lines: "${vlsi.submit.max_error_lines}" + abort_on_error: "${vlsi.submit.abort_on_error}" + error_rgxs: "vlsi.submit.error_rgxs" + error_ignore_rgxs: "vlsi.submit.error_ignore_rgxs" + command_meta: lazysubst + settings_meta: lazytrycrossref + max_output_lines_meta: lazytrysubst + max_error_lines_meta: lazytrysubst + abort_on_error_meta: lazytrysubst + error_rgxs_meta: lazytrycrossref + error_ignore_rgxs_meta: lazytrycrossref + +#---------------------------------------------------------------------------- # DRC settings +#---------------------------------------------------------------------------- drc.inputs: # Top RTL module. # type: str @@ -641,14 +730,25 @@ drc.inputs: # type: List[str] drc_rules_to_run: [] -# inherit settings from vlsi.submit but allow us to override them drc.submit: - command: "${vlsi.submit.command}" - command_meta: lazysubst - settings: "vlsi.submit.settings" - settings_meta: lazycrossref - + command: "${vlsi.submit.command}" + settings: "vlsi.submit.settings" + max_output_lines: "${vlsi.submit.max_output_lines}" + max_error_lines: "${vlsi.submit.max_error_lines}" + abort_on_error: "${vlsi.submit.abort_on_error}" + error_rgxs: "vlsi.submit.error_rgxs" + error_ignore_rgxs: "vlsi.submit.error_ignore_rgxs" + command_meta: lazysubst + settings_meta: lazytrycrossref + max_output_lines_meta: lazytrysubst + max_error_lines_meta: lazytrysubst + abort_on_error_meta: lazytrysubst + error_rgxs_meta: lazytrycrossref + error_ignore_rgxs_meta: lazytrycrossref + +#---------------------------------------------------------------------------- # LVS settings +#---------------------------------------------------------------------------- lvs.inputs: # Top RTL module. top_module: null @@ -669,46 +769,67 @@ lvs.inputs: additional_lvs_text: "" lvs.submit: - command: "${vlsi.submit.command}" - command_meta: lazysubst - settings: "vlsi.submit.settings" - settings_meta: lazycrossref - -# inherit settings from vlsi.submit but allow us to override them + command: "${vlsi.submit.command}" + settings: "vlsi.submit.settings" + max_output_lines: "${vlsi.submit.max_output_lines}" + max_error_lines: "${vlsi.submit.max_error_lines}" + abort_on_error: "${vlsi.submit.abort_on_error}" + error_rgxs: "vlsi.submit.error_rgxs" + error_ignore_rgxs: "vlsi.submit.error_ignore_rgxs" + command_meta: lazysubst + settings_meta: lazytrycrossref + max_output_lines_meta: lazytrysubst + max_error_lines_meta: lazytrysubst + abort_on_error_meta: lazytrysubst + error_rgxs_meta: lazytrycrossref + error_ignore_rgxs_meta: lazytrycrossref + +#---------------------------------------------------------------------------- +# sram-generator settings +#---------------------------------------------------------------------------- sram_generator.submit: - command: "${vlsi.submit.command}" - command_meta: lazysubst - settings: "vlsi.submit.settings" - settings_meta: lazycrossref - + command: "${vlsi.submit.command}" + settings: "vlsi.submit.settings" + max_output_lines: "${vlsi.submit.max_output_lines}" + max_error_lines: "${vlsi.submit.max_error_lines}" + abort_on_error: "${vlsi.submit.abort_on_error}" + error_rgxs: "vlsi.submit.error_rgxs" + error_ignore_rgxs: "vlsi.submit.error_ignore_rgxs" + command_meta: lazysubst + settings_meta: lazytrycrossref + max_output_lines_meta: lazytrysubst + max_error_lines_meta: lazytrysubst + abort_on_error_meta: lazytrysubst + error_rgxs_meta: lazytrycrossref + error_ignore_rgxs_meta: lazytrycrossref + + +#---------------------------------------------------------------------------- # PCB deliverable settings. +#---------------------------------------------------------------------------- pcb.inputs: # Top RTL module. top_module: null pcb.submit: - # Submission command settings - # inherit settings from vlsi.submit but allow us to override them - command: "${vlsi.submit.command}" - command_meta: lazysubst - settings: "vlsi.submit.settings" - settings_meta: lazycrossref - - -technology.core: - # Key name for the technology stackup - # This should exist in the stackups list in the tech json - # type: str - stackup: null - # This should specify the TOPMOST metal layer the standard cells use for power rails - # Note that this is not usually stackup specific; it is based on the std cell library - # type: str - std_cell_rail_layer: null - # This is used to provide a reference master for generating standard cell rails - # Can be a wildcard/glob - # type: str - tap_cell_rail_reference: null - + command: "${vlsi.submit.command}" + settings: "vlsi.submit.settings" + max_output_lines: "${vlsi.submit.max_output_lines}" + max_error_lines: "${vlsi.submit.max_error_lines}" + abort_on_error: "${vlsi.submit.abort_on_error}" + error_rgxs: "vlsi.submit.error_rgxs" + error_ignore_rgxs: "vlsi.submit.error_ignore_rgxs" + command_meta: lazysubst + settings_meta: lazytrycrossref + max_output_lines_meta: lazytrysubst + max_error_lines_meta: lazytrysubst + abort_on_error_meta: lazytrysubst + error_rgxs_meta: lazytrycrossref + error_ignore_rgxs_meta: lazytrycrossref + +#---------------------------------------------------------------------------- +# sim settings +#---------------------------------------------------------------------------- # Specific inputs for the simulation tool. # These inputs will vary based on if the simulation is for RTL, post synthesis, or post PAR. # There may be HAMMER simulation tool plugins available that include common simulator flags and inputs that can be modified/augmented for the user's needs. @@ -797,19 +918,19 @@ sim.inputs: execute_sim: true sim.submit: - command: "${vlsi.submit.command}" - command_meta: lazysubst - settings: "vlsi.submit.settings" - settings_meta: lazycrossref + command: "${vlsi.submit.command}" + settings: "vlsi.submit.settings" + max_output_lines: "${vlsi.submit.max_output_lines}" + max_error_lines: "${vlsi.submit.max_error_lines}" + abort_on_error: "${vlsi.submit.abort_on_error}" + error_rgxs: "vlsi.submit.error_rgxs" + error_ignore_rgxs: "vlsi.submit.error_ignore_rgxs" + command_meta: lazysubst + settings_meta: lazytrycrossref + max_output_lines_meta: lazytrysubst + max_error_lines_meta: lazytrysubst + abort_on_error_meta: lazytrysubst + error_rgxs_meta: lazytrycrossref + error_ignore_rgxs_meta: lazytrycrossref -technology.pcb: - # Desired pad opening diameter (in post-shrink um) - # Typically this should be defined in the technology plugin, assuming everyone will use the same pad and board strategy - # type: Decimal - bump_pad_opening_diameter: null - - # Desired pad metal diameter (in post-shrink um) - # Typically this should be defined in the technology plugin, assuming everyone will use the same pad and board strategy - # type: Decimal - bump_pad_metal_diameter: null diff --git a/src/hammer-vlsi/generate_properties.py b/src/hammer-vlsi/generate_properties.py index 39a297b7a..795a63c4d 100755 --- a/src/hammer-vlsi/generate_properties.py +++ b/src/hammer-vlsi/generate_properties.py @@ -110,25 +110,27 @@ def main(args) -> int: parsed_args = parser.parse_args(args[1:]) HammerSynthesisTool = Interface(module="HammerSynthesisTool", - filename="hammer_vlsi/hammer_vlsi_impl.py", - inputs=[ - InterfaceVar("input_files", "List[str]", - "input collection of source RTL files (e.g. *.v)") - ], - outputs=[ - InterfaceVar("output_files", "List[str]", - "output collection of mapped (post-synthesis) RTL files"), - InterfaceVar("output_sdc", "str", - "(optional) output post-synthesis SDC constraints file"), - InterfaceVar("output_all_regs", "List[str]", - "output list of all registers in the design with output pin for gate level simulation"), - InterfaceVar("output_seq_cells", "List[str]", - "output collection of all sequential standard cells in design"), - InterfaceVar("sdf_file", "str", - "output SDF file to be read for timing annotated gate level sims") - # TODO: model CAD junk - ] - ) + filename="hammer_vlsi/hammer_vlsi_impl.py", + inputs=[ + InterfaceVar("input_files", "List[str]", + "input collection of source RTL files (e.g. *.v)"), + InterfaceVar("is_physical", "bool", + "true if the synthesis tool should use lef/def for PLE") + ], + outputs=[ + InterfaceVar("output_files", "List[str]", + "output collection of mapped (post-synthesis) RTL files"), + InterfaceVar("output_sdc", "str", + "(optional) output post-synthesis SDC constraints file"), + InterfaceVar("output_all_regs", "List[str]", + "output list of all registers in the design with output pin for gate level simulation"), + InterfaceVar("output_seq_cells", "List[str]", + "output collection of all sequential standard cells in design"), + InterfaceVar("sdf_file", "str", + "output SDF file to be read for timing annotated gate level sims") + # TODO: model CAD junk + ] + ) HammerPlaceAndRouteTool = Interface(module="HammerPlaceAndRouteTool", filename="hammer_vlsi/hammer_vlsi_impl.py", diff --git a/src/hammer-vlsi/hammer_vlsi/constraints.py b/src/hammer-vlsi/hammer_vlsi/constraints.py index 76865b2a4..6343a8dce 100644 --- a/src/hammer-vlsi/hammer_vlsi/constraints.py +++ b/src/hammer-vlsi/hammer_vlsi/constraints.py @@ -14,7 +14,7 @@ from hammer_utils import reverse_dict, get_or_else, add_dicts from hammer_tech import MacroSize -from .units import TimeValue, VoltageValue, TemperatureValue +from .units import TimeValue, VoltageValue, TemperatureValue, CapacitanceValue from decimal import Decimal import math @@ -372,7 +372,7 @@ def name_bump(self, definition: BumpsDefinition, assignment: BumpAssignment) -> OutputLoadConstraint = NamedTuple('OutputLoadConstraint', [ ('name', str), - ('load', float) + ('load', CapacitanceValue) ]) diff --git a/src/hammer-vlsi/hammer_vlsi/driver.py b/src/hammer-vlsi/hammer_vlsi/driver.py index 08aa04d33..4e519a7e1 100644 --- a/src/hammer-vlsi/hammer_vlsi/driver.py +++ b/src/hammer-vlsi/hammer_vlsi/driver.py @@ -217,6 +217,7 @@ def set_up_synthesis_tool(self, syn_tool: HammerSynthesisTool, self.database.get_setting("vlsi.inputs.hierarchical.mode")) syn_tool.input_files = self.database.get_setting("synthesis.inputs.input_files") syn_tool.top_module = self.database.get_setting("synthesis.inputs.top_module", nullvalue="") + syn_tool.is_physical = self.database.get_setting("synthesis.inputs.is_physical") syn_tool.submit_command = HammerSubmitCommand.get("synthesis", self.database) syn_tool.output_all_regs = [] syn_tool.output_seq_cells = [] diff --git a/src/hammer-vlsi/hammer_vlsi/hammer_tool.py b/src/hammer-vlsi/hammer_vlsi/hammer_tool.py index 2519f6aa9..36e992d2e 100644 --- a/src/hammer-vlsi/hammer_vlsi/hammer_tool.py +++ b/src/hammer-vlsi/hammer_vlsi/hammer_tool.py @@ -17,7 +17,7 @@ import hammer_config import hammer_tech -from hammer_logging import HammerVLSILoggingContext +from hammer_logging import HammerVLSILoggingContext, HammerVLSILogging from hammer_tech import LibraryFilter, Stackup, RoutingDirection, Metal from hammer_utils import (add_lists, assert_function_type, get_or_else, optional_map) @@ -26,7 +26,7 @@ from .hammer_vlsi_impl import HammerToolPauseException, HierarchicalMode from .hooks import (HammerStepFunction, HammerToolHookAction, HammerToolStep, HookLocation) -from .submit_command import HammerSubmitCommand +from .submit_command import HammerSubmitCommand, HammerSubmitResult from .units import TemperatureValue, TimeValue, VoltageValue __all__ = ['HammerTool'] @@ -829,7 +829,7 @@ def replace_tcl_set(variable: str, value: str, tcl_path: str, quotes: bool = Tru f.write(new_tcl_contents) # TODO(edwardw): consider pulling this out so that hammer_tech can also use this - def run_executable(self, args: List[str], cwd: str = None) -> str: + def run_executable(self, args: List[str], cwd: str = None) -> HammerSubmitResult: """ Run an executable and log the command to the log while also capturing the output. @@ -837,8 +837,15 @@ def run_executable(self, args: List[str], cwd: str = None) -> str: :param cwd: Working directory (leave as None to use the current working directory). :return: Output from the command or an error message. """ + # Temporarily disable colours/tag to make run output more readable. + # TODO: think of a more elegant way to do this? + HammerVLSILogging.enable_colour = False + HammerVLSILogging.enable_tag = False + ret = self.submit_command.submit(args, self._subprocess_env, self.logger, cwd) + HammerVLSILogging.enable_colour = True + HammerVLSILogging.enable_tag = True + return ret - return self.submit_command.submit(args, self._subprocess_env, self.logger, cwd) # TODO: these helper functions might get a bit out of hand, put them somewhere more organized? def get_clock_ports(self) -> List[ClockPort]: @@ -1130,7 +1137,7 @@ def get_output_load_constraints(self) -> List[OutputLoadConstraint]: for load_src in output_loads: load = OutputLoadConstraint( name=str(load_src["name"]), - load=float(load_src["load"]) + load=CapacitanceValue(load_src["load"]) ) output.append(load) return output @@ -1185,3 +1192,18 @@ def verbose_tcl_append(cmd: str, output_buffer: List[str], clean: bool = False) cleaned = cleandoc(cmd) if clean else cmd output_buffer.append("""puts "{0}" """.format(cleaned.replace('"', '\"'))) output_buffer.append(cleaned) + + @staticmethod + def verbose_tcl_append_wrap(cmd: str, output_buffer: List[str], + clean: bool = False) -> None: + """ + Helper function to echo and run a command using a wrapper function. + + :param cmd: TCL command to run + :param output_buffer: Buffer in which to enqueue the resulting TCL lines + :param clean: True if you want to trim the leading + indendation from the string, False otherwise + """ + cleaned = cleandoc(cmd) if clean else cmd + output_buffer.append("HAMMERCMD { "+cleaned+" }\n") + diff --git a/src/hammer-vlsi/hammer_vlsi/hammer_vlsi_impl.py b/src/hammer-vlsi/hammer_vlsi/hammer_vlsi_impl.py index 3878c0df8..dfe30d8cb 100644 --- a/src/hammer-vlsi/hammer_vlsi/hammer_vlsi_impl.py +++ b/src/hammer-vlsi/hammer_vlsi/hammer_vlsi_impl.py @@ -14,6 +14,7 @@ import os import sys import json +import textwrap from typing import Callable, Iterable, List, NamedTuple, Optional, Dict, Any, Union from decimal import Decimal @@ -231,7 +232,9 @@ def export_config_outputs(self) -> Dict[str, Any]: # in techX16 you can generate only ever generate a single SRAM per run but can # generate multiple corners at once def generate_all_srams_and_corners(self) -> bool: - srams = reduce(list.__add__, list(map(lambda c: self.generate_all_srams(c), self.get_mmmc_corners()))) # type: List[ExtraLibrary] + srams = list(map(lambda c: self.generate_all_srams(c), self.get_mmmc_corners())) + if len(srams): + srams = reduce(list.__add__, srams) # type: List[ExtraLibrary] self.output_libraries = srams return True @@ -280,6 +283,26 @@ def input_files(self, value: List[str]) -> None: self.attr_setter("_input_files", value) + @property + def is_physical(self) -> bool: + """ + Get the true if the synthesis tool should use lef/def for PLE. + + :return: The true if the synthesis tool should use lef/def for PLE. + """ + try: + return self.attr_getter("_is_physical", None) + except AttributeError: + raise ValueError("Nothing set for the true if the synthesis tool should use lef/def for PLE yet") + + @is_physical.setter + def is_physical(self, value: bool) -> None: + """Set the true if the synthesis tool should use lef/def for PLE.""" + if not (isinstance(value, bool)): + raise TypeError("is_physical must be a bool") + self.attr_setter("_is_physical", value) + + ### Outputs ### @property @@ -1307,23 +1330,42 @@ class HasSDCSupport(HammerTool): @property def sdc_clock_constraints(self) -> str: """Generate TCL fragments for top module clock constraints.""" + output = [] # type: List[str] groups = {} # type: Dict[str, List[str]] ungrouped_clocks = [] # type: List[str] + # set time unit in sdc. this is supported in genus + # TODO: is this supported in innovus + output.append("set_time_unit -nanoseconds 1.0") + output.append("set_load_unit -picofarads 1.0") + #time_unit = self.get_time_unit().value_prefix + self.get_time_unit().unit + clocks = self.get_clock_ports() - time_unit = self.get_time_unit().value_prefix + self.get_time_unit().unit for clock in clocks: - # TODO: FIXME This assumes that library units are always in ns!!! if get_or_else(clock.generated, False): - output.append("create_generated_clock -name {n} -source {m_path} -divide_by {div} {path}". - format(n=clock.name, m_path=clock.source_path, div=clock.divisor, path=clock.path)) + output.append(textwrap.dedent(""" + create_generated_clock \\ + -name {n} \\ + -source {m_path} \\ + -divide_by {div} \\ + {path} + """).format(n=clock.name, m_path=clock.source_path, + div=clock.divisor, path=clock.path)) elif clock.path is not None: - output.append("create_clock {0} -name {1} -period {2}".format(clock.path, clock.name, clock.period.value_in_units(time_unit))) + output.append(textwrap.dedent(""" + create_clock {0} \\ + -name {1} \\ + -period {2} + """ + ).format(clock.path, clock.name, + clock.period.value_in_units("ns"))) else: - output.append("create_clock {0} -name {0} -period {1}".format(clock.name, clock.period.value_in_units(time_unit))) + output.append("create_clock {0} -name {0} -period {1}".format( + clock.name, clock.period.value_in_units("ns"))) if clock.uncertainty is not None: - output.append("set_clock_uncertainty {1} [get_clocks {0}]".format(clock.name, clock.uncertainty.value_in_units(time_unit))) + output.append("set_clock_uncertainty {1} [get_clocks {0}]".format( + clock.name, clock.uncertainty.value_in_units("ns"))) if clock.group is not None: if clock.group in groups: groups[clock.group].append(clock.name) @@ -1332,11 +1374,15 @@ def sdc_clock_constraints(self) -> str: else: ungrouped_clocks.append(clock.name) if len(groups): - output.append("set_clock_groups -asynchronous {grouped} {ungrouped}".format( - grouped = " ".join(["{{ {c} }}".format(c=" ".join(clks)) for clks in groups.values()]), - ungrouped = " ".join(["{{ {c} }}".format(c=clk) for clk in ungrouped_clocks]) - )) - + output.append(textwrap.dedent(""" + set_clock_groups \\ + -asynchronous {grouped} \\ + {ungrouped} + """.format( + grouped=" ".join(["{{ {c} }}" + .format(c=" ".join(clks)) for clks in groups.values()]), + ungrouped=" ".join(["{{ {c} }}" + .format(c=clk) for clk in ungrouped_clocks])))) output.append("\n") return "\n".join(output) @@ -1345,31 +1391,72 @@ def sdc_pin_constraints(self) -> str: """Generate a fragment for I/O pin constraints.""" output = [] # type: List[str] - default_output_load = float(self.get_setting("vlsi.inputs.default_output_load")) + #-------------------------------------------------------------------- - # Specify default load. - output.append("set_load {load} [all_outputs]".format( - load=default_output_load - )) + # set time unit in sdc. this is supported in genus + output.append("set_time_unit -nanoseconds 1.0") + output.append("set_load_unit -picofarads 1.0") + + # Specify default load, transitions + try: + default_output_load = \ + CapacitanceValue(self.get_setting("vlsi.inputs.default_output_load")) + output.append("set_load {load} [all_outputs]".format( + load=default_output_load.value_in_units("pf") + )) + except: + pass + + try: + default_max_transition = \ + TimeValue(self.get_setting("vlsi.inputs.default_max_transition")) + output.append("set_max_transition {slew} [current_design]".format( + slew=default_max_transition.value_in_units("ns") + )) + except: + pass + + try: + default_clock_max_transition = \ + TimeValue(self.get_setting("vlsi.inputs.default_clock_max_transition")) + output.append("set_max_transition {slew} [all_clocks]".format( + slew=default_clock_max_transition.value_in_units("ns") + )) + except: + pass + + try: + default_max_fanout = \ + int(self.get_setting("vlsi.inputs.default_max_fanout")) + output.append("set_max_fanout {slew} [current_design]".format( + slew=default_max_fanout + )) + except: + pass # Also specify loads for specific pins. for load in self.get_output_load_constraints(): output.append("set_load {load} [get_port \"{name}\"]".format( - load=load.load, + load=load.load.value_in_units("pf"), name=load.name )) # Also specify delays for specific pins. for delay in self.get_delay_constraints(): - output.append("set_{direction}_delay {delay} -clock {clock} [get_port \"{name}\"]".format( - delay=delay.delay.value_in_units(self.get_time_unit().value_prefix + self.get_time_unit().unit), + output.append(textwrap.dedent(""" + set_{direction}_delay {delay} \\ + -clock {clock} \\ + [get_port \"{name}\"] + """.format( + delay=delay.delay.value_in_units("ns"), clock=delay.clock, direction=delay.direction, name=delay.name - )) + ))) # Custom sdc constraints that are verbatim appended - custom_sdc_constraints = self.get_setting("vlsi.inputs.custom_sdc_constraints") # type: List[str] + custom_sdc_constraints = \ + self.get_setting("vlsi.inputs.custom_sdc_constraints") # type: List[str] for custom in custom_sdc_constraints: output.append(str(custom)) @@ -1393,11 +1480,27 @@ def output(self) -> List[str]: """ Buffered output to be put in .tcl """ - return self.attr_getter("_output", []) + return self.attr_getter("_output", [textwrap.dedent(""" + proc HAMMERCMD {args} { + puts "\[HAMMER\]: {*}$args" + if {[catch {eval "{*}$args"} err]} { + puts "\[HAMMER-ERROR\]: cmd failed\\n$::errorInfo" + exit 1 + } + }""")]) # Python doesn't have Scala's nice currying syntax (e.g. val newfunc = func(_, fixed_arg)) - def verbose_append(self, cmd: str, clean: bool = False) -> None: - self.verbose_tcl_append(cmd, self.output, clean) + def verbose_append(self, cmd: Union[List[str],str], clean: bool = False) -> None: + if type(cmd) is list: + self.verbose_tcl_append(" \\\n ".join(cmd), self.output, clean) + else: + self.verbose_tcl_append(cmd, self.output, clean) + + def verbose_append_wrap(self, cmd: Union[List[str],str], clean: bool = False) -> None: + if type(cmd) is list: + self.verbose_tcl_append_wrap(" \\\n ".join(cmd), self.output, clean) + else: + self.verbose_tcl_append_wrap(cmd, self.output, clean) def append(self, cmd: str, clean: bool = False) -> None: self.tcl_append(cmd, self.output, clean) @@ -1450,9 +1553,20 @@ def get_timing_libs(self, corner: Optional[MMMCCorner] = None) -> str: pre_filters = optional_map(corner, lambda c: [self.filter_for_mmmc(voltage=c.voltage, temp=c.temp)]) # type: Optional[List[Callable[[hammer_tech.Library],bool]]] - lib_args = self.technology.read_libs([hammer_tech.filters.timing_lib_with_ecsm_filter], - hammer_tech.HammerTechnologyUtils.to_plain_item, - extra_pre_filters=pre_filters) + # ssteffl: we add an option to use nldm for 1 stdcell lib to make + # tool bootup times much quicker. + filters = [] + if self.get_setting("vlsi.inputs.use_nldm_libs") == True: + filters.append(hammer_tech.filters.timing_lib_nldm_filter( + vts=self.get_setting("vlsi.inputs.use_lib_vts"))) + else: + raise Exception("AHHH") + filters.append(hammer_tech.filters.timing_lib_with_ecsm_filter) + + lib_args = self.technology.read_libs(filters, + hammer_tech.HammerTechnologyUtils.to_plain_item, + extra_pre_filters=pre_filters) + return " ".join(lib_args) def get_mmmc_qrc(self, corner: MMMCCorner) -> str: diff --git a/src/hammer-vlsi/hammer_vlsi/submit_command.py b/src/hammer-vlsi/hammer_vlsi/submit_command.py index 74e8e1ac7..d17a85dcb 100644 --- a/src/hammer-vlsi/hammer_vlsi/submit_command.py +++ b/src/hammer-vlsi/hammer_vlsi/submit_command.py @@ -8,9 +8,11 @@ # pylint: disable=bad-continuation +import re +import os import atexit import subprocess -import datetime +from datetime import datetime from abc import abstractmethod from functools import reduce from typing import Any, Dict, List, NamedTuple, Optional @@ -19,77 +21,190 @@ from hammer_logging import HammerVLSILoggingContext from hammer_utils import add_dicts, get_or_else -__all__ = ['HammerSubmitCommand', 'HammerLocalSubmitCommand', +__all__ = ['HammerSubmitResult', + 'HammerSubmitCommand', 'HammerLocalSubmitCommand', 'HammerLSFSettings', 'HammerLSFSubmitCommand'] +#============================================================================= +# SubmitResult +#============================================================================= -class HammerSubmitCommand: - - @abstractmethod - def submit(self, args: List[str], env: Dict[str, str], - logger: HammerVLSILoggingContext, cwd: str = None) -> str: - """ - Submit the job to the job submission system. This function MUST block - until the command is complete. - - :param args: Command-line to run; each item in the list is one token. - The first token should be the command to run. - :param env: The environment variables to set for the command - :param logger: The logging context - :param cwd: Working directory (leave as None to use the current working directory). - :return: The command output - """ - pass +class HammerSubmitResult(NamedTuple('SubmitResult', [ + ('success', bool), + ('code', int), + ('errors', List[str]), + ('output', List[str]) +])): + __slots__ = () - @abstractmethod - def read_settings(self, settings: Dict[str, Any], tool_namespace: str) -> None: - """ - Read the settings object (a Dict[str, Any]) into meaningful class variables. +#============================================================================= +# SubmitCommand base class +#============================================================================= - :param settings: A Dict[str, Any] comprising the settings for this command. - :param tool_namespace: The namespace for the tool (useful for logging). - """ - pass +class HammerSubmitCommand: @staticmethod def get(tool_namespace: str, database: HammerDatabase) -> "HammerSubmitCommand": """ Get a concrete instance of a HammerSubmitCommand for a tool - :param tool_namespace: The tool namespace to use when querying the HammerDatabase (e.g. "synthesis" or "par"). :param database: The HammerDatabase object with tool settings """ - - submit_command_mode = database.get_setting(tool_namespace + ".submit.command", - nullvalue="none") - # pylint: disable=line-too-long - submit_command_settings = database.get_setting(tool_namespace + ".submit.settings", - nullvalue=[]) # type: List[Dict[str, Dict[str, Any]]] + def safe_get( key, default) -> Any: + try: + return database.get_setting(key) + except: + return default + + # TODO: should these be required to exist? + ns = "{}.submit".format(tool_namespace) + mode = safe_get(ns+".command", "none") + settings = safe_get(ns+".settings", []) + max_outputs = safe_get(ns+".max_output_lines", None) + max_errors = safe_get(ns+".max_error_lines", None) + abort_on_error = safe_get(ns+".abort_on_error", False) + error_rgxs = safe_get(ns+".error_rgxs", []) + error_ignore_rgxs = safe_get(ns+".error_ignore_rgxs", []) # Settings is a List[Dict[str, Dict[str, Any]]] object. The first Dict # key is the submit command name. - # Its value is a Dict[str, Any] comprising the settings for that command. + # Its value is a Dict[str, Any] comprising the settings for that + # command. # The top-level list elements are merged from 0 to the last index, with # later indices overriding previous entries. - def combine_settings(settings: List[Dict[str, Dict[str, Any]]], key: str) -> Dict[str, Any]: + def combine_settings(settings: List[Dict[str, Dict[str, Any]]], + key: str) -> Dict[str, Any]: return reduce(add_dicts, map(lambda d: d[key], settings), {}) - submit_command = None # type: Optional[HammerSubmitCommand] - if submit_command_mode in {"none", "local"}: # pylint: disable=no-else-return - # Do not read the options, return immediately - return HammerLocalSubmitCommand() - elif submit_command_mode == "lsf": - submit_command = HammerLSFSubmitCommand() + cmd = None + if (mode == "none") or (mode == "local"): + cmd = HammerLocalSubmitCommand() + cmd.settings = None + elif mode == "lsf": + cmd = HammerLSFSubmitCommand() + settings = combine_settings(settings, mode) + cmd.settings = HammerLSFSettings.from_setting(settings) else: raise NotImplementedError( "Submit command key for {0}: {1} is not implemented".format( tool_namespace, submit_command_mode)) - submit_command.read_settings( - combine_settings(submit_command_settings, submit_command_mode), - tool_namespace) - return submit_command + cmd.max_outputs = max_outputs + cmd.max_errors = max_errors + cmd.abort_on_error = abort_on_error + cmd.error_rgxs = error_rgxs + cmd.error_ignore_rgxs = error_ignore_rgxs + + return cmd + + def write_run_script(self, tag:str, args:List[str], cwd:str=None) -> str: + """ + writes a script that is directly executed. if you run this script + outside hammer, you should get the same results as when hammer ran + """ + cwd = cwd if cwd is not None else os.getcwd() + cmd_script = "{}/{}.sh".format(cwd, tag) + output = ["#!/bin/bash", + "[ -f ./enter ] && source ./enter", + "exec {} \\".format(args[0])] + for arg in args[1:]: + output += [" '{}' \\".format(re.sub("'", "\\'", arg))] + output += [""] + with open(cmd_script, "w") as f: + f.write("\n".join(output)) + os.chmod(cmd_script, 0o755) + return cmd_script + + @abstractmethod + def get_cmd_array(self, tag:str, args: List[str], env: Dict[str, str], + logger: HammerVLSILoggingContext, cwd: str = None) -> List[str]: + raise NotImplementedError() + + def submit(self, args: List[str], env: Dict[str, str], + logger: HammerVLSILoggingContext, + cwd: str = None) -> HammerSubmitResult: + """ + Submit the job to the job submission system. This function MUST block + until the command is complete. + :param args: Command-line to run; each item in the list is one token. + The first token should be the command to run. + :param env: The environment variables to set for the command + :param logger: The logging context + :param cwd: Working directory (leave as None to use the current + working directory). + :return: The command output + """ + cwd = cwd if cwd is not None else os.getcwd() + prog_tag = self.get_program_tag(args) + + subprocess_logger = logger.context("Exec " + prog_tag) + + proc = subprocess.Popen( + self.get_cmd_array("submit_command", args, env, subprocess_logger, cwd), + shell=False, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + env=env, + cwd=cwd) + + output_lines = [] + log_file = "{}/{}.log".format(cwd, "submit_command") + output_clipped = False + + with open(log_file, "w") as f: + def print_to_all(line): + f.write(line) + subprocess_logger.debug(line.rstrip()) + if (self.max_outputs is not None) and \ + (len(output_lines) > int(self.max_outputs)): + output_clipped = True + else: + output_lines.append(line.rstrip()) + + while True: + line = proc.stdout.readline().decode("utf-8") + if line != '': + print_to_all(line) + else: + break + + if output_clipped: + print_to_all("[HAMMER]: max output lines exeeded...") + + proc.communicate() + code = proc.returncode + success = (code == 0) + + if self.abort_on_error and (code != 0): + raise ChildProcessError("Failed command: {}, code={}" + .format(prog_tag, code)) + + # check the output_lines for matching error strings + error_lines = [] + for line in output_lines: + if len(list(filter(lambda r: re.search(r, line), + self.error_rgxs))) > 0: + if len(list(filter(lambda r: re.search(r, line), + self.error_ignore_rgxs))) == 0: + success = False + if (self.max_errors is not None) and \ + (len(error_lines) >= int(self.max_errors)): + error_lines.append("[HAMMER]: max errors exceeded...") + break + else: + error_lines.append(line) + + if self.abort_on_error and (len(error_lines) > 0): + raise ChildProcessError("Failed command: {}, error in output={}" + .format(prog_tag, error_lines[0])) + + return HammerSubmitResult( + success=success, + code=code, + errors=error_lines, + output=output_lines) + @staticmethod def get_program_tag(args: List[str], program_name_length: int = 14, @@ -116,37 +231,21 @@ def get_program_tag(args: List[str], program_name_length: int = 14, return prog_name + " " + prog_args -class HammerLocalSubmitCommand(HammerSubmitCommand): - - def submit(self, args: List[str], env: Dict[str, str], - logger: HammerVLSILoggingContext, cwd: str = None) -> str: - # Just run the command on this host. +#============================================================================= +# HammerLocalSubmit +#============================================================================= - prog_tag = self.get_program_tag(args) - - logger.debug("Executing subprocess: " + ' '.join(args)) - subprocess_logger = logger.context("Exec " + prog_tag) - proc = subprocess.Popen(args, shell=False, stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, env=env, cwd=cwd) - atexit.register(proc.kill) - - output_buf = "" - # Log output and also capture output at the same time. - while True: - line = proc.stdout.readline().decode("utf-8") - if line != '': - subprocess_logger.debug(line.rstrip()) - output_buf += line - else: - break - # TODO: check errors +class HammerLocalSubmitCommand(HammerSubmitCommand): - return output_buf + def get_cmd_array(self, tag:str, args: List[str], env: Dict[str, str], + logger: HammerVLSILoggingContext, cwd: str = None) -> List[str]: - def read_settings(self, settings: Dict[str, Any], tool_namespace: str) -> None: - # Should never get here - raise ValueError("Local submission command does not have settings") + logger.debug('Executing subprocess: "{}"'.format(' '.join(args))) + return [self.write_run_script(tag=tag, args=args, cwd=cwd)] +#============================================================================= +# HammerLSFSubmit +#============================================================================= class HammerLSFSettings(NamedTuple('HammerLSFSettings', [ ('bsub_binary', str), @@ -158,7 +257,7 @@ class HammerLSFSettings(NamedTuple('HammerLSFSettings', [ __slots__ = () @staticmethod - def from_setting(settings: Dict[str, Any]) -> "HammerLSFSettings": + def from_setting(settings: Dict[str, Any]) -> 'HammerLSFSettings': if not isinstance(settings, dict): raise ValueError("Must be a dictionary") try: @@ -186,33 +285,24 @@ def from_setting(settings: Dict[str, Any]) -> "HammerLSFSettings": extra_args=get_or_else(settings["extra_args"], []) ) - class HammerLSFSubmitCommand(HammerSubmitCommand): - # TODO(johnwright): log the command output + def get_cmd_array(self, tag:str, args: List[str], env: Dict[str, str], + logger: HammerVLSILoggingContext, cwd: str = None) -> List[str]: - @property - def settings(self) -> HammerLSFSettings: - if not hasattr(self, "_settings"): - raise ValueError("Nothing set for settings yet") - return getattr(self, "_settings") + logger.debug('Executing subprocess: {bsub_args} "{args}"'.format( + bsub_args=' '.join(self.bsub_args()), + args=' '.join(args))) - @settings.setter - def settings(self, value: HammerLSFSettings) -> None: - """ - Set the settings class variable - - :param value: The HammerLSFSettings NapedTuple to use - """ - setattr(self, "_settings", value) - - def read_settings(self, settings: Dict[str, Any], tool_namespace: str) -> None: # pylint: disable=unused-argument - self.settings = HammerLSFSettings.from_setting(settings) + return self.bsub_args() + \ + [self.write_run_script(tag=tag, args=args, cwd=cwd)] def bsub_args(self) -> List[str]: args = [self.settings.bsub_binary, "-K"] # always use -K to block - args.extend(["-o", self.settings.log_file if self.settings.log_file is not None else - datetime.datetime.now().strftime("hammer-vlsi-bsub-%Y%m%d-%H%M%S.log")]) # always use -o to log to a file + log_file = self.settings.log_file \ + if self.settings.log_file is not None \ + else datetime.now().strftime("hammer-vlsi-bsub-%Y%m%d-%H%M%S.log") + args.extend(["-o", log_file]) if self.settings.queue is not None: args.extend(["-q", self.settings.queue]) if self.settings.num_cpus is not None: @@ -220,29 +310,5 @@ def bsub_args(self) -> List[str]: args.extend(self.settings.extra_args) return args - def submit(self, args: List[str], env: Dict[str, str], - logger: HammerVLSILoggingContext, cwd: str = None) -> str: - # TODO fix output capturing - prog_tag = self.get_program_tag(args) - - subprocess_format_str = 'Executing subprocess: {bsub_args} "{args}"' - logger.debug(subprocess_format_str.format(bsub_args=' '.join(self.bsub_args()), - args=' '.join(args))) - subprocess_logger = logger.context("Exec " + prog_tag) - proc = subprocess.Popen(self.bsub_args() + [' '.join(args)], - shell=False, stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, env=env, cwd=cwd) - - output_buf = "" - # Log output and also capture output at the same time. - while True: - line = proc.stdout.readline().decode("utf-8") - if line != '': - subprocess_logger.debug(line.rstrip()) - output_buf += line - else: - break - # TODO: check errors - return output_buf diff --git a/src/hammer-vlsi/hammer_vlsi/units.py b/src/hammer-vlsi/hammer_vlsi/units.py index 81409a000..e6f412260 100644 --- a/src/hammer-vlsi/hammer_vlsi/units.py +++ b/src/hammer-vlsi/hammer_vlsi/units.py @@ -272,3 +272,22 @@ def unit(self) -> str: @property def unit_type(self) -> str: return "voltage" + +class CapacitanceValue(ValueWithUnit): + """Capacitance value in Farads - e.g. "1 fF", "12 fF". + Mainly used for specifying load constraints in SDC + """ + + @property + def default_prefix(self) -> str: + """Default is picofarads""" + return "p" + + @property + def unit(self) -> str: + return "F" + + @property + def unit_type(self) -> str: + return "capacitance" + diff --git a/src/hammer-vlsi/submit_command.sh b/src/hammer-vlsi/submit_command.sh new file mode 100755 index 000000000..ad141ea6d --- /dev/null +++ b/src/hammer-vlsi/submit_command.sh @@ -0,0 +1,6 @@ +#!/bin/bash +[ -f ./enter ] && source ./enter +exec echo \ + 'go' \ + 'bears' \ + '!' \ diff --git a/src/hammer-vlsi/synthesis/mocksynth/__init__.py b/src/hammer-vlsi/synthesis/mocksynth/__init__.py index 6608378cf..fa9e7727a 100644 --- a/src/hammer-vlsi/synthesis/mocksynth/__init__.py +++ b/src/hammer-vlsi/synthesis/mocksynth/__init__.py @@ -31,6 +31,7 @@ def temp_file(self, filename: str) -> str: def steps(self) -> List[HammerToolStep]: return self.make_steps_from_methods([ self.step1, + self.step1a, self.step2, self.step3, self.step4 @@ -48,6 +49,33 @@ def step1(self) -> bool: return False return True + def step1a(self) -> bool: + submit_pass = None + submit_fail = None + try: + submit_pass = \ + self.get_setting("synthesis.mocksynth.fake_submit_pass") + except: + pass + + try: + submit_fail =\ + self.get_setting("synthesis.mocksynth.fake_submit_fail") + except: + pass + + try: + if submit_pass is not None and submit_pass: + self.run_executable(["cat", self.temp_file("step1.txt")], + cwd=self.run_dir) + elif submit_fail is not None and submit_fail: + self.run_executable(["i_hope_this_isnt_in_your_path"], + cwd=self.run_dir) + except Exception: + return False + return True + + def step2(self) -> bool: try: with open(self.temp_file("step2.txt"), "w") as f: diff --git a/src/hammer-vlsi/test.py b/src/hammer-vlsi/test.py index 1ca9a916b..8a9ac58a7 100644 --- a/src/hammer-vlsi/test.py +++ b/src/hammer-vlsi/test.py @@ -1122,10 +1122,10 @@ def step5(x: hammer_vlsi.HammerTool) -> bool: else: self.assertFalse(os.path.exists(file)) - class HammerSubmitCommandTestContext: - def __init__(self, test: unittest.TestCase, cmd_type: str) -> None: + def __init__(self, test: unittest.TestCase, cmd_type: str, + extra_configs: Dict[str, str]={}) -> None: self.echo_command_args = ["go", "bears", "!"] self.echo_command = ["echo"] + self.echo_command_args self.test = test # type unittest.TestCase @@ -1135,6 +1135,7 @@ def __init__(self, test: unittest.TestCase, cmd_type: str) -> None: raise NotImplementedError("Have not built a test for %s yet" % cmd_type) self._cmd_type = cmd_type self._submit_command = None # type: Optional[hammer_vlsi.HammerSubmitCommand] + self.extra_configs = extra_configs # Helper property to check that the driver did get initialized. @property @@ -1160,7 +1161,27 @@ def __enter__(self) -> "HammerSubmitCommandTestContext": "synthesis.inputs.top_module": "dummy", "synthesis.inputs.input_files": ("/dev/null",), "synthesis.mocksynth.temp_folder": temp_dir, - "synthesis.submit.command": self._cmd_type + "synthesis.submit.command": self._cmd_type, + # NOTE: we should be using the defaults.yml here... + "vlsi.submit.settings": [], + "vlsi.submit.max_output_lines": None, + "vlsi.submit.max_error_lines": None, + "vlsi.submit.abort_on_error": False, + "vlsi.submit.error_rgxs": ["ERROR:", "error:", "Error:"], + "vlsi.submit.error_ignore_rgxs": [], + "synthesis.submit.settings": "vlsi.submit.settings", + "synthesis.submit.max_output_lines": "${vlsi.submit.max_output_lines}", + "synthesis.submit.max_error_lines": "${vlsi.submit.max_error_lines}", + "synthesis.submit.abort_on_error": "${vlsi.submit.abort_on_error}", + "synthesis.submit.error_rgxs": "vlsi.submit.error_rgxs", + "synthesis.submit.error_ignore_rgxs": "vlsi.submit.error_ignore_rgxs", + "synthesis.submit.command_meta": "lazysubst", + "synthesis.submit.settings_meta": "lazytrycrossref", + "synthesis.submit.max_output_lines_meta": "lazytrysubst", + "synthesis.submit.max_error_lines_meta": "lazytrysubst", + "synthesis.submit.abort_on_error_meta": "lazytrysubst", + "synthesis.submit.error_rgxs_meta": "lazytrycrossref", + "synthesis.submit.error_ignore_rgxs_meta": "lazytrycrossref" } if self._cmd_type is "lsf": json_content.update({ @@ -1178,6 +1199,7 @@ def __enter__(self) -> "HammerSubmitCommandTestContext": ], "vlsi.submit.settings_meta": "lazyappend" }) + json_content.update(self.extra_configs) with open(json_path, "w") as f: f.write(json.dumps(json_content, cls=HammerJSONEncoder, indent=4)) @@ -1196,8 +1218,7 @@ def __enter__(self) -> "HammerSubmitCommandTestContext": def __exit__(self, type, value, traceback) -> bool: """Clean up the context by removing the temp_dir.""" shutil.rmtree(self.temp_dir) - # Return True (normal execution) if no exception occurred. - return True if type is None else False + return False @property def database(self) -> hammer_config.HammerDatabase: @@ -1207,42 +1228,145 @@ def database(self) -> hammer_config.HammerDatabase: def env(self) -> Dict[str, str]: return {} - class HammerSubmitCommandTest(unittest.TestCase): - def create_context(self, cmd_type: str) -> HammerSubmitCommandTestContext: - return HammerSubmitCommandTestContext(self, cmd_type) + def create_context(self, cmd_type: str, + extra_configs:Dict[str,str]={}) -> HammerSubmitCommandTestContext: + return HammerSubmitCommandTestContext(self, cmd_type, extra_configs) - def test_local_submit(self) -> None: - """ Test that a local submission produces the desired output """ + def syn_error_texts(self): + return [ + {"synthesis.mocksynth.step1": "a\nb\nfoo\nbar ERROR: baz\n"}, + {"synthesis.mocksynth.step1": "a\nb\nfoo\nbar error: baz\n"} + ] + + def expect_syn(self, context, result): + context.driver.load_synthesis_tool() + self.assertEqual(context.driver.run_synthesis()[0], result) + + def test_all_commands(self) -> None: + """ Test that all commands behave certain similar ways""" + for cmd in ["local", "lsf"]: + #for cmd in ["lsf"]: + for lvl in ["vlsi", "synthesis"]: + #for lvl in ["synthesis"]: + # command fails when exit code is not 0 and abort set to true + config = {"synthesis.mocksynth.fake_submit_fail": True, + "{}.submit.abort_on_error".format(lvl): True } + with self.create_context(cmd, config) as c: + self.expect_syn(c, False) + + # command succeeds when exit code is not 0 and abort set to false + config = {"synthesis.mocksynth.fake_submit_fail": True, + "{}.submit.abort_on_error".format(lvl): False } + with self.create_context(cmd, config) as c: + self.expect_syn(c, True) + + # command succeeds when exit code is 0 + config = {"synthesis.mocksynth.fake_submit_pass": True, + "{}.submit.abort_on_error".format(lvl): True } + with self.create_context(cmd, config) as c: + self.expect_syn(c, True) + + # processing text output + for err_text in self.syn_error_texts(): + err_text = add_dicts(err_text, { + "synthesis.mocksynth.fake_submit_pass": True, + "{}.submit.abort_on_error".format(lvl): True + }) + + # should capture max_output_lines if set + config = add_dicts(err_text, {"{}.submit.max_output_lines".format(lvl): 1}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, True) + + config = add_dicts(err_text, {"{}.submit.max_output_lines".format(lvl): 20}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, False) + + # should capture max_error_lines if set + config = add_dicts(err_text, {"{}.submit.max_error_lines".format(lvl): 0}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, False) + + config = add_dicts(err_text, {"{}.submit.max_error_lines".format(lvl): 20}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, False) + + # should not error when an error regex occurs and not abort_on_error + config = add_dicts(err_text, {"{}.submit.abort_on_error".format(lvl): False }) + with self.create_context(cmd, config) as c: + self.expect_syn(c, True) + + # should error when an error regex occurs and not abort_on_error + config = add_dicts(err_text, {"{}.submit.abort_on_error".format(lvl): True }) + with self.create_context(cmd, config) as c: + self.expect_syn(c, False) + + # should not error when user overrides error_rgxs and an error regex occurs + config = add_dicts(err_text, {"{}.submit.error_rgxs".format(lvl): ["NO_MATCH"]}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, True) + + config = add_dicts(err_text, {"{}.submit.error_rgxs".format(lvl): []}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, True) + + # should not error when user overrides ignore_error_rgxs and an error regex occurs + config = add_dicts(err_text, + {"{}.submit.error_ignore_rgxs".format(lvl): ["ERROR:", "Error:", "error:"]}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, True) + + # should error when user overrides error_rgxs and an error regex occurs + config = add_dicts(err_text, {"{}.submit.error_rgxs".format(lvl): ["baz"]}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, False) + + # should not error when user overrides error_rgx and ignore_error_rgxs and an error regex occurs + config = add_dicts(err_text, { + "{}.submit.error_rgxs".format(lvl): ["baz"], + "{}.submit.error_ignore_rgxs".format(lvl): ["ba"]}) + with self.create_context(cmd, config) as c: + self.expect_syn(c, True) + + def test_lsf_submit(self) -> None: + """ Test that an local submission produces the desired output """ with self.create_context("local") as c: cmd = c.submit_command - output = cmd.submit(c.echo_command, c.env, c.logger).splitlines() + res = cmd.submit(c.echo_command, c.env, c.logger) - self.assertEqual(output[0], ' '.join(c.echo_command_args)) + self.assertEqual(res.success, True) + self.assertEqual(res.code, 0) + self.assertEqual(len(res.errors), 0) + self.assertEqual(res.output[0], ' '.join(c.echo_command_args)) def test_lsf_submit(self) -> None: """ Test that an LSF submission produces the desired output """ with self.create_context("lsf") as c: cmd = c.submit_command assert isinstance(cmd, hammer_vlsi.HammerLSFSubmitCommand) - output = cmd.submit(c.echo_command, c.env, c.logger).splitlines() + res = cmd.submit(c.echo_command, c.env, c.logger, c.temp_dir) - self.assertEqual(output[0], "BLOCKING is: 1") - self.assertEqual(output[1], "QUEUE is: %s" % get_or_else(cmd.settings.queue, "")) - self.assertEqual(output[2], "NUMCPU is: %d" % get_or_else(cmd.settings.num_cpus, 0)) - self.assertEqual(output[3], "OUTPUT is: %s" % get_or_else(cmd.settings.log_file, "")) + self.assertEqual(res.success, True) + self.assertEqual(res.code, 0) + self.assertEqual(len(res.errors), 0) + self.assertEqual(res.output[0], "BLOCKING is: 1") + self.assertEqual(res.output[1], "QUEUE is: %s" % get_or_else(cmd.settings.queue, "")) + self.assertEqual(res.output[2], "NUMCPU is: %d" % get_or_else(cmd.settings.num_cpus, 0)) + self.assertEqual(res.output[3], "OUTPUT is: %s" % get_or_else(cmd.settings.log_file, "")) extra = cmd.settings.extra_args has_resource = 0 if "-R" in extra: has_resource = 1 - self.assertEqual(output[4], "RESOURCE is: %s" % extra[extra.index("-R") + 1]) + self.assertEqual(res.output[4], "RESOURCE is: %s" % extra[extra.index("-R") + 1]) else: raise NotImplementedError("You forgot to test the extra_args!") - self.assertEqual(output[4 + has_resource], "COMMAND is: %s" % ' '.join(c.echo_command)) - self.assertEqual(output[5 + has_resource], ' '.join(c.echo_command_args)) + # not longer correct + #self.assertEqual(res.output[4 + has_resource], "COMMAND is: %s" % ' '.join(c.echo_command)) + self.assertEqual(res.output[5 + has_resource], ' '.join(c.echo_command_args)) class HammerSignoffToolTestContext: @@ -1893,4 +2017,5 @@ def test_deliverables_exist(self) -> None: if __name__ == '__main__': + #unittest.main(verbosity=2) unittest.main() diff --git a/src/hammer_config/config_src.py b/src/hammer_config/config_src.py index f7a2e2404..1af6af73a 100644 --- a/src/hammer_config/config_src.py +++ b/src/hammer_config/config_src.py @@ -235,9 +235,11 @@ def perform_subst(value: Union[str, List[str]]) -> Union[str, List[str]]: newval = "" # type: Union[str, List[str]] if isinstance(value, list): - newval = list(map(lambda input_str: subst_str(input_str, lambda key: config_dict[key]), value)) + #newval = list(map(lambda input_str: subst_str(input_str, lambda key: config_dict[key]), value)) + newval = list(map(lambda input_str: trysubst_str(input_str, + lambda key: config_dict[key]), value)) else: - newval = subst_str(value, lambda key: config_dict[key]) + newval = trysubst_str(value, lambda key: config_dict[key]) return newval config_dict[key] = perform_subst(value) @@ -278,6 +280,68 @@ def subst_rename(key: str, value: Any, target_setting: str, replacement_setting: target_settings=subst_targets, rename_target=subst_rename) + def trysubst_str(input_str: str, replacement_func: Callable[[str], str]) -> str: + """Substitute ${...}""" + + # this check allows a reference like "${foo}" to just be replaced + # instead of using regexes. therefore substituting int/bool/null works + split = re.split(__VARIABLE_EXPANSION_REGEX, input_str) + if len(split) == 3 and split[0] == "" and split[2] == "" and \ + input_str == "${" + split[1] + "}": + return replacement_func(split[1]) + else: + return re.sub(__VARIABLE_EXPANSION_REGEX, lambda x: replacement_func(x.group(1)), input_str) + + def trysubst_action(config_dict: dict, key: str, value: Any, params: MetaDirectiveParams) -> None: + def perform_subst(value: Union[str, List[str]]) -> Union[str, List[str]]: + """ + Perform substitutions for the given value. + If value is a string, perform substitutions in the string. + If value is a list, then perform substitutions + in every string in the list. + :param value: String or list + :return: String or list but with everything substituted. + """ + newval = "" # type: Union[str, List[str]] + + if isinstance(value, list): + return list(map(lambda input_str: + trysubst_str(input_str, lambda key: config_dict[key]), value)) + elif isinstance(value, str): + return trysubst_str(value, lambda key: config_dict[key]) + else: + return value + + config_dict[key] = perform_subst(value) + + def trysubst_targets(key: str, value: Any) -> List[str]: + # subst can operate on either a string or a list + + # subst_strings is e.g. ["${a} 1", "${b} 2"] + subst_strings = [] # type: List[str] + if isinstance(value, str): + subst_strings.append(value) + elif isinstance(value, list): + for i in value: + assert isinstance(i, str) + subst_strings = value + else: + return [] + + output_vars = [] # type: List[str] + + for subst_value in subst_strings: + matches = re.finditer(__VARIABLE_EXPANSION_REGEX, subst_value, re.DOTALL) + for match in matches: + output_vars.append(match.group(1)) + + return output_vars + + directives['trysubst'] = MetaDirective(action=trysubst_action, + target_settings=trysubst_targets, + rename_target=subst_rename) + + def crossref_check_and_cast(k: Any) -> str: if not isinstance(k, str): raise ValueError("crossref (if used with lists) can only be used only with lists of strings") @@ -336,6 +400,44 @@ def change_if_target(x: str) -> str: target_settings=crossref_targets, rename_target=crossref_rename) + def trycrossref_action(config_dict: dict, key: str, value: Any, params: MetaDirectiveParams) -> None: + """ + Similar to crossref, but it is supposed to work whether you override + it with a concrete value downstream. For example. if originally a + key defaulted to a trycrossref of 'vlsi.foo.bar', and a later config + changed it to [1,2,3], this will be valid, and [1,2,3] is the final + value returned. + """ + if isinstance(value, str): + if value in config_dict: + config_dict[key] = config_dict[value] + else: + config_dict[key] = value + + def trycrossref_targets(key: str, value: Any) -> List[str]: + if type(value) == str: + return [value] + return [] + + def trycrossref_rename(key: str, value: Any, target_setting: str, + replacement_setting: str) -> Optional[ + Tuple[Any, str]]: + def change_if_target(x: str) -> str: + if x == target_setting: + return replacement_setting + else: + return x + + if isinstance(value, str): + return [change_if_target(value)], "trycrossref" + else: + raise NotImplementedError("trycrossref not implemented on other types yet") + + directives['trycrossref'] = MetaDirective(action=trycrossref_action, + target_settings=trycrossref_targets, + rename_target=trycrossref_rename) + + def transclude_action(config_dict: dict, key: str, value: Any, params: MetaDirectiveParams) -> None: """Transclude the contents of the file pointed to by value.""" assert isinstance(value, str), "Path to file for transclusion must be a string" diff --git a/src/hammer_config_test/__init__.py b/src/hammer_config_test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/hammer_config_test/test.py b/src/hammer_config_test/test.py index e98102e86..9856a8cd7 100644 --- a/src/hammer_config_test/test.py +++ b/src/hammer_config_test/test.py @@ -137,6 +137,30 @@ def test_meta_subst(self) -> None: self.assertEqual(db.get_setting("foo.pipeline"), "yesman") self.assertEqual(db.get_setting("foo.uint"), ["1", "2"]) + def test_meta_trysubst(self) -> None: + """ + Test that the meta attribute "trysubst" works on non-string references + """ + db = hammer_config.HammerDatabase() + base = hammer_config.load_config_from_string(""" +foo: + flash: null + one: true + two: 2 +""", is_yaml=True) + meta = hammer_config.load_config_from_string(""" +{ + "foo.pipeline": "${foo.flash}", + "foo.pipeline_meta": "trysubst", + "foo.uint": ["${foo.one}", "${foo.two}"], + "foo.uint_meta": "trysubst" +} +""", is_yaml=False) + db.update_core([base, meta]) + self.assertEqual(db.get_setting("foo.flash"), None) + self.assertEqual(db.get_setting("foo.pipeline"), None) + self.assertEqual(db.get_setting("foo.uint"), [True, 2]) + def test_meta_lazysubst(self) -> None: """ Test that the meta attribute "lazysubst" works. @@ -176,6 +200,46 @@ def test_meta_lazysubst(self) -> None: self.assertEqual(db.get_setting("foo.later"), "later") self.assertEqual(db.get_setting("foo.methodology"), "agile design") + def test_meta_lazytrysubst(self) -> None: + """ + Test that the meta attribute "lazytrysubst" works. + """ + db = hammer_config.HammerDatabase() + base = hammer_config.load_config_from_string(""" +foo: + flash: 10 + one: 1 + two: 2 +style: "waterfall" +""", is_yaml=True) + meta = hammer_config.load_config_from_string(""" +{ + "foo.pipeline": "${foo.flash}", + "foo.pipeline_meta": "trysubst", + "foo.reg": "Wire", + "foo.reginit": "${foo.reg}Init", + "foo.reginit_meta": "lazytrysubst", + "foo.later": "${later}", + "foo.later_meta": "lazytrysubst", + "foo.methodology": "${style}", + "foo.methodology_meta": "lazytrysubst" +} +""", is_yaml=False) + project = hammer_config.load_config_from_string(""" +{ + "later": "later", + "style": True +} +""", is_yaml=True) + db.update_core([base, meta]) + db.update_project([project]) + self.assertEqual(db.get_setting("foo.flash"), 10) + self.assertEqual(db.get_setting("foo.pipeline"), 10) + self.assertEqual(db.get_setting("foo.reginit"), "WireInit") + self.assertEqual(db.get_setting("foo.later"), "later") + self.assertEqual(db.get_setting("foo.methodology"), True) + + def test_meta_lazysubst_array(self) -> None: """ Check that lazysubst works correctly with an array. @@ -280,6 +344,33 @@ def test_meta_crossref(self) -> None: self.assertEqual(db.get_setting("bools"), [False, True]) self.assertEqual(db.get_setting("indirect.numbers"), False) + def test_meta_trycrossref(self) -> None: + """ + Test that the meta attribute "trycrossref" works. + """ + db = hammer_config.HammerDatabase() + base = hammer_config.load_config_from_string(""" +my: + numbers: ["1", "2", "3"] +""", is_yaml=True) + meta = hammer_config.load_config_from_string(""" +{ + "just.numbers": "my.numbers", + "just.numbers_meta": "trycrossref", + "just.numbers2": [1,3,5], + "just.numbers2_meta": "trycrossref", + "just.numbers3": null, + "just.numbers3_meta": "trycrossref", + "just.numbers4": [True, "3"], + "just.numbers4_meta": "trycrossref", +} +""", is_yaml=True) + db.update_core([base, meta]) + self.assertEqual(db.get_setting("just.numbers"), ["1", "2", "3"]) + self.assertEqual(db.get_setting("just.numbers2"), [1,3,5]) + self.assertEqual(db.get_setting("just.numbers3"), None) + self.assertEqual(db.get_setting("just.numbers4"), [True, "3"]) + def test_meta_lazycrossref(self) -> None: """ Test that lazy crossref works. @@ -298,6 +389,27 @@ def test_meta_lazycrossref(self) -> None: db.update_core([base, meta]) self.assertEqual(db.get_setting("lazy.numbers"), ["1", "2", "3"]) + def test_meta_lazytrycrossref(self) -> None: + """ + Test that lazy trycrossref works. + """ + db = hammer_config.HammerDatabase() + base = hammer_config.load_config_from_string(""" +my: + numbers: ["1", "2", "3"] + """, is_yaml=True) + meta = hammer_config.load_config_from_string(""" +numbers: "my.numbers" +numbers_meta: trycrossref +lazy.numbers: "numbers" +lazy.numbers_meta: lazytrycrossref +lazy.numbers2: [1,2,3] +lazy.numbers2_meta: lazytrycrossref + """, is_yaml=True) + db.update_core([base, meta]) + self.assertEqual(db.get_setting("lazy.numbers"), ["1", "2", "3"]) + self.assertEqual(db.get_setting("lazy.numbers2"), [1,2,3]) + def test_meta_crossref_errors(self) -> None: """ Test that the meta attribute "crossref" raises errors appropriately. diff --git a/src/test/.gitignore b/src/test/.gitignore new file mode 100644 index 000000000..9dfda6376 --- /dev/null +++ b/src/test/.gitignore @@ -0,0 +1 @@ +submit_command.sh