From 54a5bc3b2c537b6403e6f760251bfd7d1bc2b526 Mon Sep 17 00:00:00 2001 From: Xinyang Li Date: Wed, 18 Sep 2024 14:49:54 -0600 Subject: [PATCH] Add example scripts and update hyperopt docs --- docs/source/examples/hyperopt.rst | 96 +++++--- .../QM7_ax_example.py | 203 +++++++++++++++++ .../hyperparameter_optimization/ax_opt.py | 124 ++++++++++ .../hyperparameter_optimization/ax_opt_ray.py | 215 ++++++++++++++++++ .../prameters.json | 29 +++ 5 files changed, 637 insertions(+), 30 deletions(-) create mode 100644 examples/hyperparameter_optimization/QM7_ax_example.py create mode 100644 examples/hyperparameter_optimization/ax_opt.py create mode 100644 examples/hyperparameter_optimization/ax_opt_ray.py create mode 100644 examples/hyperparameter_optimization/prameters.json diff --git a/docs/source/examples/hyperopt.rst b/docs/source/examples/hyperopt.rst index eb52bb42..7846ac54 100644 --- a/docs/source/examples/hyperopt.rst +++ b/docs/source/examples/hyperopt.rst @@ -7,24 +7,28 @@ sequentially (with Ax) or parallelly (with Ax and Ray). Prerequisites ------------- -The packages required to perform this task are `Ax`_ and `ray`_.:: +The packages required to perform this task are `Ax`_ and `ray`_. +:: conda install -c conda-forge "ray < 2.7.0" - pip install ax-platform + pip install ax-platform!=0.4.1 .. note:: - The scripts have been tested with `ax-platform 0.3.1` and `ray 2.3.0`, and - some previous versions of the two packages. Unfortunately, several changes + The scripts have been tested with `ax-platform 0.4.0` and `ray 2.6.3`, and + many previous versions of the two packages. Unfortunately, several changes made in recent versions of `ray` will break this script. You should install `ray < 2.7.0`. ``pip install`` is recommended by the Ax developers even if a conda environment is used. + As of now (Sep 2024), `ax-platform 0.4.1` is broken. See the `issue`_ here. + Please avoid this version in your setup. + .. note:: If you can update this example and scripts to accommodate the changes in the latest Ray package, feel free to submit a pull request. -How it works ------------- +Typical workflow +---------------- Ax is a package that can perform Bayesian optimization. With the given parameter range, a set of initial trials are generated. Then based on the metrics returned @@ -35,10 +39,12 @@ parameters and then automatically distribute the trials to available resources. With this, we can perform asynchronous parallelized hyperparameter optimization. -Ax experiments -^^^^^^^^^^^^^^ +Create an Ax experiment +^^^^^^^^^^^^^^^^^^^^^^^ + +You can create a basic Ax experiment this way -You can create a basic Ax experiment this way:: +.. code-block:: python from ax.service.ax_client import AxClient ax_client = AxClient() @@ -122,9 +128,11 @@ Training function ^^^^^^^^^^^^^^^^^ You only need a minimal change to your existing training script to use it with -Ax. In most case, you just have to wrap the whole script into a function:: +Ax. In most case, you just have to wrap the whole script into a function - def training(parameter_a, parameter_b, parameter_c, parameter_d): +.. code-block:: python + + def training(trial_index, parameter_a, parameter_b, parameter_c, parameter_d): # set up the network with the parameters ... network_params = { @@ -137,14 +145,15 @@ Ax. In most case, you just have to wrap the whole script into a function:: ) # train the network # `metric_tracker` contains the losses from HIPPYNN - metric_tracker = train_model( - training_modules, - database, - controller, - metric_tracker, - callbacks=None, - batch_callbacks=None, - ) + with hippynn.tools.active_directory(str(trial_index)): + metric_tracker = train_model( + training_modules, + database, + controller, + metric_tracker, + callbacks=None, + batch_callbacks=None, + ) # return the desired metric to Ax, for example, validation loss return { "Metric": metric_tracker.best_metric_values["valid"]["Loss"] @@ -154,16 +163,25 @@ Note how we can utilize the parameters passed in and return **Metric** at the end. Apparently, we have the freedom to choose different metrics to return here. We can even use mathematically expressions to combine some metrics together. +.. note:: + Ax does NOT create a directory for a trial. If your training function does + not take care of the working directory, all results will be saved into the + same folder, i.e., `cwd`. To avoid this, the training function need create an + unique path for each trial. In this example, we use the `trial_index` to + achieve this purpose. With Ray, this step is NOT necessary. + .. _run-sequential-experiments: Run sequential experiments ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Next, we can run the experiments:: +Next, we can run the experiments + +.. code-block:: python for k in range(30): parameter, trial_index = ax_client.get_next_trial() - ax_client.complete_trial(trial_index=trial_index, raw_data=training(parameter)) + ax_client.complete_trial(trial_index=trial_index, raw_data=training(trial_index, **parameter)) # Save experiment to file as JSON file ax_client.save_to_json_file(filepath="hyperopt.json") data_frame = ax_client.get_trials_data_frame().sort_values("Metric") @@ -180,13 +198,16 @@ Asynchronous parallelized optimization with Ray ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To use Ray to distribute the trials across GPUs parallelly, a small update is -needed for the training function:: +needed for the training function + +.. code-block:: python from ray.air import session def training(parameter_a, parameter_b, parameter_c, parameter_d): # setup and train are the same + # `with hippynn.tools.active_directory() line` is not needed .... # instead of return, we use `session.report` to communicate with `ray` session.report( @@ -199,7 +220,10 @@ Instead of a simple `return`, we need the `report` method from `ray.air.session` to report the final metric to `ray`. Also, to run the trials, instead of a loop in :ref:`run-sequential-experiments`, -we have to use the interfaces between the two packages from `ray`:: +we have to use the interfaces between the two packages from `ray` + +.. code-block:: python + from ray.tune.experiment.trial import Trial from ray.tune.search import ConcurrencyLimiter from ray.tune.search.ax import AxSearch @@ -245,7 +269,9 @@ Relative import """"""""""""""" If you save the training function into a separated file and import it into the -Ray script, one line has to be added before the trials start,:: +Ray script, one line has to be added before the trials start, + +.. code-block:: python ray.init(runtime_env={"working_dir": "."}) @@ -260,7 +286,9 @@ When running `ray.tune`, a set of callback functions can be called during the process. Ray has a `documentation`_ on the callback functions. You can build your own for your convenience. However, here is a callback function to save the json and csv files at the end of each trial and handle failed trials, which -should cover the most basic functionalities.:: +should cover the most basic functionalities. + +.. code-block:: python from ray.tune.logger import JsonLoggerCallback, LoggerCallback @@ -340,7 +368,7 @@ should cover the most basic functionalities.:: delta = trial.time_completed - trial.time_run_started return int(delta.total_seconds()) -To use callback functions, simple add a line in `ray.RunConfig`:: +To use callback functions, simple add a line in ``ray.RunConfig``:: ax_logger = AxLogger(ax_client, "hyperopt_ray.json", "hyperopt.csv") run_config=air.RunConfig( @@ -360,19 +388,25 @@ Restart/extend an experiment Restarting an experiment or adding additional trials to an experiment share the same workflow. The key is the json file saved from the experiment. To reload the -experiment state:: +experiment state: +.. code-block:: python + ax_client = AxClient.load_from_json_file(filepath="hyperopt_ray.json") Then we can pull new parameters from this experiment, and these parameters will be generated based on all finished trials. If more trials need to be added to -this experiment, simply increase `num_samples` in `ray.tune.TuneConfig`:: +this experiment, simply increase `num_samples` in `ray.tune.TuneConfig`: + +.. code-block:: python # this will end the experiment when 20 trials are finished tune_config=tune.TuneConfig(search_alg=algo, num_samples=20) Sometime, you may want to make changes to the experiment itself when reloading -the experiment, for example, the search space. This can easily achieved by:: +the experiment, for example, the search space. This can easily achieved by + +.. code-block:: python ax_client.set_search_space( [ @@ -407,8 +441,10 @@ after the `ax_client` object is reloaded. If the original experiment is not created with this option, there is not much we can do. -A full example script is provided in the examples (WIP). +The example scripts with a modified QM7 training are provided in `examples`_. .. _ray: https://docs.ray.io/en/latest/ .. _Ax: https://github.com/facebook/Ax +.. _issue: https://github.com/facebook/Ax/issues/2711 .. _documentation: https://docs.ray.io/en/latest/tune/tutorials/tune-metrics.html +.. _examples: https://github.com/lanl/hippynn/tree/development/examples/hyperparameter_optimization diff --git a/examples/hyperparameter_optimization/QM7_ax_example.py b/examples/hyperparameter_optimization/QM7_ax_example.py new file mode 100644 index 00000000..bd6f34be --- /dev/null +++ b/examples/hyperparameter_optimization/QM7_ax_example.py @@ -0,0 +1,203 @@ +""" +To obtain the data files needed for this example, use the script process_QM7_data.py, +also located in the parent folder. The script contains further instructions for use. +""" + +PERFORM_PLOTTING = True # Make sure you have matplotlib if you want to set this to TRUE + +import os + +#### Setup pytorch things +import torch + +torch.set_default_dtype(torch.float32) + +if torch.cuda.is_available(): + torch.cuda.set_device(0) # Don't try this if you want CPU training! + +import hippynn + +hippynn.settings.PROGRESS = None + + +def training(dist_soft_min, dist_soft_max, dist_hard_max): + # Log the output of python to `training_log.txt` + with hippynn.tools.log_terminal("training_log.txt", "wt"): + + # Hyperparameters for the network + + network_params = { + "possible_species": [0, 1, 6, 7, 8, 16], # Z values of the elements + "n_features": 10, # Number of neurons at each layer + "n_sensitivities": 10, # Number of sensitivity functions in an interaction layer + "dist_soft_min": dist_soft_min, # + "dist_soft_max": dist_soft_max, + "dist_hard_max": dist_hard_max, + "n_interaction_layers": 1, # Number of interaction blocks + "n_atom_layers": 1, # Number of atom layers in an interaction block + } + + # Define a model + + from hippynn.graphs import inputs, networks, physics, targets + + # model inputs + species = inputs.SpeciesNode(db_name="Z") + positions = inputs.PositionsNode(db_name="R") + + # Model computations + network = networks.Hipnn( + "HIPNN", (species, positions), module_kwargs=network_params + ) + henergy = targets.HEnergyNode("HEnergy", network) + molecule_energy = henergy.mol_energy + molecule_energy.db_name = "T" + hierarchicality = henergy.hierarchicality + + # define loss quantities + from hippynn.graphs import loss + + rmse_energy = loss.MSELoss.of_node(molecule_energy) ** (1 / 2) + mae_energy = loss.MAELoss.of_node(molecule_energy) + rsq_energy = loss.Rsq.of_node(molecule_energy) + + ### More advanced usage of loss graph + + pred_per_atom = physics.PerAtom( + "PeratomPredicted", (molecule_energy, species) + ).pred + true_per_atom = physics.PerAtom( + "PeratomTrue", (molecule_energy.true, species.true) + ) + mae_per_atom = loss.MAELoss(pred_per_atom, true_per_atom) + + ### End more advanced usage of loss graph + + loss_error = rmse_energy + mae_energy + + rbar = loss.Mean.of_node(hierarchicality) + l2_reg = loss.l2reg(network) + loss_regularization = ( + 1e-6 * l2_reg + rbar + ) # L2 regularization and hierarchicality regularization + + train_loss = loss_error + loss_regularization + + # Validation losses are what we check on the data between epochs -- we can only train to + # a single loss, but we can check other metrics too to better understand how the model is training. + # There will also be plots of these things over time when training completes. + validation_losses = { + "T-RMSE": rmse_energy, + "T-MAE": mae_energy, + "T-RSQ": rsq_energy, + "TperAtom MAE": mae_per_atom, + "T-Hier": rbar, + "L2Reg": l2_reg, + "Loss-Err": loss_error, + "Loss-Reg": loss_regularization, + "Loss": train_loss, + } + early_stopping_key = "Loss-Err" + + if PERFORM_PLOTTING: + + from hippynn import plotting + + plot_maker = plotting.PlotMaker( + # Simple plots which compare the network to the database + plotting.Hist2D.compare(molecule_energy, saved=True), + # Slightly more advanced control of plotting! + plotting.Hist2D( + true_per_atom, + pred_per_atom, + xlabel="True Energy/Atom", + ylabel="Predicted Energy/Atom", + saved="PerAtomEn.pdf", + ), + plotting.HierarchicalityPlot( + hierarchicality.pred, + molecule_energy.pred - molecule_energy.true, + saved="HierPlot.pdf", + ), + plot_every=10, # How often to make plots -- here, epoch 0, 10, 20... + ) + else: + plot_maker = None + + from hippynn.experiment import assemble_for_training + + # This piece of code glues the stuff together as a pytorch model, + # dropping things that are irrelevant for the losses defined. + training_modules, db_info = assemble_for_training( + train_loss, validation_losses, plot_maker=plot_maker + ) + training_modules[0].print_structure() + + max_batch_size = 12 + database_params = { + "name": "qm7", # Prefix for arrays in folder + # for Ax, a relative path can be used + # with Ray, an absolute path must be used + "directory": "/path/to/dataset", + "quiet": False, + "test_size": 0.1, + "valid_size": 0.1, + "seed": 2001, + # How many samples from the training set to use during evaluation + **db_info, # Adds the inputs and targets names from the model as things to load + } + + from hippynn.databases import DirectoryDatabase + + database = DirectoryDatabase(**database_params) + + # Now that we have a database and a model, we can + # Fit the non-interacting energies by examining the database. + + from hippynn.pretraining import hierarchical_energy_initialization + + hierarchical_energy_initialization(henergy, database, trainable_after=False) + + min_epochs = 50 + max_epochs = 800 + patience_epochs = 20 + + from hippynn.experiment.controllers import ( + PatienceController, + RaiseBatchSizeOnPlateau, + ) + + optimizer = torch.optim.Adam(training_modules.model.parameters(), lr=1e-3) + + scheduler = RaiseBatchSizeOnPlateau( + optimizer=optimizer, + max_batch_size=2048, + patience=5, + ) + + controller = PatienceController( + optimizer=optimizer, + scheduler=scheduler, + batch_size=512, + eval_batch_size=512, + max_epochs=10, + termination_patience=20, + fraction_train_eval=0.1, + stopping_key=early_stopping_key, + ) + + experiment_params = hippynn.experiment.SetupParams( + controller=controller, + ) + print(experiment_params) + + # Parameters describing the training procedure. + from hippynn.experiment import setup_and_train + + metric_tracker = setup_and_train( + training_modules=training_modules, + database=database, + setup_params=experiment_params, + ) + + return metric_tracker.best_metric_values diff --git a/examples/hyperparameter_optimization/ax_opt.py b/examples/hyperparameter_optimization/ax_opt.py new file mode 100644 index 00000000..c2260712 --- /dev/null +++ b/examples/hyperparameter_optimization/ax_opt.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# fmt: off +#SBATCH --time=2-00:00:00 +#SBATCH --nodes=1 +#SBATCH --ntasks=40 +#SBATCH --mail-type=all +#SBATCH -J hyperopt +#SBATCH --qos=long +#SBATCH -o run.log +import os +import sys +import warnings + +# SLURM copies the script to a tmp folder +# so to find the local package `QM7_ax_example` we need add cwd to path +# per https://stackoverflow.com/a/39574373/7066315 +sys.path.append(os.getcwd()) +# fmt: on +""" + B-Opt tuning for HIPNN using AX. + +""" + +import json + +import numpy as np +from ax.service.ax_client import AxClient +from ax.service.utils.instantiation import ObjectiveProperties +from QM7_ax_example import training + +import hippynn + +warnings.warn( + """Make sure to modify the dataset path in QM7_ax_example.py before running this example. For this test (Ax sequential optimization), a relative path can be used. The work directory for each trial will be ./test_ax/index. If the dataset is in ./dataset, the relative path should be ../../dataset. However, an absolute path is strongly recommended.""" +) + + +def evaluate(parameters: dict, trial_index: int): + """Evaluate a trial for QM7 + + Args: + parameter (dict): Python dictionary for trial values of hipnn-hyper params. + trial_index (int): Index of the current trial. A new folder will be created + based on this index to host the files associated with this trial. + + Returns: + dictionary : Loss metrics to be minimized. + """ + + # train model + with hippynn.tools.active_directory(str(trial_index)): + out = training(**parameters) + # get the loss + return {"Metric": out["valid"]["Loss"]} + + +# Initalize the Client and experiment. +restart = False +# load parameters from a json file +if len(sys.argv) == 2: + with open(sys.argv[1], "r") as param: + parameters = json.load(param) +# or directly provide them here +else: + parameters = [ + { + "name": "dist_soft_min", + "type": "range", + "value_type": "float", + "bounds": [0.5, 1.5], + }, + { + "name": "dist_soft_max", + "type": "range", + "value_type": "float", + "bounds": [3.0, 20.0], + }, + { + "name": "dist_hard_max", + "type": "range", + "value_type": "float", + "bounds": [5.0, 40.0], + }, + ] + +# create or reload the Ax experiment +if restart: + ax_client = AxClient.load_from_json_file(filepath="hyperopt.json") + # update existing experiment + # `immutable_search_space_and_opt_config` has to be False + # when the experiment was created + ax_client.set_search_space(parameters) +else: + ax_client = AxClient(verbose_logging=False) + ax_client.create_experiment( + name="QM7_opt", + parameters=parameters, + objectives={ + "Metric": ObjectiveProperties(minimize=True), + }, + overwrite_existing_experiment=True, + immutable_search_space_and_opt_config=False, + is_test=False, + parameter_constraints=[ + "dist_soft_min <= dist_soft_max", + "dist_soft_max <= dist_hard_max", + ], + ) + +if not os.path.exists("test_ax"): + os.mkdir("test_ax") +os.chdir("test_ax") +# Run the Optimization Loop. +for k in range(8): + parameter, trial_index = ax_client.get_next_trial() + ax_client.complete_trial( + trial_index=trial_index, raw_data=evaluate(parameter, trial_index) + ) + + # Save experiment to file as JSON file + ax_client.save_to_json_file(filepath="hyperopt.json") + # Save a human-friendly summary as csv file + data_frame = ax_client.get_trials_data_frame().sort_values("Metric") + data_frame.to_csv("hyperopt.csv", header=True) diff --git a/examples/hyperparameter_optimization/ax_opt_ray.py b/examples/hyperparameter_optimization/ax_opt_ray.py new file mode 100644 index 00000000..76b34944 --- /dev/null +++ b/examples/hyperparameter_optimization/ax_opt_ray.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# fmt: off +#SBATCH --time=4-00:00:00 +#SBATCH --nodes=1 +#SBATCH --mail-type=all +#SBATCH -p gpu +#SBATCH -J parallel_hyperopt +#SBATCH --qos=long +#SBATCH -o run.log +# black always format pure comments as of now +# add some codes here to keep SLURM derivatives valid +import json +import os +import sys +import warnings + +# SLURM copies the script to a tmp folder +# so to find the local package `training` we need add cwd to path +# per https://stackoverflow.com/a/39574373/7066315 +sys.path.append(os.getcwd()) +# fmt: on +""" + Hyperparameter tuning for HIPNN using AX and Ray. + +""" + +import shutil + +import numpy as np +import ray +from ax.core import Trial as AXTrial +from ax.service.ax_client import AxClient +from ax.service.utils.instantiation import ObjectiveProperties +from QM7_ax_example import training +from ray import air, tune +from ray.air import session +from ray.tune.experiment.trial import Trial +from ray.tune.logger import JsonLoggerCallback, LoggerCallback +from ray.tune.search import ConcurrencyLimiter +from ray.tune.search.ax import AxSearch + +# to make sure ray loads correct the local package +ray.init(runtime_env={"working_dir": "."}) + + +def evaluate(parameters: dict, checkpoint_dir=None): + """ + Evaluate a trial for QM7 + + Args: + parameter (dict): Python dictionary for trial values of HIPNN hyperparameters. + checkpoint_dir (str, optional): To enable checkpoints for ray. Defaults to None. + + Returns: + dict : Loss metrics to be minimized. + """ + + out = training(**parameters) + + session.report({"Metric": out["valid"]["Loss"]}) + + +class AxLogger(LoggerCallback): + def __init__(self, ax_client: AxClient, json_name: str, csv_name: str): + """ + A logger callback to save the progress to json file after every trial ends. + Similar to running `ax_client.save_to_json_file` every iteration in sequential + searches. + + Args: + ax_client (AxClient): ax client to save + json_name (str): name for the json file. Append a path if you want to save the \ + json file to somewhere other than cwd. + csv_name (str): name for the csv file. Append a path if you want to save the \ + csv file to somewhere other than cwd. + """ + self.ax_client = ax_client + self.json = json_name + self.csv = csv_name + + def log_trial_end( + self, trial: Trial, id: int, metric: float, runtime: int, failed: bool = False + ): + self.ax_client.save_to_json_file(filepath=self.json) + shutil.copy(self.json, f"{trial.local_dir}/{self.json}") + try: + data_frame = self.ax_client.get_trials_data_frame().sort_values("Metric") + data_frame.to_csv(self.csv, header=True) + except KeyError: + pass + shutil.copy(self.csv, f"{trial.local_dir}/{self.csv}") + if failed: + status = "failed" + else: + status = "finished" + print( + f"AX trial {id} {status}. Final loss: {metric}. Time taken" + f" {runtime} seconds. Location directory: {trial.local_path}." + ) + + def on_trial_error(self, iteration: int, trials: list[Trial], trial: Trial, **info): + id = int(trial.experiment_tag.split("_")[0]) - 1 + ax_trial = self.ax_client.get_trial(id) + ax_trial.mark_abandoned(reason="Error encountered") + self.log_trial_end( + trial, id + 1, "not available", self.calculate_runtime(ax_trial), True + ) + + def on_trial_complete( + self, iteration: int, trials: list["Trial"], trial: Trial, **info + ): + # trial.trial_id is the random id generated by ray, not ax + # the default experiment_tag starts with ax' trial index + # but this workaround is totally fragile, as users can + # customize the tag or folder name + id = int(trial.experiment_tag.split("_")[0]) - 1 + ax_trial = self.ax_client.get_trial(id) + failed = False + try: + loss = ax_trial.objective_mean + except ValueError: + failed = True + loss = "not available" + else: + if np.isnan(loss) or np.isinf(loss): + failed = True + loss = "not available" + if failed: + ax_trial.mark_failed() + self.log_trial_end( + trial, id + 1, loss, self.calculate_runtime(ax_trial), failed + ) + + @classmethod + def calculate_runtime(cls, trial: AXTrial): + delta = trial.time_completed - trial.time_run_started + return int(delta.total_seconds()) + + +# initialize the client and experiment. +if __name__ == "__main__": + + warnings.warn( + """Make sure to modify the dataset path in QM7_ax_example.py before running this example. For this test (Ray parallelized optimization), you MUST provide an absolute path to the dataset""" + ) + + if len(sys.argv) == 2: + with open(sys.argv[1], "r") as param: + parameters = json.load(param) + else: + parameters = [ + { + "name": "dist_soft_min", + "type": "range", + "value_type": "float", + "bounds": [0.5, 1.5], + }, + { + "name": "dist_soft_max", + "type": "range", + "value_type": "float", + "bounds": [3.0, 20.0], + }, + { + "name": "dist_hard_max", + "type": "range", + "value_type": "float", + "bounds": [5.0, 40.0], + }, + ] + # TODO: better way to handle restarting of searches + restart = False + if restart: + ax_client = AxClient.load_from_json_file(filepath="hyperopt_ray.json") + # update existing experiment + # `immutable_search_space_and_opt_config` has to be False + # when the experiment was created + ax_client.set_search_space(parameters) + else: + ax_client = AxClient( + verbose_logging=False, + enforce_sequential_optimization=False, + ) + ax_client.create_experiment( + name="QM7_ray_opt", + parameters=parameters, + objectives={ + "Metric": ObjectiveProperties(minimize=True), + }, + overwrite_existing_experiment=True, + is_test=False, + # slightly more overhead + # but make it possible to adjust the experiment setups + immutable_search_space_and_opt_config=False, + parameter_constraints=[ + "dist_soft_min <= dist_soft_max", + "dist_soft_max <= dist_hard_max", + ], + ) + + # run the optimization Loop. + algo = AxSearch(ax_client=ax_client) + algo = ConcurrencyLimiter(algo, max_concurrent=4) + ax_logger = AxLogger(ax_client, "hyperopt_ray.json", "hyperopt.csv") + tuner = tune.Tuner( + tune.with_resources(evaluate, resources={"gpu": 1}), + tune_config=tune.TuneConfig(search_alg=algo, num_samples=8), + run_config=air.RunConfig( + local_dir="test_ray", + verbose=0, + callbacks=[ax_logger, JsonLoggerCallback()], + log_to_file=True, + ), + ) + tuner.fit() diff --git a/examples/hyperparameter_optimization/prameters.json b/examples/hyperparameter_optimization/prameters.json new file mode 100644 index 00000000..c7815505 --- /dev/null +++ b/examples/hyperparameter_optimization/prameters.json @@ -0,0 +1,29 @@ +[ + { + "name": "dist_soft_min", + "type": "range", + "value_type": "float", + "bounds": [ + 0.5, + 1.5 + ] + }, + { + "name": "dist_soft_max", + "type": "range", + "value_type": "float", + "bounds": [ + 3.0, + 20.0 + ] + }, + { + "name": "dist_hard_max", + "type": "range", + "value_type": "float", + "bounds": [ + 5.0, + 40.0 + ] + } +] \ No newline at end of file