From 2d18c49cddf2ac58e7bd003e4297ce4224ae3d87 Mon Sep 17 00:00:00 2001 From: Anonymous-submission-repo Date: Wed, 12 Oct 2022 04:31:51 +0000 Subject: [PATCH] update --- flaml/automl.py | 25 ++- flaml/tune/analysis.py | 42 ----- flaml/tune/searcher/blendsearch.py | 13 ++ flaml/tune/searcher/flow2.py | 13 ++ flaml/tune/searcher/search_thread.py | 4 + flaml/tune/tune.py | 53 ++++-- notebook/tune_lexicographic.ipynb | 2 +- test/automl/test_lexiflow.py | 20 +-- test/tune/test_lexiflow.py | 10 +- .../Examples/Tune-Lexicographic-objectives.md | 165 ++++++++++++++++++ .../docs/Use-Cases/Task-Oriented-AutoML.md | 27 +++ .../Use-Cases/Tune-User-Defined-Function.md | 16 +- 12 files changed, 290 insertions(+), 100 deletions(-) create mode 100644 website/docs/Examples/Tune-Lexicographic-objectives.md diff --git a/flaml/automl.py b/flaml/automl.py index cda468217..63a2a14c8 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -2406,16 +2406,19 @@ class AutoML(BaseEstimator): skip_transform: boolean, default=False | Whether to pre-process data prior to modeling. lexico_objectives: A dictionary with four elements. - It specifics the information used for multiple objectives optimization with lexicographic preference. - e.g.,```lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"], - "tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0,"pred_time":0.0}}``` + It specifics the information used for multiple objectives optimization with lexicographic preference. + e.g., + ```python + lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"], + "tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0,"pred_time":0.0}} + ``` + Either "metrics" or "modes" is a list of str. + It represents the optimization objectives, the objective as minimization or maximization respectively. + Both "metrics" and "modes" are ordered by priorities from high to low. + "tolerances" is a dictionary to specify the optimality tolerance of each objective. + "targets" is a dictionary to specify the optimization targets for each objective. + If providing lexico_objectives, the arguments metric, hpo_method will be invalid. - Either "metrics" or "modes" is a list of str. - It represents the optimization objectives, the objective as minimization or maximization respectively. - Both "metrics" and "modes" are ordered by priorities from high to low. - "tolerances" is a dictionary to specify the optimality tolerance of each objective. - "targets" is a dictionary to specify the optimization targets for each objective. - If providing lexico_objectives, the arguments metric, hpo_method will be invalid. fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name. For TransformersEstimator, available fit_kwargs can be found from @@ -2520,6 +2523,10 @@ class AutoML(BaseEstimator): if lexico_objectives is None: hpo_method = hpo_method or self._settings.get("hpo_method") else: + if hpo_method != "cfo": + logger.warning( + "If lexico_objectives is not None, hpo_method is forced to be cfo" + ) hpo_method = "cfo" learner_selector = learner_selector or self._settings.get("learner_selector") diff --git a/flaml/tune/analysis.py b/flaml/tune/analysis.py index 4aedfb543..af20ad748 100644 --- a/flaml/tune/analysis.py +++ b/flaml/tune/analysis.py @@ -17,11 +17,8 @@ # Copyright (c) Microsoft Corporation. from typing import Dict, Optional import numpy as np - -from flaml.tune import result from .trial import Trial from collections import defaultdict - import logging logger = logging.getLogger(__name__) @@ -91,42 +88,6 @@ class ExperimentAnalysis: raise ValueError("If set, `mode` has to be one of [min, max]") return mode or self.default_mode - def lexico_best(self, trials): - results = {index: trial.last_result for index, trial in enumerate(trials)} - metrics = self.lexico_objectives["metrics"] - modes = self.lexico_objectives["modes"] - f_best = {} - keys = list(results.keys()) - length = len(keys) - histories = defaultdict(list) - for time_index in range(length): - for objective, mode in zip(metrics, modes): - histories[objective].append( - results[keys[time_index]][objective] - if mode == "min" - else trials[keys[time_index]][objective] * -1 - ) - obj_initial = self.lexico_objectives["metrics"][0] - feasible_index = [*range(len(histories[obj_initial]))] - for k_metric in self.lexico_objectives["metrics"]: - k_values = np.array(histories[k_metric]) - f_best[k_metric] = np.min(k_values.take(feasible_index)) - feasible_index_prior = np.where( - k_values - <= max( - [ - f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric], - self.lexico_objectives["targets"][k_metric], - ] - ) - )[0].tolist() - feasible_index = [ - val for val in feasible_index if val in feasible_index_prior - ] - best_trial = trials[feasible_index[-1]] - return best_trial - def get_best_trial( self, metric: Optional[str] = None, @@ -158,9 +119,6 @@ class ExperimentAnalysis: values are disregarded and these trials are never selected as the best trial. """ - if self.lexico_objectives is not None: - best_trial = self.lexico_best(self.trials) - return best_trial metric = self._validate_metric(metric) mode = self._validate_mode(mode) if scope not in ["all", "last", "avg", "last-5-avg", "last-10-avg"]: diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py index 841fa5ff5..244ac0a8e 100644 --- a/flaml/tune/searcher/blendsearch.py +++ b/flaml/tune/searcher/blendsearch.py @@ -113,6 +113,19 @@ class BlendSearch(Searcher): Default is "auto", which means that we will automatically chose the cost attribute to use (depending on the nature of the resource budget). When cost_attr is set to None, cost differences between different trials will be omitted in our search algorithm. + lexico_objectives: A dictionary with four elements. + It specifics the information used for multiple objectives optimization with lexicographic preference. + e.g., + ```python + lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"], + "tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0,"pred_time":0.0}} + ``` + Either "metrics" or "modes" is a list of str. + It represents the optimization objectives, the objective as minimization or maximization respectively. + Both "metrics" and "modes" are ordered by priorities from high to low. + "tolerances" is a dictionary to specify the optimality tolerance of each objective. + "targets" is a dictionary to specify the optimization targets for each objective. + If providing lexico_objectives, the arguments metric, mode will be invalid. experimental: A bool of whether to use experimental features. """ self._eps = SEARCH_THREAD_EPS diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py index 6d774b543..5134f3b49 100644 --- a/flaml/tune/searcher/flow2.py +++ b/flaml/tune/searcher/flow2.py @@ -70,6 +70,19 @@ class FLOW2(Searcher): resource_multiple_factor: A float of the multiplicative factor used for increasing resource. cost_attr: A string of the attribute used for cost. + lexico_objectives: A dictionary with four elements. + It specifics the information used for multiple objectives optimization with lexicographic preference. + e.g., + ```python + lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"], + "tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0,"pred_time":0.0}} + ``` + Either "metrics" or "modes" is a list of str. + It represents the optimization objectives, the objective as minimization or maximization respectively. + Both "metrics" and "modes" are ordered by priorities from high to low. + "tolerances" is a dictionary to specify the optimality tolerance of each objective. + "targets" is a dictionary to specify the optimization targets for each objective. + If providing lexico_objectives, the arguments metric, mode will be invalid. seed: An integer of the random seed. """ if mode: diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py index 9866abed9..6e2cc0986 100644 --- a/flaml/tune/searcher/search_thread.py +++ b/flaml/tune/searcher/search_thread.py @@ -141,6 +141,8 @@ class SearchThread: not hasattr(self._search_alg, "lexico_objectives") or self._search_alg.lexico_objectives is None ): + # TODO: Improve this behavior. When lexico_objectives is provided to CFO, + # related variables are not callable. obj = result[self._search_alg.metric] * self._metric_op if obj < self.obj_best1 or self.best_result is None: self.cost_best2 = self.cost_best1 @@ -153,6 +155,8 @@ class SearchThread: not hasattr(self._search_alg, "lexico_objectives") or self._search_alg.lexico_objectives is None ): + # TODO: Improve this behavior. When lexico_objectives is provided to CFO, + # related variables are not callable. self._update_speed() self.running -= 1 assert self.running >= 0 diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py index c6f333e80..fc06d2f3e 100644 --- a/flaml/tune/tune.py +++ b/flaml/tune/tune.py @@ -2,7 +2,7 @@ # * Copyright (c) FLAML authors. All rights reserved. # * Licensed under the MIT License. See LICENSE file in the # * project root for license information. -from typing import Optional, Union, List, Callable, Tuple +from typing import Optional, Union, List, Callable, Tuple, Dict import numpy as np import datetime import time @@ -34,22 +34,32 @@ _training_iteration = 0 INCUMBENT_RESULT = "__incumbent_result__" -def is_nan_or_inf(value): - return np.isnan(value) or np.isinf(value) - - class ExperimentAnalysis(EA): """Class for storing the experiment results.""" - def __init__(self, trials, metric, mode, lexico_objectives): + def __init__(self, trials, metric, mode, lexico_objectives=None): try: super().__init__(self, None, trials, metric, mode) except (TypeError, ValueError): self.trials = trials self.default_metric = metric or DEFAULT_METRIC - self.default_mode = mode or DEFAULT_MODE + self.default_mode = mode self.lexico_objectives = lexico_objectives + @property + def best_trial(self) -> Trial: + if self.lexico_objectives is None: + return super().best_trial + else: + return self.get_best_trial(self.default_metric, self.default_mode) + + @property + def best_config(self) -> Dict: + if self.lexico_objectives is None: + return super().best_config + else: + return self.get_best_config(self.default_metric, self.default_mode) + def lexico_best(self, trials): results = {index: trial.last_result for index, trial in enumerate(trials)} metrics = self.lexico_objectives["metrics"] @@ -99,6 +109,13 @@ class ExperimentAnalysis(EA): best_trial = super().get_best_trial(metric, mode, scope, filter_nan_and_inf) return best_trial + @property + def best_result(self) -> Dict: + if self.lexico_best is None: + return super().best_result + else: + return self.best_trial.last_result + def report(_metric=None, **kwargs): @@ -357,16 +374,18 @@ def run( a trial before the tuning is terminated. use_ray: A boolean of whether to use ray as the backend. lexico_objectives: A dictionary with four elements. - It specifics the information used for multiple objectives optimization with lexicographic preference. - e.g.,```lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"], - "tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0,"pred_time":0.0}}``` - - Either "metrics" or "modes" is a list of str. - It represents the optimization objectives, the objective as minimization or maximization respectively. - Both "metrics" and "modes" are ordered by priorities from high to low. - "tolerances" is a dictionary to specify the optimality tolerance of each objective. - "targets" is a dictionary to specify the optimization targets for each objective. - If providing lexico_objectives, the arguments metric, mode, and search_alg will be invalid. + It specifics the information used for multiple objectives optimization with lexicographic preference. + e.g., + ```python + lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"], + "tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0,"pred_time":0.0}} + ``` + Either "metrics" or "modes" is a list of str. + It represents the optimization objectives, the objective as minimization or maximization respectively. + Both "metrics" and "modes" are ordered by priorities from high to low. + "tolerances" is a dictionary to specify the optimality tolerance of each objective. + "targets" is a dictionary to specify the optimization targets for each objective. + If providing lexico_objectives, the arguments metric, mode, and search_alg will be invalid. log_file_name: A string of the log file name. Default to None. When set to None: diff --git a/notebook/tune_lexicographic.ipynb b/notebook/tune_lexicographic.ipynb index 0490f89a0..7208d0c44 100644 --- a/notebook/tune_lexicographic.ipynb +++ b/notebook/tune_lexicographic.ipynb @@ -243,7 +243,7 @@ "\n", "analysis = tune.run(\n", " evaluate_function,\n", - " num_samples=100000000,\n", + " num_samples=-1,\n", " time_budget_s=100,\n", " config=search_space,\n", " use_ray=False,\n", diff --git a/test/automl/test_lexiflow.py b/test/automl/test_lexiflow.py index 97da819e4..c6d5afb56 100644 --- a/test/automl/test_lexiflow.py +++ b/test/automl/test_lexiflow.py @@ -2,7 +2,7 @@ from flaml import AutoML from flaml.data import load_openml_dataset -def _test_lexiflow(): +def test_lexiflow(): X_train, X_test, y_train, y_test = load_openml_dataset( dataset_id=179, data_dir="test/data" @@ -13,28 +13,18 @@ def _test_lexiflow(): lexico_objectives["tolerances"] = {"val_loss": 0.01, "pred_time": 0.0} lexico_objectives["targets"] = {"val_loss": 0.0, "pred_time": 0.0} lexico_objectives["modes"] = ["min", "min"] - automl = AutoML() + settings = { "time_budget": 100, "lexico_objectives": lexico_objectives, - "estimator_list": ["xgboost"], - "use_ray": True, + "use_ray": False, "task": "classification", - "max_iter": 10000000, - "train_time_limit": 60, - "verbose": 0, - "eval_method": "holdout", + "max_iter": -1, "mem_thres": 128 * (1024**3), - "seed": 1, } automl.fit(X_train=X_train, y_train=y_train, X_val=X_test, y_val=y_test, **settings) - print(automl.predict(X_train)) - print(automl.model) - print(automl.config_history) - print(automl.best_iteration) - print(automl.best_estimator) if __name__ == "__main__": - _test_lexiflow() + test_lexiflow() diff --git a/test/tune/test_lexiflow.py b/test/tune/test_lexiflow.py index 49422bd9d..056750528 100644 --- a/test/tune/test_lexiflow.py +++ b/test/tune/test_lexiflow.py @@ -12,7 +12,7 @@ N_TRAIN_EXAMPLES = BATCHSIZE * 30 N_VALID_EXAMPLES = BATCHSIZE * 10 -def _test_lexiflow(): +def test_lexiflow(): train_dataset = torchvision.datasets.FashionMNIST( "test/data", train=True, @@ -109,18 +109,16 @@ def _test_lexiflow(): "n_epoch": 1, } - analysis = tune.run( + tune.run( evaluate_function, - num_samples=100000000, + num_samples=-1, time_budget_s=100, config=search_space, use_ray=False, lexico_objectives=lexico_objectives, low_cost_partial_config=low_cost_partial_config, ) - result = analysis.best_result - print(result) if __name__ == "__main__": - _test_lexiflow() + test_lexiflow() diff --git a/website/docs/Examples/Tune-Lexicographic-objectives.md b/website/docs/Examples/Tune-Lexicographic-objectives.md new file mode 100644 index 000000000..eef68088a --- /dev/null +++ b/website/docs/Examples/Tune-Lexicographic-objectives.md @@ -0,0 +1,165 @@ +# Tune - Lexicographic Objectives + +## Requirements + +```python +pip install thop torchvision torch +``` + +## Tuning accurate and efficient neural networks with lexicographic preference + +### Data + +```python +import torch +import thop +import torch.nn as nn +from flaml import tune +import torch.nn.functional as F +import torchvision +import numpy as np +import os + +DEVICE = torch.device("cpu") +BATCHSIZE = 128 +N_TRAIN_EXAMPLES = BATCHSIZE * 30 +N_VALID_EXAMPLES = BATCHSIZE * 10 +data_dir = os.path.abspath("data") + +train_dataset = torchvision.datasets.FashionMNIST( + data_dir, + train=True, + download=True, + transform=torchvision.transforms.ToTensor(), +) + +train_loader = torch.utils.data.DataLoader( + torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))), + batch_size=BATCHSIZE, + shuffle=True, +) + +val_dataset = torchvision.datasets.FashionMNIST( + data_dir, train=False, transform=torchvision.transforms.ToTensor() +) + +val_loader = torch.utils.data.DataLoader( + torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))), + batch_size=BATCHSIZE, + shuffle=True, +``` + +### Specific the model + +```python +def define_model(configuration): + n_layers = configuration["n_layers"] + layers = [] + in_features = 28 * 28 + for i in range(n_layers): + out_features = configuration["n_units_l{}".format(i)] + layers.append(nn.Linear(in_features, out_features)) + layers.append(nn.ReLU()) + p = configuration["dropout_{}".format(i)] + layers.append(nn.Dropout(p)) + in_features = out_features + layers.append(nn.Linear(in_features, 10)) + layers.append(nn.LogSoftmax(dim=1)) + return nn.Sequential(*layers) +``` + +### Train + +```python +def train_model(model, optimizer, train_loader): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE) + optimizer.zero_grad() + F.nll_loss(model(data), target).backward() + optimizer.step() +``` + +### Metrics + +```python +def eval_model(model, valid_loader): + model.eval() + correct = 0 + with torch.no_grad(): + for batch_idx, (data, target) in enumerate(valid_loader): + data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE) + pred = model(data).argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + + accuracy = correct / N_VALID_EXAMPLES + flops, params = thop.profile( + model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False + ) + return np.log2(flops), 1 - accuracy, params +``` + + + +### Evaluation function + +```python +def evaluate_function(configuration): + model = define_model(configuration).to(DEVICE) + optimizer = torch.optim.Adam(model.parameters(), configuration["lr"]) + n_epoch = configuration["n_epoch"] + for epoch in range(n_epoch): + train_model(model, optimizer, train_loader) + flops, error_rate, params = eval_model(model, val_loader) + return {"error_rate": error_rate, "flops": flops, "params": params} +``` + +### Search space +```python +search_space = { + "n_layers": tune.randint(lower=1, upper=3), + "n_units_l0": tune.randint(lower=4, upper=128), + "n_units_l1": tune.randint(lower=4, upper=128), + "n_units_l2": tune.randint(lower=4, upper=128), + "dropout_0": tune.uniform(lower=0.2, upper=0.5), + "dropout_1": tune.uniform(lower=0.2, upper=0.5), + "dropout_2": tune.uniform(lower=0.2, upper=0.5), + "lr": tune.loguniform(lower=1e-5, upper=1e-1), + "n_epoch": tune.randint(lower=1, upper=20), +} +``` + +### Launch the tuning process + +```python + +# Low cost initial point +low_cost_partial_config = { + "n_layers": 1, + "n_units_l0": 4, + "n_units_l1": 4, + "n_units_l2": 4, + "n_epoch": 1, +} + +# Specific lexicographic preference +lexico_objectives = {} +lexico_objectives["metrics"] = ["error_rate", "flops"] +lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0} +lexico_objectives["targets"] = {"error_rate": 0.0, "flops": 0.0} +lexico_objectives["modes"] = ["min", "min"] + +# launch the tuning process +analysis = tune.run( + evaluate_function, + num_samples=-1, + time_budget_s=100, + config=search_space, # search space of NN + use_ray=False, + lexico_objectives=lexico_objectives, + low_cost_partial_config=low_cost_partial_config, # low cost initial point +) +``` + + +[Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/tune_lexicographic.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/tune_lexicographic.ipynb) \ No newline at end of file diff --git a/website/docs/Use-Cases/Task-Oriented-AutoML.md b/website/docs/Use-Cases/Task-Oriented-AutoML.md index de7c11820..dfce055d3 100644 --- a/website/docs/Use-Cases/Task-Oriented-AutoML.md +++ b/website/docs/Use-Cases/Task-Oriented-AutoML.md @@ -422,6 +422,33 @@ automl2.fit(X_train, y_train, time_budget=7200, starting_points=automl1.best_con `starting_points` is a dictionary or a str to specify the starting hyperparameter config. (1) When it is a dictionary, the keys are the estimator names. If you do not need to specify starting points for an estimator, exclude its name from the dictionary. The value for each key can be either a dictionary of a list of dictionaries, corresponding to one hyperparameter configuration, or multiple hyperparameter configurations, respectively. (2) When it is a str: if "data", use data-dependent defaults; if "data:path", use data-dependent defaults which are stored at path; if "static", use data-independent defaults. Please find more details about data-dependent defaults in [zero shot AutoML](Zero-Shot-AutoML#combine-zero-shot-automl-and-hyperparameter-tuning). +### Lexicographic objectives +We support automl for multiple objectives with lexicographic preference by providing argument `lexico_objectives` for `automl.fit()`. +`lexico_objectives` is a dictionary with four mandatory elements: + - `metrics`: A list of optimization objectives. The objectives are ordered by their priority from high to low. + - `modes`: A list to specify each objective as minimization or maximization in `metrics` correspondingly. + - `tolerances`: A dictionary to specify the "tolerance" for each objective. "tolerance" is the amount of performance degradation the user is willing to compromise in order to find choices with better performance on the objectives of lower priorities. + - `targets`: A dictionary to specify the "goals" for each objective. When the objective is better than or equal to the "goals", further minimization is no longer needed. + +In the following example, we want to minimize `val_loss` and `pred_time` of the model where `val_loss` has high priority. The tolerances for `val_loss` and `pre_time` are 0.02 and 0 respectively. We do not set targets for these two objectives and we set them to -inf for both objectives. + +```python +lexico_objectives = {} +lexico_objectives["metrics"] = ["val_loss","pred_time"] +lexico_objectives["pred_time"] = ["min","min"] +lexico_objectives["tolerances"] = {"val_loss": 0.02, "pred_time":0.0} +lexico_objectives["targets"] = {"val_loss": -float('inf'), "pred_time": -float('inf')} + +# provide the lexico_objectives to automl.fit +automl.fit(..., lexico_objectives=lexico_objectives, ...) + +``` +*Please note that this is a new feature in version 1.1.0 and subject to change in the future version* + + + + + ### Log the trials diff --git a/website/docs/Use-Cases/Tune-User-Defined-Function.md b/website/docs/Use-Cases/Tune-User-Defined-Function.md index 5271cfb29..5ded45b93 100644 --- a/website/docs/Use-Cases/Tune-User-Defined-Function.md +++ b/website/docs/Use-Cases/Tune-User-Defined-Function.md @@ -516,14 +516,14 @@ analysis = tune.run( ``` ### Lexicographic Objectives -We support tuning multiple objectives with lexicographic preference by providing argument `lexico_objectives` for `tune.tun()` and `automl.fit()`. -`lexico_objectives` is a dictionary with four elements: +We support tuning multiple objectives with lexicographic preference by providing argument `lexico_objectives` for `tune.tun()`. +`lexico_objectives` is a dictionary with four mandatory elements: - `metrics`: A list of optimization objectives. The objectives are ordered by their priority from high to low. - - `modes`: A list to specify each objective as minimization or maximization in `metrics` correspondingly. + - `modes`: A list to specify each objective as minimization or maximization in `metrics` correspondingly. - `tolerances`: A dictionary to specify the "tolerance" for each objective. "tolerance" is the amount of performance degradation the user is willing to compromise in order to find choices with better performance on the objectives of lower priorities. - - `targets`: A dictionary to specify the "goals" for each objective. When the objective is smaller than or equal to the "goals", further minimization is no longer needed. + - `targets`: A dictionary to specify the "goals" for each objective. When the objective is better than or equal to the "goals", further minimization is no longer needed. -In the following example, we want to minimize `val_loss` and `pred_time` of the model where `val_loss` has high priority. The tolerances for `val_loss` and `pre_time` are 0.02 and 0 respectively. We do not set targets for these two objectives and we set them to -inf for both objectives. +In the following example, we want to minimize `val_loss` and `pred_time` of the model where `val_loss` has high priority. The tolerances for `val_loss` and `pre_time` are 0.02 and 0 respectively. We do not set targets for these two objectives and we set them to -inf for both objectives. ```python lexico_objectives = {} @@ -534,12 +534,8 @@ lexico_objectives["targets"] = {"val_loss": -float('inf'), "pred_time": -float(' # provide the lexico_objectives to tune.run tune.run(..., lexico_objectives=lexico_objectives, ...) - -# provide the lexico_objectives to automl.fit -automl.fit(..., lexico_objectives=lexico_objectives, ...) - ``` - +*Please note that this is a new feature in version 1.1.0 and subject to change in the future version*