mirror of
https://github.com/microsoft/autogen.git
synced 2025-11-02 02:40:21 +00:00
clean up
This commit is contained in:
Anonymous-submission-repo
parent
44883f7463
commit
2daaa4c637
@ -3690,4 +3690,3 @@ class AutoML(BaseEstimator):
|
||||
q += inv[i] / s
|
||||
if p < q:
|
||||
return estimator_list[i]
|
||||
|
||||
@ -113,23 +113,23 @@ class BlendSearch(Searcher):
|
||||
Default is "auto", which means that we will automatically chose the cost attribute to use (depending
|
||||
on the nature of the resource budget). When cost_attr is set to None, cost differences between different trials will be omitted
|
||||
in our search algorithm.
|
||||
lexico_objectives: dict, default=None | It specifics information needed to perform multi-objective
|
||||
optimization with lexicographic preferences. This is only supported in CFO currently.
|
||||
When lexico_objectives is not None, the arguments metric, mode will be invalid.
|
||||
This dictionary shall contain the following fields of key-value pairs:
|
||||
- "metrics": a list of optimization objectives with the orders reflecting the priorities/preferences of the
|
||||
lexico_objectives: dict, default=None | It specifics information needed to perform multi-objective
|
||||
optimization with lexicographic preferences. This is only supported in CFO currently.
|
||||
When lexico_objectives is not None, the arguments metric, mode will be invalid.
|
||||
This dictionary shall contain the following fields of key-value pairs:
|
||||
- "metrics": a list of optimization objectives with the orders reflecting the priorities/preferences of the
|
||||
objectives.
|
||||
- "modes" (optional): a list of optimization modes (each mode either "min" or "max") corresponding to the
|
||||
- "modes" (optional): a list of optimization modes (each mode either "min" or "max") corresponding to the
|
||||
objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives.
|
||||
- "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the
|
||||
metric names (provided in "metric"), and the values are the numerical target values.
|
||||
- "tolerances"(optional): a dictionary to specify the optimality tolerances on objectives. The keys are the
|
||||
metric names (provided in "metrics"), and the values are the numerical tolerances values.
|
||||
- "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the
|
||||
metric names (provided in "metric"), and the values are the numerical target values.
|
||||
- "tolerances"(optional): a dictionary to specify the optimality tolerances on objectives. The keys are the
|
||||
metric names (provided in "metrics"), and the values are the numerical tolerances values.
|
||||
E.g.,
|
||||
```python
|
||||
lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"],
|
||||
"tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0}}
|
||||
```
|
||||
```
|
||||
experimental: A bool of whether to use experimental features.
|
||||
"""
|
||||
self._eps = SEARCH_THREAD_EPS
|
||||
|
||||
@ -70,22 +70,22 @@ class FLOW2(Searcher):
|
||||
resource_multiple_factor: A float of the multiplicative factor
|
||||
used for increasing resource.
|
||||
cost_attr: A string of the attribute used for cost.
|
||||
lexico_objectives: dict, default=None | It specifics information needed to perform multi-objective
|
||||
optimization with lexicographic preferences. When lexico_objectives is not None, the arguments metric,
|
||||
mode will be invalid. This dictionary shall contain the following fields of key-value pairs:
|
||||
- "metrics": a list of optimization objectives with the orders reflecting the priorities/preferences of the
|
||||
lexico_objectives: dict, default=None | It specifics information needed to perform multi-objective
|
||||
optimization with lexicographic preferences. When lexico_objectives is not None, the arguments metric,
|
||||
mode will be invalid. This dictionary shall contain the following fields of key-value pairs:
|
||||
- "metrics": a list of optimization objectives with the orders reflecting the priorities/preferences of the
|
||||
objectives.
|
||||
- "modes" (optional): a list of optimization modes (each mode either "min" or "max") corresponding to the
|
||||
- "modes" (optional): a list of optimization modes (each mode either "min" or "max") corresponding to the
|
||||
objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives
|
||||
- "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the
|
||||
metric names (provided in "metric"), and the values are the numerical target values.
|
||||
- "tolerances"(optional): a dictionary to specify the optimality tolerances on objectives. The keys are the
|
||||
metric names (provided in "metrics"), and the values are the numerical tolerances values.
|
||||
- "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the
|
||||
metric names (provided in "metric"), and the values are the numerical target values.
|
||||
- "tolerances"(optional): a dictionary to specify the optimality tolerances on objectives. The keys are the
|
||||
metric names (provided in "metrics"), and the values are the numerical tolerances values.
|
||||
E.g.,
|
||||
```python
|
||||
lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"],
|
||||
"tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0}}
|
||||
```
|
||||
```
|
||||
seed: An integer of the random seed.
|
||||
"""
|
||||
if mode:
|
||||
@ -110,13 +110,19 @@ class FLOW2(Searcher):
|
||||
self.min_resource = min_resource
|
||||
self.lexico_objectives = lexico_objectives
|
||||
if self.lexico_objectives is not None:
|
||||
if "modes" not in self.lexico_objectives.keys():
|
||||
self.lexico_objectives["modes"] = ["min"]*len(self.lexico_objectives["metrics"])
|
||||
for t_metric, t_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
|
||||
if "modes" not in self.lexico_objectives.keys():
|
||||
self.lexico_objectives["modes"] = ["min"] * len(
|
||||
self.lexico_objectives["metrics"]
|
||||
)
|
||||
for t_metric, t_mode in zip(
|
||||
self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
|
||||
):
|
||||
if t_metric not in self.lexico_objectives["tolerances"].keys():
|
||||
self.lexico_objectives["tolerances"][t_metric] = 0
|
||||
if t_metric not in self.lexico_objectives["targets"].keys():
|
||||
self.lexico_objectives["targets"][t_metric] = -float("inf") if t_mode == "min" else float("inf")
|
||||
self.lexico_objectives["targets"][t_metric] = (
|
||||
-float("inf") if t_mode == "min" else float("inf")
|
||||
)
|
||||
self.resource_multiple_factor = (
|
||||
resource_multiple_factor or SAMPLE_MULTIPLY_FACTOR
|
||||
)
|
||||
@ -125,7 +131,7 @@ class FLOW2(Searcher):
|
||||
self._resource = None
|
||||
self._f_best = None # only use for lexico_comapre. It represent the best value achieved by lexico_flow.
|
||||
self._step_lb = np.Inf
|
||||
self._histories = None # only use for lexico_comapre. It records the result of historical configurations.
|
||||
self._histories = None # only use for lexico_comapre. It records the result of historical configurations.
|
||||
if space is not None:
|
||||
self._init_search()
|
||||
|
||||
@ -299,7 +305,7 @@ class FLOW2(Searcher):
|
||||
flow2.best_obj = {}
|
||||
for k, v in obj.items():
|
||||
flow2.best_obj[k] = (
|
||||
v * -1
|
||||
-v
|
||||
if self.lexico_objectives["modes"][
|
||||
self.lexico_objectives["metrics"].index(k)
|
||||
]
|
||||
@ -345,7 +351,9 @@ class FLOW2(Searcher):
|
||||
self._init_search()
|
||||
return True
|
||||
|
||||
def update_fbest(self,):
|
||||
def update_fbest(
|
||||
self,
|
||||
):
|
||||
obj_initial = self.lexico_objectives["metrics"][0]
|
||||
feasible_index = [*range(len(self._histories[obj_initial]))]
|
||||
for k_metric in self.lexico_objectives["metrics"]:
|
||||
@ -376,10 +384,32 @@ class FLOW2(Searcher):
|
||||
for k in self.lexico_objectives["metrics"]:
|
||||
self._histories[k].append(result[k])
|
||||
self.update_fbest()
|
||||
for k_metric, k_mode in zip(self.lexico_objectives["metrics"],self.lexico_objectives["modes"]):
|
||||
k_target = self.lexico_objectives["targets"][k_metric] if k_mode == "min" else -1*self.lexico_objectives["targets"][k_metric]
|
||||
if (result[k_metric] < max([self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric], k_target])) and (
|
||||
self.best_obj[k_metric] < max([self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric], k_target])
|
||||
for k_metric, k_mode in zip(
|
||||
self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
|
||||
):
|
||||
k_target = (
|
||||
self.lexico_objectives["targets"][k_metric]
|
||||
if k_mode == "min"
|
||||
else -self.lexico_objectives["targets"][k_metric]
|
||||
)
|
||||
if (
|
||||
result[k_metric]
|
||||
< max(
|
||||
[
|
||||
self._f_best[k_metric]
|
||||
+ self.lexico_objectives["tolerances"][k_metric],
|
||||
k_target,
|
||||
]
|
||||
)
|
||||
) and (
|
||||
self.best_obj[k_metric]
|
||||
< max(
|
||||
[
|
||||
self._f_best[k_metric]
|
||||
+ self.lexico_objectives["tolerances"][k_metric],
|
||||
k_target,
|
||||
]
|
||||
)
|
||||
):
|
||||
continue
|
||||
elif result[k_metric] < self.best_obj[k_metric]:
|
||||
|
||||
@ -150,9 +150,7 @@ class SearchThread:
|
||||
self.obj_best1 = obj
|
||||
self.cost_best = self.cost_last
|
||||
self.best_result = result
|
||||
if (
|
||||
getattr(self._search_alg, "lexico_objectives", None) is None
|
||||
):
|
||||
if getattr(self._search_alg, "lexico_objectives", None) is None:
|
||||
# TODO: Improve this behavior. When lexico_objectives is provided to CFO,
|
||||
# related variables are not callable.
|
||||
self._update_speed()
|
||||
|
||||
@ -74,13 +74,19 @@ class ExperimentAnalysis(EA):
|
||||
histories[objective].append(
|
||||
results[keys[time_index]][objective]
|
||||
if mode == "min"
|
||||
else trials[keys[time_index]][objective] * -1
|
||||
else -trials[keys[time_index]][objective]
|
||||
)
|
||||
obj_initial = self.lexico_objectives["metrics"][0]
|
||||
feasible_index = [*range(len(histories[obj_initial]))]
|
||||
for k_metric, k_mode in zip(self.lexico_objectives["metrics"],self.lexico_objectives["modes"]):
|
||||
for k_metric, k_mode in zip(
|
||||
self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
|
||||
):
|
||||
k_values = np.array(histories[k_metric])
|
||||
k_target = self.lexico_objectives["targets"][k_metric] * -1 if k_mode == "max" else self.lexico_objectives["targets"][k_metric]
|
||||
k_target = (
|
||||
-self.lexico_objectives["targets"][k_metric]
|
||||
if k_mode == "max"
|
||||
else self.lexico_objectives["targets"][k_metric]
|
||||
)
|
||||
f_best[k_metric] = np.min(k_values.take(feasible_index))
|
||||
feasible_index_prior = np.where(
|
||||
k_values
|
||||
@ -375,24 +381,24 @@ def run(
|
||||
max_failure: int | the maximal consecutive number of failures to sample
|
||||
a trial before the tuning is terminated.
|
||||
use_ray: A boolean of whether to use ray as the backend.
|
||||
lexico_objectives: dict, default=None | It specifics information needed to perform multi-objective
|
||||
optimization with lexicographic preferences. When lexico_objectives is not None, the arguments metric,
|
||||
lexico_objectives: dict, default=None | It specifics information needed to perform multi-objective
|
||||
optimization with lexicographic preferences. When lexico_objectives is not None, the arguments metric,
|
||||
mode, will be invalid, and flaml's tune uses CFO
|
||||
as the `search_alg`, which makes the input (if provided) `search_alg' invalid.
|
||||
This dictionary shall contain the following fields of key-value pairs:
|
||||
- "metrics": a list of optimization objectives with the orders reflecting the priorities/preferences of the
|
||||
This dictionary shall contain the following fields of key-value pairs:
|
||||
- "metrics": a list of optimization objectives with the orders reflecting the priorities/preferences of the
|
||||
objectives.
|
||||
- "modes" (optional): a list of optimization modes (each mode either "min" or "max") corresponding to the
|
||||
- "modes" (optional): a list of optimization modes (each mode either "min" or "max") corresponding to the
|
||||
objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives.
|
||||
- "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the
|
||||
metric names (provided in "metric"), and the values are the numerical target values.
|
||||
- "tolerances"(optional): a dictionary to specify the optimality tolerances on objectives. The keys are the
|
||||
metric names (provided in "metrics"), and the values are the numerical tolerances values.
|
||||
- "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the
|
||||
metric names (provided in "metric"), and the values are the numerical target values.
|
||||
- "tolerances"(optional): a dictionary to specify the optimality tolerances on objectives. The keys are the
|
||||
metric names (provided in "metrics"), and the values are the numerical tolerances values.
|
||||
E.g.,
|
||||
```python
|
||||
lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"],
|
||||
"tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0}}
|
||||
```
|
||||
```
|
||||
log_file_name: A string of the log file name. Default to None.
|
||||
When set to None:
|
||||
if local_dir is not given, no log file is created;
|
||||
@ -449,8 +455,11 @@ def run(
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
|
||||
from .searcher.blendsearch import BlendSearch, CFO
|
||||
if lexico_objectives != None:
|
||||
logger.warning("If lexico_objectives is not None, search_alg is forced to be CFO")
|
||||
|
||||
if lexico_objectives is not None:
|
||||
logger.warning(
|
||||
"If lexico_objectives is not None, search_alg is forced to be CFO"
|
||||
)
|
||||
search_alg = None
|
||||
if search_alg is None:
|
||||
flaml_scheduler_resource_attr = (
|
||||
@ -469,9 +478,12 @@ def run(
|
||||
if lexico_objectives is None:
|
||||
try:
|
||||
import optuna as _
|
||||
SearchAlgorithm = BlendSearch
|
||||
|
||||
SearchAlgorithm = BlendSearch
|
||||
logger.info(
|
||||
"Using search algorithm {}.".format(SearchAlgorithm.__class__.__name__)
|
||||
"Using search algorithm {}.".format(
|
||||
SearchAlgorithm.__class__.__name__
|
||||
)
|
||||
)
|
||||
except ImportError:
|
||||
SearchAlgorithm = CFO
|
||||
|
||||
@ -109,7 +109,7 @@ def test_lexiflow():
|
||||
"n_epoch": 1,
|
||||
}
|
||||
|
||||
# lexico tune
|
||||
# lexico tune
|
||||
analysis = tune.run(
|
||||
evaluate_function,
|
||||
num_samples=5,
|
||||
@ -122,7 +122,7 @@ def test_lexiflow():
|
||||
print(analysis.best_config)
|
||||
print(analysis.best_result)
|
||||
|
||||
# Non lexico tune
|
||||
# Non lexico tune
|
||||
analysis = tune.run(
|
||||
evaluate_function,
|
||||
metric="error_rate",
|
||||
|
||||
@ -103,7 +103,7 @@ def eval_model(model, valid_loader):
|
||||
|
||||
### Evaluation function
|
||||
|
||||
```python
|
||||
```python
|
||||
def evaluate_function(configuration):
|
||||
model = define_model(configuration).to(DEVICE)
|
||||
optimizer = torch.optim.Adam(model.parameters(), configuration["lr"])
|
||||
@ -156,7 +156,7 @@ analysis = tune.run(
|
||||
time_budget_s=100,
|
||||
config=search_space, # search space of NN
|
||||
use_ray=False,
|
||||
lexico_objectives=lexico_objectives,
|
||||
lexico_objectives=lexico_objectives,
|
||||
low_cost_partial_config=low_cost_partial_config, # low cost initial point
|
||||
)
|
||||
```
|
||||
|
||||
@ -516,12 +516,12 @@ analysis = tune.run(
|
||||
```
|
||||
|
||||
### Lexicographic Objectives
|
||||
We support tuning multiple objectives with lexicographic preference by providing argument `lexico_objectives` for `tune.tun()`.
|
||||
We support tuning multiple objectives with lexicographic preference by providing argument `lexico_objectives` for `tune.tun()`.
|
||||
`lexico_objectives` is a dictionary that contains the following fields of key-value pairs:
|
||||
- `metrics`: a list of optimization objectives with the orders reflecting the priorities/preferences of the objectives.
|
||||
- `modes`: (optional) a list of optimization modes (each mode either "min" or "max") corresponding to the objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives.
|
||||
- `tolerances`: (optional) a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the numerical tolerances values.
|
||||
- `targets`: (optional) a dictionary to specify the optimization targets on the objectives. The keys are the metric names (provided in "metric"), and the values are the numerical target values.
|
||||
- `tolerances`: (optional) a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the numerical tolerances values.
|
||||
- `targets`: (optional) a dictionary to specify the optimization targets on the objectives. The keys are the metric names (provided in "metric"), and the values are the numerical target values.
|
||||
|
||||
In the following example, we want to minimize `val_loss` and `pred_time` of the model where `val_loss` has high priority. The tolerances for `val_loss` and `pre_time` are 0.02 and 0 respectively. We do not set targets for these two objectives and we set them to -inf for both objectives.
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user