mirror of
https://github.com/microsoft/autogen.git
synced 2025-09-02 12:57:21 +00:00
646 lines
26 KiB
Python
646 lines
26 KiB
Python
# !
|
|
# * Copyright (c) FLAML authors. All rights reserved.
|
|
# * Licensed under the MIT License. See LICENSE file in the
|
|
# * project root for license information.
|
|
from typing import Optional, Union, List, Callable, Tuple, Dict
|
|
import numpy as np
|
|
import datetime
|
|
import time
|
|
import os
|
|
from collections import defaultdict
|
|
|
|
try:
|
|
from ray import __version__ as ray_version
|
|
|
|
assert ray_version >= "1.10.0"
|
|
from ray.tune.analysis import ExperimentAnalysis as EA
|
|
|
|
ray_import = True
|
|
except (ImportError, AssertionError):
|
|
ray_import = False
|
|
from .analysis import ExperimentAnalysis as EA
|
|
|
|
from .trial import Trial
|
|
from .result import DEFAULT_METRIC, DEFAULT_MODE
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
_use_ray = True
|
|
_runner = None
|
|
_verbose = 0
|
|
_running_trial = None
|
|
_training_iteration = 0
|
|
|
|
INCUMBENT_RESULT = "__incumbent_result__"
|
|
|
|
|
|
class ExperimentAnalysis(EA):
|
|
"""Class for storing the experiment results."""
|
|
|
|
def __init__(self, trials, metric, mode, lexico_objectives=None):
|
|
try:
|
|
super().__init__(self, None, trials, metric, mode)
|
|
except (TypeError, ValueError):
|
|
self.trials = trials
|
|
self.default_metric = metric or DEFAULT_METRIC
|
|
self.default_mode = mode
|
|
self.lexico_objectives = lexico_objectives
|
|
|
|
@property
|
|
def best_trial(self) -> Trial:
|
|
if self.lexico_objectives is None:
|
|
return super().best_trial
|
|
else:
|
|
return self.get_best_trial(self.default_metric, self.default_mode)
|
|
|
|
@property
|
|
def best_config(self) -> Dict:
|
|
if self.lexico_objectives is None:
|
|
return super().best_config
|
|
else:
|
|
return self.get_best_config(self.default_metric, self.default_mode)
|
|
|
|
def lexico_best(self, trials):
|
|
results = {index: trial.last_result for index, trial in enumerate(trials)}
|
|
metrics = self.lexico_objectives["metrics"]
|
|
modes = self.lexico_objectives["modes"]
|
|
f_best = {}
|
|
keys = list(results.keys())
|
|
length = len(keys)
|
|
histories = defaultdict(list)
|
|
for time_index in range(length):
|
|
for objective, mode in zip(metrics, modes):
|
|
histories[objective].append(
|
|
results[keys[time_index]][objective]
|
|
if mode == "min"
|
|
else trials[keys[time_index]][objective] * -1
|
|
)
|
|
obj_initial = self.lexico_objectives["metrics"][0]
|
|
feasible_index = [*range(len(histories[obj_initial]))]
|
|
for k_metric in self.lexico_objectives["metrics"]:
|
|
k_values = np.array(histories[k_metric])
|
|
f_best[k_metric] = np.min(k_values.take(feasible_index))
|
|
feasible_index_prior = np.where(
|
|
k_values
|
|
<= max(
|
|
[
|
|
f_best[k_metric]
|
|
+ self.lexico_objectives["tolerances"][k_metric],
|
|
self.lexico_objectives["targets"][k_metric],
|
|
]
|
|
)
|
|
)[0].tolist()
|
|
feasible_index = [
|
|
val for val in feasible_index if val in feasible_index_prior
|
|
]
|
|
best_trial = trials[feasible_index[-1]]
|
|
return best_trial
|
|
|
|
def get_best_trial(
|
|
self,
|
|
metric: Optional[str] = None,
|
|
mode: Optional[str] = None,
|
|
scope: str = "last",
|
|
filter_nan_and_inf: bool = True,
|
|
) -> Optional[Trial]:
|
|
if self.lexico_objectives is not None:
|
|
best_trial = self.lexico_best(self.trials)
|
|
else:
|
|
best_trial = super().get_best_trial(metric, mode, scope, filter_nan_and_inf)
|
|
return best_trial
|
|
|
|
@property
|
|
def best_result(self) -> Dict:
|
|
if self.lexico_best is None:
|
|
return super().best_result
|
|
else:
|
|
return self.best_trial.last_result
|
|
|
|
|
|
def report(_metric=None, **kwargs):
|
|
|
|
"""A function called by the HPO application to report final or intermediate
|
|
results.
|
|
|
|
Example:
|
|
|
|
```python
|
|
import time
|
|
from flaml import tune
|
|
|
|
def compute_with_config(config):
|
|
current_time = time.time()
|
|
metric2minimize = (round(config['x'])-95000)**2
|
|
time2eval = time.time() - current_time
|
|
tune.report(metric2minimize=metric2minimize, time2eval=time2eval)
|
|
|
|
analysis = tune.run(
|
|
compute_with_config,
|
|
config={
|
|
'x': tune.lograndint(lower=1, upper=1000000),
|
|
'y': tune.randint(lower=1, upper=1000000)
|
|
},
|
|
metric='metric2minimize', mode='min',
|
|
num_samples=1000000, time_budget_s=60, use_ray=False)
|
|
|
|
print(analysis.trials[-1].last_result)
|
|
```
|
|
|
|
Args:
|
|
_metric: Optional default anonymous metric for ``tune.report(value)``.
|
|
(For compatibility with ray.tune.report)
|
|
**kwargs: Any key value pair to be reported.
|
|
|
|
Raises:
|
|
StopIteration (when not using ray, i.e., _use_ray=False):
|
|
A StopIteration exception is raised if the trial has been signaled to stop.
|
|
SystemExit (when using ray):
|
|
A SystemExit exception is raised if the trial has been signaled to stop by ray.
|
|
"""
|
|
global _use_ray
|
|
global _verbose
|
|
global _running_trial
|
|
global _training_iteration
|
|
if _use_ray:
|
|
try:
|
|
from ray import tune
|
|
|
|
return tune.report(_metric, **kwargs)
|
|
except ImportError:
|
|
# calling tune.report() outside tune.run()
|
|
return
|
|
result = kwargs
|
|
if _metric:
|
|
result[DEFAULT_METRIC] = _metric
|
|
trial = getattr(_runner, "running_trial", None)
|
|
if not trial:
|
|
return None
|
|
if _running_trial == trial:
|
|
_training_iteration += 1
|
|
else:
|
|
_training_iteration = 0
|
|
_running_trial = trial
|
|
result["training_iteration"] = _training_iteration
|
|
result["config"] = trial.config
|
|
if INCUMBENT_RESULT in result["config"]:
|
|
del result["config"][INCUMBENT_RESULT]
|
|
for key, value in trial.config.items():
|
|
result["config/" + key] = value
|
|
_runner.process_trial_result(trial, result)
|
|
if _verbose > 2:
|
|
logger.info(f"result: {result}")
|
|
if trial.is_finished():
|
|
raise StopIteration
|
|
|
|
|
|
def run(
|
|
evaluation_function,
|
|
config: Optional[dict] = None,
|
|
low_cost_partial_config: Optional[dict] = None,
|
|
cat_hp_cost: Optional[dict] = None,
|
|
metric: Optional[str] = None,
|
|
mode: Optional[str] = None,
|
|
time_budget_s: Union[int, float] = None,
|
|
points_to_evaluate: Optional[List[dict]] = None,
|
|
evaluated_rewards: Optional[List] = None,
|
|
resource_attr: Optional[str] = None,
|
|
min_resource: Optional[float] = None,
|
|
max_resource: Optional[float] = None,
|
|
reduction_factor: Optional[float] = None,
|
|
scheduler=None,
|
|
search_alg=None,
|
|
verbose: Optional[int] = 2,
|
|
local_dir: Optional[str] = None,
|
|
num_samples: Optional[int] = 1,
|
|
resources_per_trial: Optional[dict] = None,
|
|
config_constraints: Optional[
|
|
List[Tuple[Callable[[dict], float], str, float]]
|
|
] = None,
|
|
metric_constraints: Optional[List[Tuple[str, str, float]]] = None,
|
|
max_failure: Optional[int] = 100,
|
|
use_ray: Optional[bool] = False,
|
|
use_incumbent_result_in_evaluation: Optional[bool] = None,
|
|
lexico_objectives: Optional[dict] = None,
|
|
log_file_name: Optional[str] = None,
|
|
**ray_args,
|
|
):
|
|
"""The trigger for HPO.
|
|
|
|
Example:
|
|
|
|
```python
|
|
import time
|
|
from flaml import tune
|
|
|
|
def compute_with_config(config):
|
|
current_time = time.time()
|
|
metric2minimize = (round(config['x'])-95000)**2
|
|
time2eval = time.time() - current_time
|
|
tune.report(metric2minimize=metric2minimize, time2eval=time2eval)
|
|
# if the evaluation fails unexpectedly and the exception is caught,
|
|
# and it doesn't inform the goodness of the config,
|
|
# return {}
|
|
# if the failure indicates a config is bad,
|
|
# report a bad metric value like np.inf or -np.inf
|
|
# depending on metric mode being min or max
|
|
|
|
analysis = tune.run(
|
|
compute_with_config,
|
|
config={
|
|
'x': tune.lograndint(lower=1, upper=1000000),
|
|
'y': tune.randint(lower=1, upper=1000000)
|
|
},
|
|
metric='metric2minimize', mode='min',
|
|
num_samples=-1, time_budget_s=60, use_ray=False)
|
|
|
|
print(analysis.trials[-1].last_result)
|
|
```
|
|
|
|
Args:
|
|
evaluation_function: A user-defined evaluation function.
|
|
It takes a configuration as input, outputs a evaluation
|
|
result (can be a numerical value or a dictionary of string
|
|
and numerical value pairs) for the input configuration.
|
|
For machine learning tasks, it usually involves training and
|
|
scoring a machine learning model, e.g., through validation loss.
|
|
config: A dictionary to specify the search space.
|
|
low_cost_partial_config: A dictionary from a subset of
|
|
controlled dimensions to the initial low-cost values.
|
|
e.g., ```{'n_estimators': 4, 'max_leaves': 4}```
|
|
|
|
cat_hp_cost: A dictionary from a subset of categorical dimensions
|
|
to the relative cost of each choice.
|
|
e.g., ```{'tree_method': [1, 1, 2]}```
|
|
i.e., the relative cost of the
|
|
three choices of 'tree_method' is 1, 1 and 2 respectively
|
|
metric: A string of the metric name to optimize for.
|
|
mode: A string in ['min', 'max'] to specify the objective as
|
|
minimization or maximization.
|
|
time_budget_s: int or float | The time budget in seconds.
|
|
points_to_evaluate: A list of initial hyperparameter
|
|
configurations to run first.
|
|
evaluated_rewards (list): If you have previously evaluated the
|
|
parameters passed in as points_to_evaluate you can avoid
|
|
re-running those trials by passing in the reward attributes
|
|
as a list so the optimiser can be told the results without
|
|
needing to re-compute the trial. Must be the same or shorter length than
|
|
points_to_evaluate.
|
|
e.g.,
|
|
|
|
```python
|
|
points_to_evaluate = [
|
|
{"b": .99, "cost_related": {"a": 3}},
|
|
{"b": .99, "cost_related": {"a": 2}},
|
|
]
|
|
evaluated_rewards = [3.0]
|
|
```
|
|
|
|
means that you know the reward for the first config in
|
|
points_to_evaluate is 3.0 and want to inform run().
|
|
|
|
resource_attr: A string to specify the resource dimension used by
|
|
the scheduler via "scheduler".
|
|
min_resource: A float of the minimal resource to use for the resource_attr.
|
|
max_resource: A float of the maximal resource to use for the resource_attr.
|
|
reduction_factor: A float of the reduction factor used for incremental
|
|
pruning.
|
|
scheduler: A scheduler for executing the experiment. Can be None, 'flaml',
|
|
'asha' (or 'async_hyperband', 'asynchyperband') or a custom instance of the TrialScheduler class. Default is None:
|
|
in this case when resource_attr is provided, the 'flaml' scheduler will be
|
|
used, otherwise no scheduler will be used. When set 'flaml', an
|
|
authentic scheduler implemented in FLAML will be used. It does not
|
|
require users to report intermediate results in evaluation_function.
|
|
Find more details about this scheduler in this paper
|
|
https://arxiv.org/pdf/1911.04706.pdf).
|
|
When set 'asha', the input for arguments "resource_attr",
|
|
"min_resource", "max_resource" and "reduction_factor" will be passed
|
|
to ASHA's "time_attr", "max_t", "grace_period" and "reduction_factor"
|
|
respectively. You can also provide a self-defined scheduler instance
|
|
of the TrialScheduler class. When 'asha' or self-defined scheduler is
|
|
used, you usually need to report intermediate results in the evaluation
|
|
function via 'tune.report()'.
|
|
If you would like to do some cleanup opearation when the trial is stopped
|
|
by the scheduler, you can catch the `StopIteration` (when not using ray)
|
|
or `SystemExit` (when using ray) exception explicitly,
|
|
as shown in the following example.
|
|
Please find more examples using different types of schedulers
|
|
and how to set up the corresponding evaluation functions in
|
|
test/tune/test_scheduler.py, and test/tune/example_scheduler.py.
|
|
```python
|
|
def easy_objective(config):
|
|
width, height = config["width"], config["height"]
|
|
for step in range(config["steps"]):
|
|
intermediate_score = evaluation_fn(step, width, height)
|
|
try:
|
|
tune.report(iterations=step, mean_loss=intermediate_score)
|
|
except (StopIteration, SystemExit):
|
|
# do cleanup operation here
|
|
return
|
|
```
|
|
search_alg: An instance of BlendSearch as the search algorithm
|
|
to be used. The same instance can be used for iterative tuning.
|
|
e.g.,
|
|
|
|
```python
|
|
from flaml import BlendSearch
|
|
algo = BlendSearch(metric='val_loss', mode='min',
|
|
space=search_space,
|
|
low_cost_partial_config=low_cost_partial_config)
|
|
for i in range(10):
|
|
analysis = tune.run(compute_with_config,
|
|
search_alg=algo, use_ray=False)
|
|
print(analysis.trials[-1].last_result)
|
|
```
|
|
|
|
verbose: 0, 1, 2, or 3. Verbosity mode for ray if ray backend is used.
|
|
0 = silent, 1 = only status updates, 2 = status and brief trial
|
|
results, 3 = status and detailed trial results. Defaults to 2.
|
|
local_dir: A string of the local dir to save ray logs if ray backend is
|
|
used; or a local dir to save the tuning log.
|
|
num_samples: An integer of the number of configs to try. Defaults to 1.
|
|
resources_per_trial: A dictionary of the hardware resources to allocate
|
|
per trial, e.g., `{'cpu': 1}`. It is only valid when using ray backend
|
|
(by setting 'use_ray = True'). It shall be used when you need to do
|
|
[parallel tuning](../../Use-Cases/Tune-User-Defined-Function#parallel-tuning).
|
|
config_constraints: A list of config constraints to be satisfied.
|
|
e.g., ```config_constraints = [(mem_size, '<=', 1024**3)]```
|
|
|
|
mem_size is a function which produces a float number for the bytes
|
|
needed for a config.
|
|
It is used to skip configs which do not fit in memory.
|
|
metric_constraints: A list of metric constraints to be satisfied.
|
|
e.g., `['precision', '>=', 0.9]`. The sign can be ">=" or "<=".
|
|
max_failure: int | the maximal consecutive number of failures to sample
|
|
a trial before the tuning is terminated.
|
|
use_ray: A boolean of whether to use ray as the backend.
|
|
lexico_objectives: A dictionary with four elements.
|
|
It specifics the information used for multiple objectives optimization with lexicographic preference.
|
|
e.g.,
|
|
```python
|
|
lexico_objectives = {"metrics":["error_rate","pred_time"], "modes":["min","min"],
|
|
"tolerances":{"error_rate":0.01,"pred_time":0.0}, "targets":{"error_rate":0.0,"pred_time":0.0}}
|
|
```
|
|
Either "metrics" or "modes" is a list of str.
|
|
It represents the optimization objectives, the objective as minimization or maximization respectively.
|
|
Both "metrics" and "modes" are ordered by priorities from high to low.
|
|
"tolerances" is a dictionary to specify the optimality tolerance of each objective.
|
|
"targets" is a dictionary to specify the optimization targets for each objective.
|
|
If providing lexico_objectives, the arguments metric, mode, and search_alg will be invalid.
|
|
|
|
log_file_name: A string of the log file name. Default to None.
|
|
When set to None:
|
|
if local_dir is not given, no log file is created;
|
|
if local_dir is given, the log file name will be autogenerated under local_dir.
|
|
Only valid when verbose > 0 or use_ray is True.
|
|
**ray_args: keyword arguments to pass to ray.tune.run().
|
|
Only valid when use_ray=True.
|
|
"""
|
|
global _use_ray
|
|
global _verbose
|
|
global _running_trial
|
|
global _training_iteration
|
|
old_use_ray = _use_ray
|
|
old_verbose = _verbose
|
|
old_running_trial = _running_trial
|
|
old_training_iteration = _training_iteration
|
|
if local_dir and not log_file_name and verbose > 0:
|
|
os.makedirs(local_dir, exist_ok=True)
|
|
log_file_name = os.path.join(
|
|
local_dir, "tune_" + str(datetime.datetime.now()).replace(":", "-") + ".log"
|
|
)
|
|
if not use_ray:
|
|
_verbose = verbose
|
|
old_handlers = logger.handlers
|
|
old_level = logger.getEffectiveLevel()
|
|
logger.handlers = []
|
|
global _runner
|
|
old_runner = _runner
|
|
assert not ray_args, "ray_args is only valid when use_ray=True"
|
|
if (
|
|
old_handlers
|
|
and isinstance(old_handlers[0], logging.StreamHandler)
|
|
and not isinstance(old_handlers[0], logging.FileHandler)
|
|
):
|
|
# Add the console handler.
|
|
logger.addHandler(old_handlers[0])
|
|
if verbose > 0:
|
|
if log_file_name:
|
|
logger.addHandler(logging.FileHandler(log_file_name))
|
|
elif not logger.hasHandlers():
|
|
# Add the console handler.
|
|
_ch = logging.StreamHandler()
|
|
logger_formatter = logging.Formatter(
|
|
"[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s",
|
|
"%m-%d %H:%M:%S",
|
|
)
|
|
_ch.setFormatter(logger_formatter)
|
|
logger.addHandler(_ch)
|
|
if verbose <= 2:
|
|
logger.setLevel(logging.INFO)
|
|
else:
|
|
logger.setLevel(logging.DEBUG)
|
|
else:
|
|
logger.setLevel(logging.CRITICAL)
|
|
|
|
from .searcher.blendsearch import BlendSearch, CFO
|
|
|
|
if search_alg is None:
|
|
flaml_scheduler_resource_attr = (
|
|
flaml_scheduler_min_resource
|
|
) = flaml_scheduler_max_resource = flaml_scheduler_reduction_factor = None
|
|
if scheduler in (None, "flaml"):
|
|
# when scheduler is set 'flaml' or None, we will use a scheduler that is
|
|
# authentic to the search algorithms in flaml. After setting up
|
|
# the search algorithm accordingly, we need to set scheduler to
|
|
# None in case it is later used in the trial runner.
|
|
flaml_scheduler_resource_attr = resource_attr
|
|
flaml_scheduler_min_resource = min_resource
|
|
flaml_scheduler_max_resource = max_resource
|
|
flaml_scheduler_reduction_factor = reduction_factor
|
|
scheduler = None
|
|
try:
|
|
import optuna as _
|
|
|
|
if lexico_objectives is None:
|
|
SearchAlgorithm = BlendSearch
|
|
else:
|
|
SearchAlgorithm = CFO
|
|
except ImportError:
|
|
SearchAlgorithm = CFO
|
|
logger.warning(
|
|
"Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]"
|
|
)
|
|
if lexico_objectives is None:
|
|
metric = metric or DEFAULT_METRIC
|
|
else:
|
|
metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
|
|
search_alg = SearchAlgorithm(
|
|
metric=metric,
|
|
mode=mode,
|
|
space=config,
|
|
points_to_evaluate=points_to_evaluate,
|
|
evaluated_rewards=evaluated_rewards,
|
|
low_cost_partial_config=low_cost_partial_config,
|
|
cat_hp_cost=cat_hp_cost,
|
|
time_budget_s=time_budget_s,
|
|
num_samples=num_samples,
|
|
resource_attr=flaml_scheduler_resource_attr,
|
|
min_resource=flaml_scheduler_min_resource,
|
|
max_resource=flaml_scheduler_max_resource,
|
|
reduction_factor=flaml_scheduler_reduction_factor,
|
|
config_constraints=config_constraints,
|
|
metric_constraints=metric_constraints,
|
|
use_incumbent_result_in_evaluation=use_incumbent_result_in_evaluation,
|
|
lexico_objectives=lexico_objectives,
|
|
)
|
|
else:
|
|
if metric is None or mode is None:
|
|
metric = metric or search_alg.metric or DEFAULT_METRIC
|
|
mode = mode or search_alg.mode
|
|
if ray_import:
|
|
if ray_version.startswith("1."):
|
|
from ray.tune.suggest import ConcurrencyLimiter
|
|
else:
|
|
from ray.tune.search import ConcurrencyLimiter
|
|
else:
|
|
from flaml.tune.searcher.suggestion import ConcurrencyLimiter
|
|
if (
|
|
search_alg.__class__.__name__
|
|
in [
|
|
"BlendSearch",
|
|
"CFO",
|
|
"CFOCat",
|
|
]
|
|
and use_incumbent_result_in_evaluation is not None
|
|
):
|
|
search_alg.use_incumbent_result_in_evaluation = (
|
|
use_incumbent_result_in_evaluation
|
|
)
|
|
searcher = (
|
|
search_alg.searcher
|
|
if isinstance(search_alg, ConcurrencyLimiter)
|
|
else search_alg
|
|
)
|
|
if isinstance(searcher, BlendSearch):
|
|
setting = {}
|
|
if time_budget_s:
|
|
setting["time_budget_s"] = time_budget_s
|
|
if num_samples > 0:
|
|
setting["num_samples"] = num_samples
|
|
searcher.set_search_properties(metric, mode, config, **setting)
|
|
else:
|
|
searcher.set_search_properties(metric, mode, config)
|
|
if scheduler in ("asha", "asynchyperband", "async_hyperband"):
|
|
params = {}
|
|
# scheduler resource_dimension=resource_attr
|
|
if resource_attr:
|
|
params["time_attr"] = resource_attr
|
|
if max_resource:
|
|
params["max_t"] = max_resource
|
|
if min_resource:
|
|
params["grace_period"] = min_resource
|
|
if reduction_factor:
|
|
params["reduction_factor"] = reduction_factor
|
|
if ray_import:
|
|
from ray.tune.schedulers import ASHAScheduler
|
|
|
|
scheduler = ASHAScheduler(**params)
|
|
if use_ray:
|
|
try:
|
|
from ray import tune
|
|
except ImportError:
|
|
raise ImportError(
|
|
"Failed to import ray tune. "
|
|
"Please install ray[tune] or set use_ray=False"
|
|
)
|
|
_use_ray = True
|
|
try:
|
|
analysis = tune.run(
|
|
evaluation_function,
|
|
metric=metric,
|
|
mode=mode,
|
|
search_alg=search_alg,
|
|
scheduler=scheduler,
|
|
time_budget_s=time_budget_s,
|
|
verbose=verbose,
|
|
local_dir=local_dir,
|
|
num_samples=num_samples,
|
|
resources_per_trial=resources_per_trial,
|
|
**ray_args,
|
|
)
|
|
if log_file_name:
|
|
with open(log_file_name, "w") as f:
|
|
for trial in analysis.trials:
|
|
f.write(f"result: {trial.last_result}\n")
|
|
return analysis
|
|
finally:
|
|
_use_ray = old_use_ray
|
|
_verbose = old_verbose
|
|
_running_trial = old_running_trial
|
|
_training_iteration = old_training_iteration
|
|
|
|
# simple sequential run without using tune.run() from ray
|
|
time_start = time.time()
|
|
_use_ray = False
|
|
if scheduler:
|
|
scheduler.set_search_properties(metric=metric, mode=mode)
|
|
from .trial_runner import SequentialTrialRunner
|
|
|
|
try:
|
|
_runner = SequentialTrialRunner(
|
|
search_alg=search_alg,
|
|
scheduler=scheduler,
|
|
metric=metric,
|
|
mode=mode,
|
|
)
|
|
num_trials = 0
|
|
if time_budget_s is None:
|
|
time_budget_s = np.inf
|
|
fail = 0
|
|
ub = (len(evaluated_rewards) if evaluated_rewards else 0) + max_failure
|
|
while (
|
|
time.time() - time_start < time_budget_s
|
|
and (num_samples < 0 or num_trials < num_samples)
|
|
and fail < ub
|
|
):
|
|
trial_to_run = _runner.step()
|
|
if trial_to_run:
|
|
num_trials += 1
|
|
if verbose:
|
|
logger.info(f"trial {num_trials} config: {trial_to_run.config}")
|
|
result = evaluation_function(trial_to_run.config)
|
|
if result is not None:
|
|
if isinstance(result, dict):
|
|
if result:
|
|
report(**result)
|
|
else:
|
|
# When the result returned is an empty dict, set the trial status to error
|
|
trial_to_run.set_status(Trial.ERROR)
|
|
else:
|
|
report(_metric=result)
|
|
_runner.stop_trial(trial_to_run)
|
|
fail = 0
|
|
else:
|
|
fail += 1 # break with ub consecutive failures
|
|
if fail == ub:
|
|
logger.warning(
|
|
f"fail to sample a trial for {max_failure} times in a row, stopping."
|
|
)
|
|
analysis = ExperimentAnalysis(
|
|
_runner.get_trials(),
|
|
metric=metric,
|
|
mode=mode,
|
|
lexico_objectives=lexico_objectives,
|
|
)
|
|
return analysis
|
|
finally:
|
|
# recover the global variables in case of nested run
|
|
_use_ray = old_use_ray
|
|
_verbose = old_verbose
|
|
_running_trial = old_running_trial
|
|
_training_iteration = old_training_iteration
|
|
if not use_ray:
|
|
_runner = old_runner
|
|
logger.handlers = old_handlers
|
|
logger.setLevel(old_level)
|