mirror of
https://github.com/microsoft/autogen.git
synced 2025-09-29 10:09:30 +00:00
c45741a67b
* support latest xgboost version * Update test_classification.py * Update Exists problems when installing xgb1.6.1 in py3.6 * cleanup * xgboost version * remove time_budget_s in test * remove redundancy * stop support of python 3.6 Co-authored-by: zsk <shaokunzhang529@gmail.com> Co-authored-by: Qingyun Wu <qingyun.wu@psu.edu>
125 lines
3.6 KiB
Python
125 lines
3.6 KiB
Python
from functools import partial
|
|
|
|
|
|
def _evaluation_fn(step, width, height):
|
|
return (0.1 + width * step / 100) ** (-1) + height * 0.1
|
|
|
|
|
|
def _easy_objective(use_raytune, config):
|
|
if use_raytune:
|
|
from ray import tune
|
|
else:
|
|
from flaml import tune
|
|
# Hyperparameters
|
|
width, height = config["width"], config["height"]
|
|
|
|
for step in range(config["steps"]):
|
|
# Iterative training function - can be any arbitrary training procedure
|
|
intermediate_score = _evaluation_fn(step, width, height)
|
|
# Feed the score back back to Tune.
|
|
try:
|
|
tune.report(iterations=step, mean_loss=intermediate_score)
|
|
except StopIteration:
|
|
print("Trial stopped", step)
|
|
return
|
|
|
|
|
|
def test_tune(externally_setup_searcher=False, use_ray=False, use_raytune=False):
|
|
from flaml import tune
|
|
from flaml.searcher.blendsearch import BlendSearch
|
|
|
|
easy_objective_custom_tune = partial(_easy_objective, use_raytune)
|
|
search_space = {
|
|
"steps": 100,
|
|
"width": tune.uniform(0, 20),
|
|
"height": tune.uniform(-100, 100),
|
|
}
|
|
if externally_setup_searcher:
|
|
|
|
searcher = BlendSearch(
|
|
space=search_space,
|
|
time_budget_s=5,
|
|
metric="mean_loss",
|
|
mode="min",
|
|
)
|
|
assert (
|
|
searcher.cost_attr == "time_total_s"
|
|
), "when time_budget_s is provided, cost_attr should be time_total_s"
|
|
|
|
searcher = BlendSearch(
|
|
space=search_space,
|
|
num_samples=10,
|
|
metric="mean_loss",
|
|
mode="min",
|
|
)
|
|
assert (
|
|
searcher.cost_attr is None
|
|
), "when time_budget_s is not provided, cost_attr should be None."
|
|
|
|
searcher = BlendSearch(
|
|
space=search_space,
|
|
num_samples=10,
|
|
time_budget_s=5,
|
|
metric="mean_loss",
|
|
mode="min",
|
|
)
|
|
assert (
|
|
searcher.cost_attr == "time_total_s"
|
|
), "As long as time_budget_s is provided and cost_attr not otherwise specified (i.e., using the default auto value), time_total_s is used as the cost_attr"
|
|
|
|
searcher = BlendSearch(
|
|
space=search_space,
|
|
num_samples=10,
|
|
time_budget_s=5,
|
|
metric="mean_loss",
|
|
mode="min",
|
|
cost_attr=None,
|
|
)
|
|
assert (
|
|
searcher.cost_attr is None
|
|
), "When the cost_attr is explicitly specified to be None, BS should use None as the cost_attr."
|
|
|
|
searcher = BlendSearch(
|
|
space=search_space,
|
|
metric="mean_loss",
|
|
mode="min",
|
|
)
|
|
else:
|
|
searcher = None
|
|
|
|
analysis = tune.run(
|
|
easy_objective_custom_tune,
|
|
search_alg=searcher,
|
|
metric="mean_loss",
|
|
mode="min",
|
|
num_samples=10,
|
|
# time_budget_s=5,
|
|
use_ray=use_ray,
|
|
config=search_space,
|
|
)
|
|
|
|
print("Best hyperparameters found were: ", analysis.best_config)
|
|
print("best results", analysis.best_result)
|
|
print("best results", analysis.results)
|
|
return analysis.best_config
|
|
|
|
|
|
def test_reproducibility():
|
|
best_config_1 = test_tune()
|
|
best_config_2 = test_tune()
|
|
print(best_config_1)
|
|
print(best_config_2)
|
|
assert best_config_1 == best_config_2, "flaml.tune not reproducible"
|
|
|
|
best_config_1 = test_tune(externally_setup_searcher=True)
|
|
best_config_2 = test_tune(externally_setup_searcher=True)
|
|
print(best_config_1)
|
|
print(best_config_2)
|
|
assert (
|
|
best_config_1 == best_config_2
|
|
), "flaml.tune not reproducible when the searcher is set up externally"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_reproducibility()
|