From 00da79a90b12158d3ce799099a8f09ae5c7d8750 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Sat, 20 Nov 2021 20:05:28 -0800 Subject: [PATCH] empty search space (#295) fix the error when an empty dictionary is passed to BlendSearch as the search space. --- flaml/searcher/blendsearch.py | 2 +- test/tune/test_searcher.py | 297 ++++++++++++++++++---------------- 2 files changed, 155 insertions(+), 144 deletions(-) diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py index eb8fce9da..147d1c055 100644 --- a/flaml/searcher/blendsearch.py +++ b/flaml/searcher/blendsearch.py @@ -219,7 +219,7 @@ class BlendSearch(Searcher): else: self._candidate_start_points = None self._time_budget_s, self._num_samples = time_budget_s, num_samples - if space: + if space is not None: self._init_search() def set_search_properties( diff --git a/test/tune/test_searcher.py b/test/tune/test_searcher.py index ce07670ac..52d741d52 100644 --- a/test/tune/test_searcher.py +++ b/test/tune/test_searcher.py @@ -5,153 +5,164 @@ try: assert ray_version >= "1.0.0" from ray.tune import sample + + use_ray = True except (ImportError, AssertionError): from flaml.tune import sample - from flaml.searcher.suggestion import OptunaSearch, Searcher, ConcurrencyLimiter - from flaml.searcher.blendsearch import BlendSearch, CFO, RandomSearch + use_ray = False - def define_search_space(trial): - trial.suggest_float("a", 6, 8) - trial.suggest_float("b", 1e-4, 1e-2, log=True) +from flaml.searcher.suggestion import OptunaSearch, Searcher, ConcurrencyLimiter +from flaml.searcher.blendsearch import BlendSearch, CFO, RandomSearch - def test_searcher(): - searcher = Searcher() - searcher = Searcher(metric=["m1", "m2"], mode=["max", "min"]) - searcher.set_search_properties(None, None, None) - searcher.suggest = searcher.on_pause = searcher.on_unpause = lambda _: {} - searcher.on_trial_complete = lambda trial_id, result, error: None - searcher = ConcurrencyLimiter(searcher, max_concurrent=2, batch=True) - searcher.suggest("t1") - searcher.suggest("t2") - searcher.on_pause("t1") - searcher.on_unpause("t1") - searcher.suggest("t3") - searcher.on_trial_complete("t1", {}) - searcher.on_trial_complete("t2", {}) - searcher.set_state({}) - print(searcher.get_state()) - import optuna - config = { - "a": optuna.distributions.UniformDistribution(6, 8), - "b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2), +def define_search_space(trial): + trial.suggest_float("a", 6, 8) + trial.suggest_float("b", 1e-4, 1e-2, log=True) + + +def test_searcher(): + searcher = Searcher() + searcher = Searcher(metric=["m1", "m2"], mode=["max", "min"]) + searcher.set_search_properties(None, None, None) + searcher.suggest = searcher.on_pause = searcher.on_unpause = lambda _: {} + searcher.on_trial_complete = lambda trial_id, result, error: None + searcher = ConcurrencyLimiter(searcher, max_concurrent=2, batch=True) + searcher.suggest("t1") + searcher.suggest("t2") + searcher.on_pause("t1") + searcher.on_unpause("t1") + searcher.suggest("t3") + searcher.on_trial_complete("t1", {}) + searcher.on_trial_complete("t2", {}) + searcher.set_state({}) + print(searcher.get_state()) + import optuna + + config = { + "a": optuna.distributions.UniformDistribution(6, 8), + "b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2), + } + searcher = OptunaSearch( + config, + points_to_evaluate=[{"a": 6, "b": 1e-3}], + evaluated_rewards=[{"m": 2}], + metric="m", + mode="max", + ) + config = {"a": sample.uniform(6, 8), "b": sample.loguniform(1e-4, 1e-2)} + # searcher = OptunaSearch( + # config, + # points_to_evaluate=[{"a": 6, "b": 1e-3}], + # evaluated_rewards=[{"m": 2}], + # metric="m", + # mode="max", + # ) + searcher = OptunaSearch( + define_search_space, + points_to_evaluate=[{"a": 6, "b": 1e-3}], + # evaluated_rewards=[{'m': 2}], metric='m', mode='max' + mode="max", + ) + # searcher = OptunaSearch() + # searcher.set_search_properties('m', 'min', define_search_space) + searcher.set_search_properties("m", "min", config) + searcher.suggest("t1") + searcher.on_trial_complete("t1", None, False) + searcher.suggest("t2") + searcher.on_trial_complete("t2", None, True) + searcher.suggest("t3") + searcher.on_trial_complete("t3", {"m": np.nan}) + searcher.save("test/tune/optuna.pickle") + searcher.restore("test/tune/optuna.pickle") + searcher = BlendSearch( + metric="m", global_search_alg=searcher, metric_constraints=[("c", "<", 1)] + ) + searcher.set_search_properties( + metric="m2", config=config, setting={"time_budget_s": 0} + ) + c = searcher.suggest("t1") + searcher.on_trial_complete("t1", {"config": c}, True) + c = searcher.suggest("t2") + searcher.on_trial_complete("t2", {"config": c, "m2": 1, "c": 2, "time_total_s": 1}) + config1 = config.copy() + config1["_choice_"] = 0 + searcher._expand_admissible_region( + lower={"root": [{"a": 0.5}, {"a": 0.4}]}, + upper={"root": [{"a": 0.9}, {"a": 0.8}]}, + space={"root": config1}, + ) + searcher = CFO( + metric="m", + mode="min", + space=config, + points_to_evaluate=[{"a": 7, "b": 1e-3}, {"a": 6, "b": 3e-4}], + evaluated_rewards=[1, 1], + ) + searcher.suggest("t1") + searcher.suggest("t2") + searcher.on_trial_result("t3", {}) + c = searcher.generate_parameters(1) + searcher.receive_trial_result(1, c, {"default": 0}) + searcher.update_search_space( + { + "a": { + "_value": [1, 2], + "_type": "choice", + }, + "b": { + "_value": [1, 3], + "_type": "randint", + }, + "c": { + "_value": [0.1, 3], + "_type": "uniform", + }, + "d": { + "_value": [2, 8, 2], + "_type": "quniform", + }, + "e": { + "_value": [2, 8], + "_type": "loguniform", + }, + "f": { + "_value": [2, 8, 2], + "_type": "qloguniform", + }, + "g": { + "_value": [0, 2], + "_type": "normal", + }, + "h": { + "_value": [0, 2, 2], + "_type": "qnormal", + }, } - searcher = OptunaSearch( - config, - points_to_evaluate=[{"a": 6, "b": 1e-3}], - evaluated_rewards=[{"m": 2}], - metric="m", - mode="max", - ) - config = {"a": sample.uniform(6, 8), "b": sample.loguniform(1e-4, 1e-2)} - searcher = OptunaSearch( - config, - points_to_evaluate=[{"a": 6, "b": 1e-3}], - evaluated_rewards=[{"m": 2}], - metric="m", - mode="max", - ) - searcher = OptunaSearch( - define_search_space, - points_to_evaluate=[{"a": 6, "b": 1e-3}], - # evaluated_rewards=[{'m': 2}], metric='m', mode='max' - mode="max", - ) - searcher = OptunaSearch() - # searcher.set_search_properties('m', 'min', define_search_space) - searcher.set_search_properties("m", "min", config) - searcher.suggest("t1") - searcher.on_trial_complete("t1", None, False) - searcher.suggest("t2") - searcher.on_trial_complete("t2", None, True) - searcher.suggest("t3") - searcher.on_trial_complete("t3", {"m": np.nan}) - searcher.save("test/tune/optuna.pickle") - searcher.restore("test/tune/optuna.pickle") - searcher = BlendSearch( - metric="m", global_search_alg=searcher, metric_constraints=[("c", "<", 1)] - ) - searcher.set_search_properties( - metric="m2", config=config, setting={"time_budget_s": 0} - ) - c = searcher.suggest("t1") - searcher.on_trial_complete("t1", {"config": c}, True) - c = searcher.suggest("t2") - searcher.on_trial_complete( - "t2", {"config": c, "m2": 1, "c": 2, "time_total_s": 1} - ) - config1 = config.copy() - config1["_choice_"] = 0 - searcher._expand_admissible_region( - lower={"root": [{"a": 0.5}, {"a": 0.4}]}, - upper={"root": [{"a": 0.9}, {"a": 0.8}]}, - space={"root": config1}, - ) - searcher = CFO( - metric="m", - mode="min", - space=config, - points_to_evaluate=[{"a": 7, "b": 1e-3}, {"a": 6, "b": 3e-4}], - evaluated_rewards=[1, 1], - ) - searcher.suggest("t1") - searcher.suggest("t2") - searcher.on_trial_result("t3", {}) - c = searcher.generate_parameters(1) - searcher.receive_trial_result(1, c, {"default": 0}) - searcher.update_search_space( - { - "a": { - "_value": [1, 2], - "_type": "choice", - }, - "b": { - "_value": [1, 3], - "_type": "randint", - }, - "c": { - "_value": [0.1, 3], - "_type": "uniform", - }, - "d": { - "_value": [2, 8, 2], - "_type": "quniform", - }, - "e": { - "_value": [2, 8], - "_type": "loguniform", - }, - "f": { - "_value": [2, 8, 2], - "_type": "qloguniform", - }, - "g": { - "_value": [0, 2], - "_type": "normal", - }, - "h": { - "_value": [0, 2, 2], - "_type": "qnormal", - }, - } - ) - np.random.seed(7654321) - searcher = RandomSearch( - space=config, - points_to_evaluate=[{"a": 7, "b": 1e-3}, {"a": 6, "b": 3e-4}], - ) - print(searcher.suggest("t1")) - print(searcher.suggest("t2")) - print(searcher.suggest("t3")) - print(searcher.suggest("t4")) - searcher.on_trial_complete({"t1"}, {}) - searcher.on_trial_result({"t2"}, {}) - np.random.seed(654321) - searcher = RandomSearch( - space=config, - points_to_evaluate=[{"a": 7, "b": 1e-3}, {"a": 6, "b": 3e-4}], - ) - print(searcher.suggest("t1")) - print(searcher.suggest("t2")) - print(searcher.suggest("t3")) + ) + np.random.seed(7654321) + searcher = RandomSearch( + space=config, + points_to_evaluate=[{"a": 7, "b": 1e-3}, {"a": 6, "b": 3e-4}], + ) + print(searcher.suggest("t1")) + print(searcher.suggest("t2")) + print(searcher.suggest("t3")) + print(searcher.suggest("t4")) + searcher.on_trial_complete({"t1"}, {}) + searcher.on_trial_result({"t2"}, {}) + np.random.seed(654321) + searcher = RandomSearch( + space=config, + points_to_evaluate=[{"a": 7, "b": 1e-3}, {"a": 6, "b": 3e-4}], + ) + print(searcher.suggest("t1")) + print(searcher.suggest("t2")) + print(searcher.suggest("t3")) + searcher = RandomSearch(space={}) + print(searcher.suggest("t1")) + searcher = BlendSearch(space={}) + print(searcher.suggest("t1")) + from flaml import tune + + tune.run(lambda x: 1, config={}, use_ray=use_ray)