From 6a7caa6a3d06b1d8253b250471eaa0d62c4d6df6 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Sun, 23 Jan 2022 01:24:15 -0800 Subject: [PATCH] max_iter < 2 -> no search; sign in metric constraints; test and example for forecasting (#415) * max_iter < 2 -> no search * use_ray in test * eval_method in ts example * check sign of constraints * test metric constraint sign --- flaml/automl.py | 30 +- flaml/searcher/blendsearch.py | 6 +- test/automl/test_forecast.py | 50 +- test/automl/test_training_log.py | 23 +- test/tune/test_searcher.py | 9 +- .../Examples/AutoML-Time series forecast.md | 466 +++++++++++++----- 6 files changed, 412 insertions(+), 172 deletions(-) diff --git a/flaml/automl.py b/flaml/automl.py index 834ba8f6d..04869d578 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -2424,20 +2424,6 @@ class AutoML(BaseEstimator): est_retrain_time = next_trial_time = 0 best_config_sig = None better = True # whether we find a better model in one trial - if self._ensemble: - self.best_model = {} - if self._max_iter < 2 and self.estimator_list and self._state.retrain_final: - # when max_iter is 1, no need to search - # TODO: otherwise, need to make sure SearchStates.init_config is inside search space - self._max_iter = 0 - self._best_estimator = estimator = self.estimator_list[0] - self._selected = state = self._search_states[estimator] - state.best_config_sample_size = self._state.data_size[0] - state.best_config = ( - state.init_config - if isinstance(state.init_config, dict) - else state.init_config[0] - ) for self._track_iter in range(self._max_iter): if self._estimator_index is None: estimator = self._active_estimators[0] @@ -2699,8 +2685,20 @@ class AutoML(BaseEstimator): self._warn_threshold = 10 self._selected = None self.modelcount = 0 - - if not self._use_ray: + if self._max_iter < 2 and self.estimator_list and self._state.retrain_final: + # when max_iter is 1, no need to search + # TODO: otherwise, need to make sure SearchStates.init_config is inside search space + self.modelcount = self._max_iter + self._max_iter = 0 + self._best_estimator = estimator = self.estimator_list[0] + self._selected = state = self._search_states[estimator] + state.best_config_sample_size = self._state.data_size[0] + state.best_config = ( + state.init_config + if isinstance(state.init_config, dict) + else state.init_config[0] + ) + elif not self._use_ray: self._search_sequential() else: self._search_parallel() diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py index 494000199..5873236cb 100644 --- a/flaml/searcher/blendsearch.py +++ b/flaml/searcher/blendsearch.py @@ -130,7 +130,10 @@ class BlendSearch(Searcher): self._evaluated_rewards = evaluated_rewards or [] self._config_constraints = config_constraints self._metric_constraints = metric_constraints - if self._metric_constraints: + if metric_constraints: + assert all( + x[1] in ["<=", ">="] for x in metric_constraints + ), "sign of metric constraints must be <= or >=." # metric modified by lagrange metric += self.lagrange self._cat_hp_cost = cat_hp_cost or {} @@ -348,7 +351,6 @@ class BlendSearch(Searcher): metric_constraint, sign, threshold = constraint value = result.get(metric_constraint) if value: - # sign is <= or >= sign_op = 1 if sign == "<=" else -1 violation = (value - threshold) * sign_op if violation > 0: diff --git a/test/automl/test_forecast.py b/test/automl/test_forecast.py index 797e782d3..8b247b939 100644 --- a/test/automl/test_forecast.py +++ b/test/automl/test_forecast.py @@ -95,34 +95,30 @@ def test_numpy(): X_train = np.arange("2014-01", "2021-01", dtype="datetime64[M]") y_train = np.random.random(size=len(X_train)) automl = AutoML() - try: - import prophet + automl.fit( + X_train=X_train[:72], # a single column of timestamp + y_train=y_train[:72], # value for each timestamp + period=12, # time horizon to forecast, e.g., 12 months + task="ts_forecast", + time_budget=3, # time budget in seconds + log_file_name="test/ts_forecast.log", + n_splits=3, # number of splits + ) + print(automl.predict(X_train[72:])) - automl.fit( - X_train=X_train[:72], # a single column of timestamp - y_train=y_train[:72], # value for each timestamp - period=12, # time horizon to forecast, e.g., 12 months - task="ts_forecast", - time_budget=3, # time budget in seconds - log_file_name="test/ts_forecast.log", - n_splits=3, # number of splits - ) - print(automl.predict(X_train[72:])) - except ImportError: - print("not using prophet due to ImportError") - automl = AutoML() - automl.fit( - X_train=X_train[:72], # a single column of timestamp - y_train=y_train[:72], # value for each timestamp - period=12, # time horizon to forecast, e.g., 12 months - task="ts_forecast", - time_budget=1, # time budget in seconds - estimator_list=["arima", "sarimax"], - log_file_name="test/ts_forecast.log", - ) - print(automl.predict(X_train[72:])) - # an alternative way to specify predict steps for arima/sarimax - print(automl.predict(12)) + automl = AutoML() + automl.fit( + X_train=X_train[:72], # a single column of timestamp + y_train=y_train[:72], # value for each timestamp + period=12, # time horizon to forecast, e.g., 12 months + task="ts_forecast", + time_budget=1, # time budget in seconds + estimator_list=["arima", "sarimax"], + log_file_name="test/ts_forecast.log", + ) + print(automl.predict(X_train[72:])) + # an alternative way to specify predict steps for arima/sarimax + print(automl.predict(12)) def load_multi_dataset(): diff --git a/test/automl/test_training_log.py b/test/automl/test_training_log.py index ac98c495c..ff096ccd0 100644 --- a/test/automl/test_training_log.py +++ b/test/automl/test_training_log.py @@ -9,7 +9,9 @@ from flaml.training_log import training_log_reader class TestTrainingLog(unittest.TestCase): - def test_training_log(self, path="test_training_log.log", estimator_list="auto"): + def test_training_log( + self, path="test_training_log.log", estimator_list="auto", use_ray=False + ): with TemporaryDirectory() as d: filename = os.path.join(d, path) @@ -54,6 +56,7 @@ class TestTrainingLog(unittest.TestCase): estimator_list=[estimator], n_jobs=1, starting_points={estimator: config}, + use_ray=use_ray, ) print(automl.best_config) # then the fitted model should be equivalent to model @@ -99,8 +102,16 @@ class TestTrainingLog(unittest.TestCase): print("PermissionError happens as expected in windows.") def test_each_estimator(self): - self.test_training_log(estimator_list=["xgboost"]) - self.test_training_log(estimator_list=["catboost"]) - self.test_training_log(estimator_list=["extra_tree"]) - self.test_training_log(estimator_list=["rf"]) - self.test_training_log(estimator_list=["lgbm"]) + try: + import ray + + ray.shutdown() + ray.init() + use_ray = True + except ImportError: + use_ray = False + self.test_training_log(estimator_list=["xgboost"], use_ray=use_ray) + self.test_training_log(estimator_list=["catboost"], use_ray=use_ray) + self.test_training_log(estimator_list=["extra_tree"], use_ray=use_ray) + self.test_training_log(estimator_list=["rf"], use_ray=use_ray) + self.test_training_log(estimator_list=["lgbm"], use_ray=use_ray) diff --git a/test/tune/test_searcher.py b/test/tune/test_searcher.py index 9a1e96e80..191df4167 100644 --- a/test/tune/test_searcher.py +++ b/test/tune/test_searcher.py @@ -189,8 +189,15 @@ def test_searcher(): searcher.on_trial_complete("t3", {"m": np.nan}) searcher.save("test/tune/optuna.pickle") searcher.restore("test/tune/optuna.pickle") + try: + searcher = BlendSearch( + metric="m", global_search_alg=searcher, metric_constraints=[("c", "<", 1)] + ) + except AssertionError: + # sign of metric constraints must be <= or >=. + pass searcher = BlendSearch( - metric="m", global_search_alg=searcher, metric_constraints=[("c", "<", 1)] + metric="m", global_search_alg=searcher, metric_constraints=[("c", "<=", 1)] ) searcher.set_search_properties( metric="m2", config=config, setting={"time_budget_s": 0} diff --git a/website/docs/Examples/AutoML-Time series forecast.md b/website/docs/Examples/AutoML-Time series forecast.md index 530d81cc9..e5f4ab2e3 100644 --- a/website/docs/Examples/AutoML-Time series forecast.md +++ b/website/docs/Examples/AutoML-Time series forecast.md @@ -13,84 +13,230 @@ pip install "flaml[ts_forecast]" import numpy as np from flaml import AutoML -X_train = np.arange('2014-01', '2021-01', dtype='datetime64[M]') -y_train = np.random.random(size=72) +X_train = np.arange('2014-01', '2022-01', dtype='datetime64[M]') +y_train = np.random.random(size=84) automl = AutoML() -automl.fit(X_train=X_train[:72], # a single column of timestamp +automl.fit(X_train=X_train[:84], # a single column of timestamp y_train=y_train, # value for each timestamp period=12, # time horizon to forecast, e.g., 12 months task='ts_forecast', time_budget=15, # time budget in seconds log_file_name="ts_forecast.log", + eval_method="holdout", ) -print(automl.predict(X_train[72:])) +print(automl.predict(X_train[84:])) ``` #### Sample output -``` -[flaml.automl: 11-15 18:44:49] {1485} INFO - Data split method: time -INFO:flaml.automl:Data split method: time -[flaml.automl: 11-15 18:44:49] {1489} INFO - Evaluation method: cv -INFO:flaml.automl:Evaluation method: cv -[flaml.automl: 11-15 18:44:49] {1540} INFO - Minimizing error metric: mape -INFO:flaml.automl:Minimizing error metric: mape -[flaml.automl: 11-15 18:44:49] {1577} INFO - List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax'] -INFO:flaml.automl:List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax'] -[flaml.automl: 11-15 18:44:49] {1826} INFO - iteration 0, current learner prophet -INFO:flaml.automl:iteration 0, current learner prophet -[flaml.automl: 11-15 18:45:00] {1944} INFO - Estimated sufficient time budget=104159s. Estimated necessary time budget=104s. -INFO:flaml.automl:Estimated sufficient time budget=104159s. Estimated necessary time budget=104s. -[flaml.automl: 11-15 18:45:00] {2029} INFO - at 10.5s, estimator prophet's best error=1.5681, best estimator prophet's best error=1.5681 -INFO:flaml.automl: at 10.5s, estimator prophet's best error=1.5681, best estimator prophet's best error=1.5681 -[flaml.automl: 11-15 18:45:00] {1826} INFO - iteration 1, current learner arima -INFO:flaml.automl:iteration 1, current learner arima -[flaml.automl: 11-15 18:45:00] {2029} INFO - at 10.7s, estimator arima's best error=2.3515, best estimator prophet's best error=1.5681 -INFO:flaml.automl: at 10.7s, estimator arima's best error=2.3515, best estimator prophet's best error=1.5681 -[flaml.automl: 11-15 18:45:00] {1826} INFO - iteration 2, current learner arima -INFO:flaml.automl:iteration 2, current learner arima -[flaml.automl: 11-15 18:45:01] {2029} INFO - at 11.5s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681 -INFO:flaml.automl: at 11.5s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681 -[flaml.automl: 11-15 18:45:01] {1826} INFO - iteration 3, current learner arima -INFO:flaml.automl:iteration 3, current learner arima -[flaml.automl: 11-15 18:45:01] {2029} INFO - at 11.9s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681 -INFO:flaml.automl: at 11.9s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681 -[flaml.automl: 11-15 18:45:01] {1826} INFO - iteration 4, current learner arima -INFO:flaml.automl:iteration 4, current learner arima -[flaml.automl: 11-15 18:45:02] {2029} INFO - at 12.9s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681 -INFO:flaml.automl: at 12.9s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681 -[flaml.automl: 11-15 18:45:02] {1826} INFO - iteration 5, current learner arima -INFO:flaml.automl:iteration 5, current learner arima -[flaml.automl: 11-15 18:45:04] {2029} INFO - at 14.4s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681 -INFO:flaml.automl: at 14.4s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681 -[flaml.automl: 11-15 18:45:04] {1826} INFO - iteration 6, current learner sarimax -INFO:flaml.automl:iteration 6, current learner sarimax -[flaml.automl: 11-15 18:45:04] {2029} INFO - at 14.7s, estimator sarimax's best error=2.3515, best estimator prophet's best error=1.5681 -INFO:flaml.automl: at 14.7s, estimator sarimax's best error=2.3515, best estimator prophet's best error=1.5681 -[flaml.automl: 11-15 18:45:04] {1826} INFO - iteration 7, current learner sarimax -INFO:flaml.automl:iteration 7, current learner sarimax -[flaml.automl: 11-15 18:45:04] {2029} INFO - at 15.0s, estimator sarimax's best error=1.6371, best estimator prophet's best error=1.5681 -INFO:flaml.automl: at 15.0s, estimator sarimax's best error=1.6371, best estimator prophet's best error=1.5681 -[flaml.automl: 11-15 18:45:05] {2242} INFO - retrain prophet for 0.5s -INFO:flaml.automl:retrain prophet for 0.5s -[flaml.automl: 11-15 18:45:05] {2247} INFO - retrained model: -INFO:flaml.automl:retrained model: -[flaml.automl: 11-15 18:45:05] {1608} INFO - fit succeeded -INFO:flaml.automl:fit succeeded -[flaml.automl: 11-15 18:45:05] {1610} INFO - Time taken to find the best model: 10.450132608413696 -INFO:flaml.automl:Time taken to find the best model: 10.450132608413696 -0 0.384715 -1 0.191349 -2 0.372324 -3 0.814549 -4 0.269616 -5 0.470667 -6 0.603665 -7 0.256773 -8 0.408787 -9 0.663065 -10 0.619943 -11 0.090284 -Name: yhat, dtype: float64 +```python +[flaml.automl: 01-21 08:01:20] {2018} INFO - task = ts_forecast +[flaml.automl: 01-21 08:01:20] {2020} INFO - Data split method: time +[flaml.automl: 01-21 08:01:20] {2024} INFO - Evaluation method: holdout +[flaml.automl: 01-21 08:01:20] {2124} INFO - Minimizing error metric: mape +[flaml.automl: 01-21 08:01:21] {2181} INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'xgboost', 'extra_tree', 'xgb_limitdepth', 'prophet', 'arima', 'sarimax'] +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 0, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2547} INFO - Estimated sufficient time budget=1429s. Estimated necessary time budget=1s. +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 1, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 2, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 3, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 4, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 5, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 6, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9652, best estimator lgbm's best error=0.9652 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 7, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 8, current learner lgbm +[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 9, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 10, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 11, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 12, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 13, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 14, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 15, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 16, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 17, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 18, current learner rf +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator rf's best error=1.0994, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 19, current learner rf +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 20, current learner xgboost +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0271, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 21, current learner rf +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 22, current learner xgboost +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 23, current learner xgboost +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 24, current learner xgboost +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 25, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 26, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 27, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 28, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 29, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 30, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 31, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 32, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 33, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 34, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 35, current learner xgboost +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 36, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 37, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 38, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 39, current learner xgboost +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 40, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 41, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 42, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 43, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 44, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator xgb_limitdepth's best error=1.5815, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 45, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 46, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 47, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 48, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 49, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 50, current learner extra_tree +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 51, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 52, current learner xgboost +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 53, current learner xgboost +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 54, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 55, current learner lgbm +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 56, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 57, current learner rf +[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 58, current learner xgboost +[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 59, current learner extra_tree +[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 60, current learner lgbm +[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 61, current learner extra_tree +[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 62, current learner lgbm +[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 63, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.2s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 64, current learner prophet +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.2s, estimator prophet's best error=1.5706, best estimator lgbm's best error=0.9466 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 65, current learner arima +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.2s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 66, current learner arima +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.4s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 67, current learner sarimax +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.4s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 68, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.5s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 69, current learner sarimax +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 70, current learner sarimax +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 71, current learner arima +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 72, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 73, current learner arima +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.7s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 74, current learner sarimax +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.7s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 75, current learner arima +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.8s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 76, current learner sarimax +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.9s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 77, current learner arima +[flaml.automl: 01-21 08:01:25] {2594} INFO - at 5.0s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 78, current learner sarimax +[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 79, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 80, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 81, current learner sarimax +[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 82, current learner prophet +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator prophet's best error=1.4076, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 83, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 84, current learner sarimax +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 85, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 86, current learner sarimax +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.8s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 87, current learner arima +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.8s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 88, current learner sarimax +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.9s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 89, current learner arima +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.9s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 90, current learner arima +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 91, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 92, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693 +[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 93, current learner sarimax +[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.0s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600 +[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 94, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.1s, estimator xgb_limitdepth's best error=0.9683, best estimator sarimax's best error=0.5600 +[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 95, current learner sarimax +[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600 +[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 96, current learner arima +[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator arima's best error=0.5693, best estimator sarimax's best error=0.5600 +[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 97, current learner arima +[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator arima's best error=0.5693, best estimator sarimax's best error=0.5600 +[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 98, current learner extra_tree +[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator extra_tree's best error=0.9499, best estimator sarimax's best error=0.5600 +[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 99, current learner sarimax +[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600 +[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 100, current learner xgb_limitdepth +[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator xgb_limitdepth's best error=0.9683, best estimator sarimax's best error=0.5600 ``` ### Multivariate time series @@ -131,62 +277,142 @@ automl.fit(dataframe=train_df, # training data #### Sample output ``` -[flaml.automl: 11-15 18:54:12] {1485} INFO - Data split method: time -INFO:flaml.automl:Data split method: time -[flaml.automl: 11-15 18:54:12] {1489} INFO - Evaluation method: holdout -INFO:flaml.automl:Evaluation method: holdout -[flaml.automl: 11-15 18:54:13] {1540} INFO - Minimizing error metric: mape -INFO:flaml.automl:Minimizing error metric: mape -[flaml.automl: 11-15 18:54:13] {1577} INFO - List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax'] -INFO:flaml.automl:List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax'] -[flaml.automl: 11-15 18:54:13] {1826} INFO - iteration 0, current learner prophet -INFO:flaml.automl:iteration 0, current learner prophet -[flaml.automl: 11-15 18:54:15] {1944} INFO - Estimated sufficient time budget=25297s. Estimated necessary time budget=25s. -INFO:flaml.automl:Estimated sufficient time budget=25297s. Estimated necessary time budget=25s. -[flaml.automl: 11-15 18:54:15] {2029} INFO - at 2.6s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 -INFO:flaml.automl: at 2.6s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 -[flaml.automl: 11-15 18:54:15] {1826} INFO - iteration 1, current learner prophet -INFO:flaml.automl:iteration 1, current learner prophet -[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.2s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 -INFO:flaml.automl: at 5.2s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 -[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 2, current learner arima -INFO:flaml.automl:iteration 2, current learner arima -[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.5s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 -INFO:flaml.automl: at 5.5s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 -[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 3, current learner arima -INFO:flaml.automl:iteration 3, current learner arima -[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.6s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 -INFO:flaml.automl: at 5.6s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 -[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 4, current learner prophet -INFO:flaml.automl:iteration 4, current learner prophet -[flaml.automl: 11-15 18:54:21] {2029} INFO - at 8.1s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005 -INFO:flaml.automl: at 8.1s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005 -[flaml.automl: 11-15 18:54:21] {1826} INFO - iteration 5, current learner arima -INFO:flaml.automl:iteration 5, current learner arima -[flaml.automl: 11-15 18:54:21] {2029} INFO - at 8.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005 -INFO:flaml.automl: at 8.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005 -[flaml.automl: 11-15 18:54:21] {1826} INFO - iteration 6, current learner arima -INFO:flaml.automl:iteration 6, current learner arima -[flaml.automl: 11-15 18:54:22] {2029} INFO - at 9.7s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005 -INFO:flaml.automl: at 9.7s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005 -[flaml.automl: 11-15 18:54:22] {1826} INFO - iteration 7, current learner sarimax -INFO:flaml.automl:iteration 7, current learner sarimax -[flaml.automl: 11-15 18:54:23] {2029} INFO - at 10.1s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005 -INFO:flaml.automl: at 10.1s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005 -[flaml.automl: 11-15 18:54:23] {2242} INFO - retrain prophet for 0.9s -INFO:flaml.automl:retrain prophet for 0.9s -[flaml.automl: 11-15 18:54:23] {2247} INFO - retrained model: -INFO:flaml.automl:retrained model: -[flaml.automl: 11-15 18:54:23] {1608} INFO - fit succeeded -INFO:flaml.automl:fit succeeded -[flaml.automl: 11-15 18:54:23] {1610} INFO - Time taken to find the best model: 8.118467330932617 -INFO:flaml.automl:Time taken to find the best model: 8.118467330932617 -[flaml.automl: 11-15 18:54:23] {1624} WARNING - Time taken to find the best model is 81% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget. -WARNING:flaml.automl:Time taken to find the best model is 81% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget. +[flaml.automl: 01-21 07:54:04] {2018} INFO - task = ts_forecast +[flaml.automl: 01-21 07:54:04] {2020} INFO - Data split method: time +[flaml.automl: 01-21 07:54:04] {2024} INFO - Evaluation method: holdout +[flaml.automl: 01-21 07:54:04] {2124} INFO - Minimizing error metric: mape +Importing plotly failed. Interactive plots will not work. +[flaml.automl: 01-21 07:54:04] {2181} INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'xgboost', 'extra_tree', 'xgb_limitdepth', 'prophet', 'arima', 'sarimax'] +[flaml.automl: 01-21 07:54:04] {2434} INFO - iteration 0, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2547} INFO - Estimated sufficient time budget=2145s. Estimated necessary time budget=2s. +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 0.9s, estimator lgbm's best error=0.0621, best estimator lgbm's best error=0.0621 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 1, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0574, best estimator lgbm's best error=0.0574 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 2, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0464, best estimator lgbm's best error=0.0464 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 3, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0464, best estimator lgbm's best error=0.0464 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 4, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0365, best estimator lgbm's best error=0.0365 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 5, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 6, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 7, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 8, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0110, best estimator lgbm's best error=0.0110 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 9, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0110, best estimator lgbm's best error=0.0110 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 10, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0036, best estimator lgbm's best error=0.0036 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 11, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.4s, estimator lgbm's best error=0.0023, best estimator lgbm's best error=0.0023 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 12, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.4s, estimator lgbm's best error=0.0023, best estimator lgbm's best error=0.0023 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 13, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.5s, estimator lgbm's best error=0.0021, best estimator lgbm's best error=0.0021 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 14, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.6s, estimator lgbm's best error=0.0021, best estimator lgbm's best error=0.0021 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 15, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.7s, estimator lgbm's best error=0.0020, best estimator lgbm's best error=0.0020 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 16, current learner lgbm +[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.8s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 17, current learner lgbm +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 1.9s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 18, current learner lgbm +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.0s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 19, current learner lgbm +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 20, current learner rf +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator rf's best error=0.0228, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 21, current learner rf +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator rf's best error=0.0210, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 22, current learner xgboost +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.6738, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 23, current learner xgboost +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.6738, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 24, current learner xgboost +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.1717, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 25, current learner xgboost +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0249, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 26, current learner xgboost +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0249, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 27, current learner xgboost +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0242, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 28, current learner extra_tree +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.4s, estimator extra_tree's best error=0.0245, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 29, current learner extra_tree +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.4s, estimator extra_tree's best error=0.0160, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 30, current learner lgbm +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.5s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 31, current learner lgbm +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 32, current learner rf +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator rf's best error=0.0210, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 33, current learner extra_tree +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator extra_tree's best error=0.0160, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 34, current learner lgbm +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.8s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 35, current learner extra_tree +[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.8s, estimator extra_tree's best error=0.0158, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 36, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.8s, estimator xgb_limitdepth's best error=0.0447, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 37, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.9s, estimator xgb_limitdepth's best error=0.0447, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 38, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.9s, estimator xgb_limitdepth's best error=0.0029, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 39, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.0s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 40, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.1s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 41, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.1s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 42, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.3s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017 +[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 43, current learner prophet +[flaml.automl: 01-21 07:54:09] {2594} INFO - at 5.5s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:09] {2434} INFO - iteration 44, current learner arima +[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.1s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 45, current learner sarimax +[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.4s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 46, current learner lgbm +[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.5s, estimator lgbm's best error=0.0017, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 47, current learner sarimax +[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.6s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 48, current learner sarimax +[flaml.automl: 01-21 07:54:11] {2594} INFO - at 6.9s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 49, current learner arima +[flaml.automl: 01-21 07:54:11] {2594} INFO - at 6.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 50, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.0018, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 51, current learner sarimax +[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.5s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 52, current learner xgboost +[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.6s, estimator xgboost's best error=0.0242, best estimator prophet's best error=0.0008 +[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 53, current learner prophet +[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.3s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005 +[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 54, current learner sarimax +[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.4s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005 +[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 55, current learner xgb_limitdepth +[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.8s, estimator xgb_limitdepth's best error=0.0018, best estimator prophet's best error=0.0005 +[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 56, current learner xgboost +[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.8s, estimator xgboost's best error=0.0242, best estimator prophet's best error=0.0005 +[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 57, current learner lgbm +[flaml.automl: 01-21 07:54:14] {2594} INFO - at 9.9s, estimator lgbm's best error=0.0017, best estimator prophet's best error=0.0005 +[flaml.automl: 01-21 07:54:14] {2434} INFO - iteration 58, current learner rf +[flaml.automl: 01-21 07:54:14] {2594} INFO - at 10.0s, estimator rf's best error=0.0146, best estimator prophet's best error=0.0005 +[flaml.automl: 01-21 07:54:14] {2824} INFO - retrain prophet for 0.6s +[flaml.automl: 01-21 07:54:14] {2831} INFO - retrained model: +[flaml.automl: 01-21 07:54:14] {2210} INFO - fit succeeded +[flaml.automl: 01-21 07:54:14] {2211} INFO - Time taken to find the best model: 9.339771270751953 +[flaml.automl: 01-21 07:54:14] {2222} WARNING - Time taken to find the best model is 93% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget. ``` #### Compute and plot predictions +The example plotting code requires matplotlib. + ```python flaml_y_pred = automl.predict(X_test) import matplotlib.pyplot as plt