yujunjun/autogen
1
0
Fork 0
mirror of https://github.com/microsoft/autogen.git synced 2025-11-11 23:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

max_iter < 2 -> no search; sign in metric constraints; test and example for forecasting (#415)

Browse Source 
* max_iter < 2 -> no search

* use_ray in test

* eval_method in ts example

* check sign of constraints

* test metric constraint sign
This commit is contained in:
Chi Wang 2022-01-23 01:24:15 -08:00 committed by GitHub
parent 38ad31ea25
commit 6a7caa6a3d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 412 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
flaml/automl.py
View File

@ -2424,20 +2424,6 @@ class AutoML(BaseEstimator):
est_retrain_time = next_trial_time = 0 est_retrain_time = next_trial_time = 0
best_config_sig = None best_config_sig = None
better = True # whether we find a better model in one trial better = True # whether we find a better model in one trial
if self._ensemble:
self.best_model = {}
if self._max_iter < 2 and self.estimator_list and self._state.retrain_final:
# when max_iter is 1, no need to search
# TODO: otherwise, need to make sure SearchStates.init_config is inside search space
self._max_iter = 0
self._best_estimator = estimator = self.estimator_list[0]
self._selected = state = self._search_states[estimator]
state.best_config_sample_size = self._state.data_size[0]
state.best_config = (
state.init_config
if isinstance(state.init_config, dict)
else state.init_config[0]
)
for self._track_iter in range(self._max_iter): for self._track_iter in range(self._max_iter):
if self._estimator_index is None: if self._estimator_index is None:
estimator = self._active_estimators[0] estimator = self._active_estimators[0]
@ -2699,8 +2685,20 @@ class AutoML(BaseEstimator):
self._warn_threshold = 10 self._warn_threshold = 10
self._selected = None self._selected = None
self.modelcount = 0 self.modelcount = 0
if self._max_iter < 2 and self.estimator_list and self._state.retrain_final:
if not self._use_ray: # when max_iter is 1, no need to search
# TODO: otherwise, need to make sure SearchStates.init_config is inside search space
self.modelcount = self._max_iter
self._max_iter = 0
self._best_estimator = estimator = self.estimator_list[0]
self._selected = state = self._search_states[estimator]
state.best_config_sample_size = self._state.data_size[0]
state.best_config = (
state.init_config
if isinstance(state.init_config, dict)
else state.init_config[0]
)
elif not self._use_ray:
self._search_sequential() self._search_sequential()
else: else:
self._search_parallel() self._search_parallel()

6
flaml/searcher/blendsearch.py
View File

@ -130,7 +130,10 @@ class BlendSearch(Searcher):
self._evaluated_rewards = evaluated_rewards or [] self._evaluated_rewards = evaluated_rewards or []
self._config_constraints = config_constraints self._config_constraints = config_constraints
self._metric_constraints = metric_constraints self._metric_constraints = metric_constraints
if self._metric_constraints: if metric_constraints:
assert all(
x[1] in ["<=", ">="] for x in metric_constraints
), "sign of metric constraints must be <= or >=."
# metric modified by lagrange # metric modified by lagrange
metric += self.lagrange metric += self.lagrange
self._cat_hp_cost = cat_hp_cost or {} self._cat_hp_cost = cat_hp_cost or {}
@ -348,7 +351,6 @@ class BlendSearch(Searcher):
metric_constraint, sign, threshold = constraint metric_constraint, sign, threshold = constraint
value = result.get(metric_constraint) value = result.get(metric_constraint)
if value: if value:
# sign is <= or >=
sign_op = 1 if sign == "<=" else -1 sign_op = 1 if sign == "<=" else -1
violation = (value - threshold) * sign_op violation = (value - threshold) * sign_op
if violation > 0: if violation > 0:

6
test/automl/test_forecast.py
View File

@ -95,9 +95,6 @@ def test_numpy():
X_train = np.arange("2014-01", "2021-01", dtype="datetime64[M]") X_train = np.arange("2014-01", "2021-01", dtype="datetime64[M]")
y_train = np.random.random(size=len(X_train)) y_train = np.random.random(size=len(X_train))
automl = AutoML() automl = AutoML()
try:
import prophet
automl.fit( automl.fit(
X_train=X_train[:72], # a single column of timestamp X_train=X_train[:72], # a single column of timestamp
y_train=y_train[:72], # value for each timestamp y_train=y_train[:72], # value for each timestamp
@ -108,8 +105,7 @@ def test_numpy():
n_splits=3, # number of splits n_splits=3, # number of splits
) )
print(automl.predict(X_train[72:])) print(automl.predict(X_train[72:]))
except ImportError:
print("not using prophet due to ImportError")
automl = AutoML() automl = AutoML()
automl.fit( automl.fit(
X_train=X_train[:72], # a single column of timestamp X_train=X_train[:72], # a single column of timestamp

23
test/automl/test_training_log.py
View File

@ -9,7 +9,9 @@ from flaml.training_log import training_log_reader
class TestTrainingLog(unittest.TestCase): class TestTrainingLog(unittest.TestCase):
def test_training_log(self, path="test_training_log.log", estimator_list="auto"): def test_training_log(
self, path="test_training_log.log", estimator_list="auto", use_ray=False
):
with TemporaryDirectory() as d: with TemporaryDirectory() as d:
filename = os.path.join(d, path) filename = os.path.join(d, path)
@ -54,6 +56,7 @@ class TestTrainingLog(unittest.TestCase):
estimator_list=[estimator], estimator_list=[estimator],
n_jobs=1, n_jobs=1,
starting_points={estimator: config}, starting_points={estimator: config},
use_ray=use_ray,
) )
print(automl.best_config) print(automl.best_config)
# then the fitted model should be equivalent to model # then the fitted model should be equivalent to model
@ -99,8 +102,16 @@ class TestTrainingLog(unittest.TestCase):
print("PermissionError happens as expected in windows.") print("PermissionError happens as expected in windows.")
def test_each_estimator(self): def test_each_estimator(self):
self.test_training_log(estimator_list=["xgboost"]) try:
self.test_training_log(estimator_list=["catboost"]) import ray
self.test_training_log(estimator_list=["extra_tree"])
self.test_training_log(estimator_list=["rf"]) ray.shutdown()
self.test_training_log(estimator_list=["lgbm"]) ray.init()
use_ray = True
except ImportError:
use_ray = False
self.test_training_log(estimator_list=["xgboost"], use_ray=use_ray)
self.test_training_log(estimator_list=["catboost"], use_ray=use_ray)
self.test_training_log(estimator_list=["extra_tree"], use_ray=use_ray)
self.test_training_log(estimator_list=["rf"], use_ray=use_ray)
self.test_training_log(estimator_list=["lgbm"], use_ray=use_ray)

7
test/tune/test_searcher.py
View File

@ -189,9 +189,16 @@ def test_searcher():
searcher.on_trial_complete("t3", {"m": np.nan}) searcher.on_trial_complete("t3", {"m": np.nan})
searcher.save("test/tune/optuna.pickle") searcher.save("test/tune/optuna.pickle")
searcher.restore("test/tune/optuna.pickle") searcher.restore("test/tune/optuna.pickle")
try:
searcher = BlendSearch( searcher = BlendSearch(
metric="m", global_search_alg=searcher, metric_constraints=[("c", "<", 1)] metric="m", global_search_alg=searcher, metric_constraints=[("c", "<", 1)]
) )
except AssertionError:
# sign of metric constraints must be <= or >=.
pass
searcher = BlendSearch(
metric="m", global_search_alg=searcher, metric_constraints=[("c", "<=", 1)]
)
searcher.set_search_properties( searcher.set_search_properties(
metric="m2", config=config, setting={"time_budget_s": 0} metric="m2", config=config, setting={"time_budget_s": 0}
) )

466
website/docs/Examples/AutoML-Time series forecast.md
View File

@ -13,84 +13,230 @@ pip install "flaml[ts_forecast]"
import numpy as np import numpy as np
from flaml import AutoML from flaml import AutoML
X_train = np.arange('2014-01', '2021-01', dtype='datetime64[M]') X_train = np.arange('2014-01', '2022-01', dtype='datetime64[M]')
y_train = np.random.random(size=72) y_train = np.random.random(size=84)
automl = AutoML() automl = AutoML()
automl.fit(X_train=X_train[:72], # a single column of timestamp automl.fit(X_train=X_train[:84], # a single column of timestamp
y_train=y_train, # value for each timestamp y_train=y_train, # value for each timestamp
period=12, # time horizon to forecast, e.g., 12 months period=12, # time horizon to forecast, e.g., 12 months
task='ts_forecast', time_budget=15, # time budget in seconds task='ts_forecast', time_budget=15, # time budget in seconds
log_file_name="ts_forecast.log", log_file_name="ts_forecast.log",
eval_method="holdout",
) )
print(automl.predict(X_train[72:])) print(automl.predict(X_train[84:]))
``` ```
#### Sample output #### Sample output
``` ```python
[flaml.automl: 11-15 18:44:49] {1485} INFO - Data split method: time [flaml.automl: 01-21 08:01:20] {2018} INFO - task = ts_forecast
INFO:flaml.automl:Data split method: time [flaml.automl: 01-21 08:01:20] {2020} INFO - Data split method: time
[flaml.automl: 11-15 18:44:49] {1489} INFO - Evaluation method: cv [flaml.automl: 01-21 08:01:20] {2024} INFO - Evaluation method: holdout
INFO:flaml.automl:Evaluation method: cv [flaml.automl: 01-21 08:01:20] {2124} INFO - Minimizing error metric: mape
[flaml.automl: 11-15 18:44:49] {1540} INFO - Minimizing error metric: mape [flaml.automl: 01-21 08:01:21] {2181} INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'xgboost', 'extra_tree', 'xgb_limitdepth', 'prophet', 'arima', 'sarimax']
INFO:flaml.automl:Minimizing error metric: mape [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 0, current learner lgbm
[flaml.automl: 11-15 18:44:49] {1577} INFO - List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax'] [flaml.automl: 01-21 08:01:21] {2547} INFO - Estimated sufficient time budget=1429s. Estimated necessary time budget=1s.
INFO:flaml.automl:List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax'] [flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
[flaml.automl: 11-15 18:44:49] {1826} INFO - iteration 0, current learner prophet [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 1, current learner lgbm
INFO:flaml.automl:iteration 0, current learner prophet [flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
[flaml.automl: 11-15 18:45:00] {1944} INFO - Estimated sufficient time budget=104159s. Estimated necessary time budget=104s. [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 2, current learner lgbm
INFO:flaml.automl:Estimated sufficient time budget=104159s. Estimated necessary time budget=104s. [flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
[flaml.automl: 11-15 18:45:00] {2029} INFO - at 10.5s, estimator prophet's best error=1.5681, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 3, current learner lgbm
INFO:flaml.automl: at 10.5s, estimator prophet's best error=1.5681, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
[flaml.automl: 11-15 18:45:00] {1826} INFO - iteration 1, current learner arima [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 4, current learner lgbm
INFO:flaml.automl:iteration 1, current learner arima [flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
[flaml.automl: 11-15 18:45:00] {2029} INFO - at 10.7s, estimator arima's best error=2.3515, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 5, current learner lgbm
INFO:flaml.automl: at 10.7s, estimator arima's best error=2.3515, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
[flaml.automl: 11-15 18:45:00] {1826} INFO - iteration 2, current learner arima [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 6, current learner lgbm
INFO:flaml.automl:iteration 2, current learner arima [flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9652, best estimator lgbm's best error=0.9652
[flaml.automl: 11-15 18:45:01] {2029} INFO - at 11.5s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 7, current learner lgbm
INFO:flaml.automl: at 11.5s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:01] {1826} INFO - iteration 3, current learner arima [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 8, current learner lgbm
INFO:flaml.automl:iteration 3, current learner arima [flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:01] {2029} INFO - at 11.9s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 9, current learner lgbm
INFO:flaml.automl: at 11.9s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:01] {1826} INFO - iteration 4, current learner arima [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 10, current learner lgbm
INFO:flaml.automl:iteration 4, current learner arima [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:02] {2029} INFO - at 12.9s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 11, current learner lgbm
INFO:flaml.automl: at 12.9s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:02] {1826} INFO - iteration 5, current learner arima [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 12, current learner lgbm
INFO:flaml.automl:iteration 5, current learner arima [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:04] {2029} INFO - at 14.4s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 13, current learner lgbm
INFO:flaml.automl: at 14.4s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:04] {1826} INFO - iteration 6, current learner sarimax [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 14, current learner lgbm
INFO:flaml.automl:iteration 6, current learner sarimax [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:04] {2029} INFO - at 14.7s, estimator sarimax's best error=2.3515, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 15, current learner lgbm
INFO:flaml.automl: at 14.7s, estimator sarimax's best error=2.3515, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:04] {1826} INFO - iteration 7, current learner sarimax [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 16, current learner lgbm
INFO:flaml.automl:iteration 7, current learner sarimax [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:04] {2029} INFO - at 15.0s, estimator sarimax's best error=1.6371, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 17, current learner lgbm
INFO:flaml.automl: at 15.0s, estimator sarimax's best error=1.6371, best estimator prophet's best error=1.5681 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:05] {2242} INFO - retrain prophet for 0.5s [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 18, current learner rf
INFO:flaml.automl:retrain prophet for 0.5s [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator rf's best error=1.0994, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:05] {2247} INFO - retrained model: <prophet.forecaster.Prophet object at 0x7f042ba1da50> [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 19, current learner rf
INFO:flaml.automl:retrained model: <prophet.forecaster.Prophet object at 0x7f042ba1da50> [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:05] {1608} INFO - fit succeeded [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 20, current learner xgboost
INFO:flaml.automl:fit succeeded [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0271, best estimator lgbm's best error=0.9466
[flaml.automl: 11-15 18:45:05] {1610} INFO - Time taken to find the best model: 10.450132608413696 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 21, current learner rf
INFO:flaml.automl:Time taken to find the best model: 10.450132608413696 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466
0 0.384715 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 22, current learner xgboost
1 0.191349 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
2 0.372324 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 23, current learner xgboost
3 0.814549 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
4 0.269616 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 24, current learner xgboost
5 0.470667 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
6 0.603665 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 25, current learner extra_tree
7 0.256773 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466
8 0.408787 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 26, current learner extra_tree
9 0.663065 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466
10 0.619943 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 27, current learner extra_tree
11 0.090284 [flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466
Name: yhat, dtype: float64 [flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 28, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 29, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 30, current learner lgbm
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 31, current learner lgbm
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 32, current learner lgbm
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 33, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 34, current learner lgbm
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 35, current learner xgboost
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 36, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 37, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 38, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 39, current learner xgboost
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 40, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 41, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 42, current learner lgbm
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 43, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 44, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator xgb_limitdepth's best error=1.5815, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 45, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 46, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 47, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 48, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 49, current learner lgbm
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 50, current learner extra_tree
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 51, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 52, current learner xgboost
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 53, current learner xgboost
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 54, current learner lgbm
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 55, current learner lgbm
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 56, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 57, current learner rf
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 58, current learner xgboost
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 59, current learner extra_tree
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 60, current learner lgbm
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 61, current learner extra_tree
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 62, current learner lgbm
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 63, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.2s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 64, current learner prophet
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.2s, estimator prophet's best error=1.5706, best estimator lgbm's best error=0.9466
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 65, current learner arima
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.2s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 66, current learner arima
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.4s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 67, current learner sarimax
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.4s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 68, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.5s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 69, current learner sarimax
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 70, current learner sarimax
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 71, current learner arima
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 72, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 73, current learner arima
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.7s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 74, current learner sarimax
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.7s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 75, current learner arima
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.8s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 76, current learner sarimax
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.9s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 77, current learner arima
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 5.0s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 78, current learner sarimax
[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 79, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 80, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 81, current learner sarimax
[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 82, current learner prophet
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator prophet's best error=1.4076, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 83, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 84, current learner sarimax
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 85, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 86, current learner sarimax
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.8s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 87, current learner arima
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.8s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 88, current learner sarimax
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.9s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 89, current learner arima
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.9s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 90, current learner arima
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 91, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 92, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 93, current learner sarimax
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.0s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 94, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.1s, estimator xgb_limitdepth's best error=0.9683, best estimator sarimax's best error=0.5600
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 95, current learner sarimax
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 96, current learner arima
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator arima's best error=0.5693, best estimator sarimax's best error=0.5600
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 97, current learner arima
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator arima's best error=0.5693, best estimator sarimax's best error=0.5600
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 98, current learner extra_tree
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator extra_tree's best error=0.9499, best estimator sarimax's best error=0.5600
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 99, current learner sarimax
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 100, current learner xgb_limitdepth
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator xgb_limitdepth's best error=0.9683, best estimator sarimax's best error=0.5600
``` ```
### Multivariate time series ### Multivariate time series
@ -131,62 +277,142 @@ automl.fit(dataframe=train_df, # training data
#### Sample output #### Sample output
``` ```
[flaml.automl: 11-15 18:54:12] {1485} INFO - Data split method: time [flaml.automl: 01-21 07:54:04] {2018} INFO - task = ts_forecast
INFO:flaml.automl:Data split method: time [flaml.automl: 01-21 07:54:04] {2020} INFO - Data split method: time
[flaml.automl: 11-15 18:54:12] {1489} INFO - Evaluation method: holdout [flaml.automl: 01-21 07:54:04] {2024} INFO - Evaluation method: holdout
INFO:flaml.automl:Evaluation method: holdout [flaml.automl: 01-21 07:54:04] {2124} INFO - Minimizing error metric: mape
[flaml.automl: 11-15 18:54:13] {1540} INFO - Minimizing error metric: mape Importing plotly failed. Interactive plots will not work.
INFO:flaml.automl:Minimizing error metric: mape [flaml.automl: 01-21 07:54:04] {2181} INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'xgboost', 'extra_tree', 'xgb_limitdepth', 'prophet', 'arima', 'sarimax']
[flaml.automl: 11-15 18:54:13] {1577} INFO - List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax'] [flaml.automl: 01-21 07:54:04] {2434} INFO - iteration 0, current learner lgbm
INFO:flaml.automl:List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax'] [flaml.automl: 01-21 07:54:05] {2547} INFO - Estimated sufficient time budget=2145s. Estimated necessary time budget=2s.
[flaml.automl: 11-15 18:54:13] {1826} INFO - iteration 0, current learner prophet [flaml.automl: 01-21 07:54:05] {2594} INFO - at 0.9s, estimator lgbm's best error=0.0621, best estimator lgbm's best error=0.0621
INFO:flaml.automl:iteration 0, current learner prophet [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 1, current learner lgbm
[flaml.automl: 11-15 18:54:15] {1944} INFO - Estimated sufficient time budget=25297s. Estimated necessary time budget=25s. [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0574, best estimator lgbm's best error=0.0574
INFO:flaml.automl:Estimated sufficient time budget=25297s. Estimated necessary time budget=25s. [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 2, current learner lgbm
[flaml.automl: 11-15 18:54:15] {2029} INFO - at 2.6s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0464, best estimator lgbm's best error=0.0464
INFO:flaml.automl: at 2.6s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 3, current learner lgbm
[flaml.automl: 11-15 18:54:15] {1826} INFO - iteration 1, current learner prophet [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0464, best estimator lgbm's best error=0.0464
INFO:flaml.automl:iteration 1, current learner prophet [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 4, current learner lgbm
[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.2s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0365, best estimator lgbm's best error=0.0365
INFO:flaml.automl: at 5.2s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008 [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 5, current learner lgbm
[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 2, current learner arima [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192
INFO:flaml.automl:iteration 2, current learner arima [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 6, current learner lgbm
[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.5s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192
INFO:flaml.automl: at 5.5s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 7, current learner lgbm
[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 3, current learner arima [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192
INFO:flaml.automl:iteration 3, current learner arima [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 8, current learner lgbm
[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.6s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0110, best estimator lgbm's best error=0.0110
INFO:flaml.automl: at 5.6s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008 [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 9, current learner lgbm
[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 4, current learner prophet [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0110, best estimator lgbm's best error=0.0110
INFO:flaml.automl:iteration 4, current learner prophet [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 10, current learner lgbm
[flaml.automl: 11-15 18:54:21] {2029} INFO - at 8.1s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005 [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0036, best estimator lgbm's best error=0.0036
INFO:flaml.automl: at 8.1s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005 [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 11, current learner lgbm
[flaml.automl: 11-15 18:54:21] {1826} INFO - iteration 5, current learner arima [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.4s, estimator lgbm's best error=0.0023, best estimator lgbm's best error=0.0023
INFO:flaml.automl:iteration 5, current learner arima [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 12, current learner lgbm
[flaml.automl: 11-15 18:54:21] {2029} INFO - at 8.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005 [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.4s, estimator lgbm's best error=0.0023, best estimator lgbm's best error=0.0023
INFO:flaml.automl: at 8.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005 [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 13, current learner lgbm
[flaml.automl: 11-15 18:54:21] {1826} INFO - iteration 6, current learner arima [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.5s, estimator lgbm's best error=0.0021, best estimator lgbm's best error=0.0021
INFO:flaml.automl:iteration 6, current learner arima [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 14, current learner lgbm
[flaml.automl: 11-15 18:54:22] {2029} INFO - at 9.7s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005 [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.6s, estimator lgbm's best error=0.0021, best estimator lgbm's best error=0.0021
INFO:flaml.automl: at 9.7s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005 [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 15, current learner lgbm
[flaml.automl: 11-15 18:54:22] {1826} INFO - iteration 7, current learner sarimax [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.7s, estimator lgbm's best error=0.0020, best estimator lgbm's best error=0.0020
INFO:flaml.automl:iteration 7, current learner sarimax [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 16, current learner lgbm
[flaml.automl: 11-15 18:54:23] {2029} INFO - at 10.1s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005 [flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.8s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
INFO:flaml.automl: at 10.1s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005 [flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 17, current learner lgbm
[flaml.automl: 11-15 18:54:23] {2242} INFO - retrain prophet for 0.9s [flaml.automl: 01-21 07:54:06] {2594} INFO - at 1.9s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
INFO:flaml.automl:retrain prophet for 0.9s [flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 18, current learner lgbm
[flaml.automl: 11-15 18:54:23] {2247} INFO - retrained model: <prophet.forecaster.Prophet object at 0x7f0418e21f50> [flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.0s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
INFO:flaml.automl:retrained model: <prophet.forecaster.Prophet object at 0x7f0418e21f50> [flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 19, current learner lgbm
[flaml.automl: 11-15 18:54:23] {1608} INFO - fit succeeded [flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
INFO:flaml.automl:fit succeeded [flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 20, current learner rf
[flaml.automl: 11-15 18:54:23] {1610} INFO - Time taken to find the best model: 8.118467330932617 [flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator rf's best error=0.0228, best estimator lgbm's best error=0.0017
INFO:flaml.automl:Time taken to find the best model: 8.118467330932617 [flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 21, current learner rf
[flaml.automl: 11-15 18:54:23] {1624} WARNING - Time taken to find the best model is 81% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget. [flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator rf's best error=0.0210, best estimator lgbm's best error=0.0017
WARNING:flaml.automl:Time taken to find the best model is 81% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget. [flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 22, current learner xgboost
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.6738, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 23, current learner xgboost
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.6738, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 24, current learner xgboost
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.1717, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 25, current learner xgboost
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0249, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 26, current learner xgboost
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0249, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 27, current learner xgboost
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0242, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 28, current learner extra_tree
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.4s, estimator extra_tree's best error=0.0245, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 29, current learner extra_tree
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.4s, estimator extra_tree's best error=0.0160, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 30, current learner lgbm
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.5s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 31, current learner lgbm
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 32, current learner rf
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator rf's best error=0.0210, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 33, current learner extra_tree
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator extra_tree's best error=0.0160, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 34, current learner lgbm
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.8s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 35, current learner extra_tree
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.8s, estimator extra_tree's best error=0.0158, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 36, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.8s, estimator xgb_limitdepth's best error=0.0447, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 37, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.9s, estimator xgb_limitdepth's best error=0.0447, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 38, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.9s, estimator xgb_limitdepth's best error=0.0029, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 39, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.0s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 40, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.1s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 41, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.1s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 42, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.3s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 43, current learner prophet
[flaml.automl: 01-21 07:54:09] {2594} INFO - at 5.5s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:09] {2434} INFO - iteration 44, current learner arima
[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.1s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 45, current learner sarimax
[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.4s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 46, current learner lgbm
[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.5s, estimator lgbm's best error=0.0017, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 47, current learner sarimax
[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.6s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 48, current learner sarimax
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 6.9s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 49, current learner arima
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 6.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 50, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.0018, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 51, current learner sarimax
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.5s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 52, current learner xgboost
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.6s, estimator xgboost's best error=0.0242, best estimator prophet's best error=0.0008
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 53, current learner prophet
[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.3s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005
[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 54, current learner sarimax
[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.4s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005
[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 55, current learner xgb_limitdepth
[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.8s, estimator xgb_limitdepth's best error=0.0018, best estimator prophet's best error=0.0005
[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 56, current learner xgboost
[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.8s, estimator xgboost's best error=0.0242, best estimator prophet's best error=0.0005
[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 57, current learner lgbm
[flaml.automl: 01-21 07:54:14] {2594} INFO - at 9.9s, estimator lgbm's best error=0.0017, best estimator prophet's best error=0.0005
[flaml.automl: 01-21 07:54:14] {2434} INFO - iteration 58, current learner rf
[flaml.automl: 01-21 07:54:14] {2594} INFO - at 10.0s, estimator rf's best error=0.0146, best estimator prophet's best error=0.0005
[flaml.automl: 01-21 07:54:14] {2824} INFO - retrain prophet for 0.6s
[flaml.automl: 01-21 07:54:14] {2831} INFO - retrained model: <prophet.forecaster.Prophet object at 0x7fb68ea65d60>
[flaml.automl: 01-21 07:54:14] {2210} INFO - fit succeeded
[flaml.automl: 01-21 07:54:14] {2211} INFO - Time taken to find the best model: 9.339771270751953
[flaml.automl: 01-21 07:54:14] {2222} WARNING - Time taken to find the best model is 93% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.
``` ```
#### Compute and plot predictions #### Compute and plot predictions
The example plotting code requires matplotlib.
```python ```python
flaml_y_pred = automl.predict(X_test) flaml_y_pred = automl.predict(X_test)
import matplotlib.pyplot as plt import matplotlib.pyplot as plt