mirror of
https://github.com/microsoft/autogen.git
synced 2025-11-01 02:09:58 +00:00
max_iter < 2 -> no search; sign in metric constraints; test and example for forecasting (#415)
* max_iter < 2 -> no search * use_ray in test * eval_method in ts example * check sign of constraints * test metric constraint sign
This commit is contained in:
Chi Wang
GitHub
parent
38ad31ea25
commit
6a7caa6a3d
@ -2424,20 +2424,6 @@ class AutoML(BaseEstimator):
|
||||
est_retrain_time = next_trial_time = 0
|
||||
best_config_sig = None
|
||||
better = True # whether we find a better model in one trial
|
||||
if self._ensemble:
|
||||
self.best_model = {}
|
||||
if self._max_iter < 2 and self.estimator_list and self._state.retrain_final:
|
||||
# when max_iter is 1, no need to search
|
||||
# TODO: otherwise, need to make sure SearchStates.init_config is inside search space
|
||||
self._max_iter = 0
|
||||
self._best_estimator = estimator = self.estimator_list[0]
|
||||
self._selected = state = self._search_states[estimator]
|
||||
state.best_config_sample_size = self._state.data_size[0]
|
||||
state.best_config = (
|
||||
state.init_config
|
||||
if isinstance(state.init_config, dict)
|
||||
else state.init_config[0]
|
||||
)
|
||||
for self._track_iter in range(self._max_iter):
|
||||
if self._estimator_index is None:
|
||||
estimator = self._active_estimators[0]
|
||||
@ -2699,8 +2685,20 @@ class AutoML(BaseEstimator):
|
||||
self._warn_threshold = 10
|
||||
self._selected = None
|
||||
self.modelcount = 0
|
||||
|
||||
if not self._use_ray:
|
||||
if self._max_iter < 2 and self.estimator_list and self._state.retrain_final:
|
||||
# when max_iter is 1, no need to search
|
||||
# TODO: otherwise, need to make sure SearchStates.init_config is inside search space
|
||||
self.modelcount = self._max_iter
|
||||
self._max_iter = 0
|
||||
self._best_estimator = estimator = self.estimator_list[0]
|
||||
self._selected = state = self._search_states[estimator]
|
||||
state.best_config_sample_size = self._state.data_size[0]
|
||||
state.best_config = (
|
||||
state.init_config
|
||||
if isinstance(state.init_config, dict)
|
||||
else state.init_config[0]
|
||||
)
|
||||
elif not self._use_ray:
|
||||
self._search_sequential()
|
||||
else:
|
||||
self._search_parallel()
|
||||
|
||||
@ -130,7 +130,10 @@ class BlendSearch(Searcher):
|
||||
self._evaluated_rewards = evaluated_rewards or []
|
||||
self._config_constraints = config_constraints
|
||||
self._metric_constraints = metric_constraints
|
||||
if self._metric_constraints:
|
||||
if metric_constraints:
|
||||
assert all(
|
||||
x[1] in ["<=", ">="] for x in metric_constraints
|
||||
), "sign of metric constraints must be <= or >=."
|
||||
# metric modified by lagrange
|
||||
metric += self.lagrange
|
||||
self._cat_hp_cost = cat_hp_cost or {}
|
||||
@ -348,7 +351,6 @@ class BlendSearch(Searcher):
|
||||
metric_constraint, sign, threshold = constraint
|
||||
value = result.get(metric_constraint)
|
||||
if value:
|
||||
# sign is <= or >=
|
||||
sign_op = 1 if sign == "<=" else -1
|
||||
violation = (value - threshold) * sign_op
|
||||
if violation > 0:
|
||||
|
||||
@ -95,34 +95,30 @@ def test_numpy():
|
||||
X_train = np.arange("2014-01", "2021-01", dtype="datetime64[M]")
|
||||
y_train = np.random.random(size=len(X_train))
|
||||
automl = AutoML()
|
||||
try:
|
||||
import prophet
|
||||
automl.fit(
|
||||
X_train=X_train[:72], # a single column of timestamp
|
||||
y_train=y_train[:72], # value for each timestamp
|
||||
period=12, # time horizon to forecast, e.g., 12 months
|
||||
task="ts_forecast",
|
||||
time_budget=3, # time budget in seconds
|
||||
log_file_name="test/ts_forecast.log",
|
||||
n_splits=3, # number of splits
|
||||
)
|
||||
print(automl.predict(X_train[72:]))
|
||||
|
||||
automl.fit(
|
||||
X_train=X_train[:72], # a single column of timestamp
|
||||
y_train=y_train[:72], # value for each timestamp
|
||||
period=12, # time horizon to forecast, e.g., 12 months
|
||||
task="ts_forecast",
|
||||
time_budget=3, # time budget in seconds
|
||||
log_file_name="test/ts_forecast.log",
|
||||
n_splits=3, # number of splits
|
||||
)
|
||||
print(automl.predict(X_train[72:]))
|
||||
except ImportError:
|
||||
print("not using prophet due to ImportError")
|
||||
automl = AutoML()
|
||||
automl.fit(
|
||||
X_train=X_train[:72], # a single column of timestamp
|
||||
y_train=y_train[:72], # value for each timestamp
|
||||
period=12, # time horizon to forecast, e.g., 12 months
|
||||
task="ts_forecast",
|
||||
time_budget=1, # time budget in seconds
|
||||
estimator_list=["arima", "sarimax"],
|
||||
log_file_name="test/ts_forecast.log",
|
||||
)
|
||||
print(automl.predict(X_train[72:]))
|
||||
# an alternative way to specify predict steps for arima/sarimax
|
||||
print(automl.predict(12))
|
||||
automl = AutoML()
|
||||
automl.fit(
|
||||
X_train=X_train[:72], # a single column of timestamp
|
||||
y_train=y_train[:72], # value for each timestamp
|
||||
period=12, # time horizon to forecast, e.g., 12 months
|
||||
task="ts_forecast",
|
||||
time_budget=1, # time budget in seconds
|
||||
estimator_list=["arima", "sarimax"],
|
||||
log_file_name="test/ts_forecast.log",
|
||||
)
|
||||
print(automl.predict(X_train[72:]))
|
||||
# an alternative way to specify predict steps for arima/sarimax
|
||||
print(automl.predict(12))
|
||||
|
||||
|
||||
def load_multi_dataset():
|
||||
|
||||
@ -9,7 +9,9 @@ from flaml.training_log import training_log_reader
|
||||
|
||||
|
||||
class TestTrainingLog(unittest.TestCase):
|
||||
def test_training_log(self, path="test_training_log.log", estimator_list="auto"):
|
||||
def test_training_log(
|
||||
self, path="test_training_log.log", estimator_list="auto", use_ray=False
|
||||
):
|
||||
|
||||
with TemporaryDirectory() as d:
|
||||
filename = os.path.join(d, path)
|
||||
@ -54,6 +56,7 @@ class TestTrainingLog(unittest.TestCase):
|
||||
estimator_list=[estimator],
|
||||
n_jobs=1,
|
||||
starting_points={estimator: config},
|
||||
use_ray=use_ray,
|
||||
)
|
||||
print(automl.best_config)
|
||||
# then the fitted model should be equivalent to model
|
||||
@ -99,8 +102,16 @@ class TestTrainingLog(unittest.TestCase):
|
||||
print("PermissionError happens as expected in windows.")
|
||||
|
||||
def test_each_estimator(self):
|
||||
self.test_training_log(estimator_list=["xgboost"])
|
||||
self.test_training_log(estimator_list=["catboost"])
|
||||
self.test_training_log(estimator_list=["extra_tree"])
|
||||
self.test_training_log(estimator_list=["rf"])
|
||||
self.test_training_log(estimator_list=["lgbm"])
|
||||
try:
|
||||
import ray
|
||||
|
||||
ray.shutdown()
|
||||
ray.init()
|
||||
use_ray = True
|
||||
except ImportError:
|
||||
use_ray = False
|
||||
self.test_training_log(estimator_list=["xgboost"], use_ray=use_ray)
|
||||
self.test_training_log(estimator_list=["catboost"], use_ray=use_ray)
|
||||
self.test_training_log(estimator_list=["extra_tree"], use_ray=use_ray)
|
||||
self.test_training_log(estimator_list=["rf"], use_ray=use_ray)
|
||||
self.test_training_log(estimator_list=["lgbm"], use_ray=use_ray)
|
||||
|
||||
@ -189,8 +189,15 @@ def test_searcher():
|
||||
searcher.on_trial_complete("t3", {"m": np.nan})
|
||||
searcher.save("test/tune/optuna.pickle")
|
||||
searcher.restore("test/tune/optuna.pickle")
|
||||
try:
|
||||
searcher = BlendSearch(
|
||||
metric="m", global_search_alg=searcher, metric_constraints=[("c", "<", 1)]
|
||||
)
|
||||
except AssertionError:
|
||||
# sign of metric constraints must be <= or >=.
|
||||
pass
|
||||
searcher = BlendSearch(
|
||||
metric="m", global_search_alg=searcher, metric_constraints=[("c", "<", 1)]
|
||||
metric="m", global_search_alg=searcher, metric_constraints=[("c", "<=", 1)]
|
||||
)
|
||||
searcher.set_search_properties(
|
||||
metric="m2", config=config, setting={"time_budget_s": 0}
|
||||
|
||||
@ -13,84 +13,230 @@ pip install "flaml[ts_forecast]"
|
||||
import numpy as np
|
||||
from flaml import AutoML
|
||||
|
||||
X_train = np.arange('2014-01', '2021-01', dtype='datetime64[M]')
|
||||
y_train = np.random.random(size=72)
|
||||
X_train = np.arange('2014-01', '2022-01', dtype='datetime64[M]')
|
||||
y_train = np.random.random(size=84)
|
||||
automl = AutoML()
|
||||
automl.fit(X_train=X_train[:72], # a single column of timestamp
|
||||
automl.fit(X_train=X_train[:84], # a single column of timestamp
|
||||
y_train=y_train, # value for each timestamp
|
||||
period=12, # time horizon to forecast, e.g., 12 months
|
||||
task='ts_forecast', time_budget=15, # time budget in seconds
|
||||
log_file_name="ts_forecast.log",
|
||||
eval_method="holdout",
|
||||
)
|
||||
print(automl.predict(X_train[72:]))
|
||||
print(automl.predict(X_train[84:]))
|
||||
```
|
||||
|
||||
#### Sample output
|
||||
|
||||
```
|
||||
[flaml.automl: 11-15 18:44:49] {1485} INFO - Data split method: time
|
||||
INFO:flaml.automl:Data split method: time
|
||||
[flaml.automl: 11-15 18:44:49] {1489} INFO - Evaluation method: cv
|
||||
INFO:flaml.automl:Evaluation method: cv
|
||||
[flaml.automl: 11-15 18:44:49] {1540} INFO - Minimizing error metric: mape
|
||||
INFO:flaml.automl:Minimizing error metric: mape
|
||||
[flaml.automl: 11-15 18:44:49] {1577} INFO - List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax']
|
||||
INFO:flaml.automl:List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax']
|
||||
[flaml.automl: 11-15 18:44:49] {1826} INFO - iteration 0, current learner prophet
|
||||
INFO:flaml.automl:iteration 0, current learner prophet
|
||||
[flaml.automl: 11-15 18:45:00] {1944} INFO - Estimated sufficient time budget=104159s. Estimated necessary time budget=104s.
|
||||
INFO:flaml.automl:Estimated sufficient time budget=104159s. Estimated necessary time budget=104s.
|
||||
[flaml.automl: 11-15 18:45:00] {2029} INFO - at 10.5s, estimator prophet's best error=1.5681, best estimator prophet's best error=1.5681
|
||||
INFO:flaml.automl: at 10.5s, estimator prophet's best error=1.5681, best estimator prophet's best error=1.5681
|
||||
[flaml.automl: 11-15 18:45:00] {1826} INFO - iteration 1, current learner arima
|
||||
INFO:flaml.automl:iteration 1, current learner arima
|
||||
[flaml.automl: 11-15 18:45:00] {2029} INFO - at 10.7s, estimator arima's best error=2.3515, best estimator prophet's best error=1.5681
|
||||
INFO:flaml.automl: at 10.7s, estimator arima's best error=2.3515, best estimator prophet's best error=1.5681
|
||||
[flaml.automl: 11-15 18:45:00] {1826} INFO - iteration 2, current learner arima
|
||||
INFO:flaml.automl:iteration 2, current learner arima
|
||||
[flaml.automl: 11-15 18:45:01] {2029} INFO - at 11.5s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681
|
||||
INFO:flaml.automl: at 11.5s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681
|
||||
[flaml.automl: 11-15 18:45:01] {1826} INFO - iteration 3, current learner arima
|
||||
INFO:flaml.automl:iteration 3, current learner arima
|
||||
[flaml.automl: 11-15 18:45:01] {2029} INFO - at 11.9s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681
|
||||
INFO:flaml.automl: at 11.9s, estimator arima's best error=2.1774, best estimator prophet's best error=1.5681
|
||||
[flaml.automl: 11-15 18:45:01] {1826} INFO - iteration 4, current learner arima
|
||||
INFO:flaml.automl:iteration 4, current learner arima
|
||||
[flaml.automl: 11-15 18:45:02] {2029} INFO - at 12.9s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681
|
||||
INFO:flaml.automl: at 12.9s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681
|
||||
[flaml.automl: 11-15 18:45:02] {1826} INFO - iteration 5, current learner arima
|
||||
INFO:flaml.automl:iteration 5, current learner arima
|
||||
[flaml.automl: 11-15 18:45:04] {2029} INFO - at 14.4s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681
|
||||
INFO:flaml.automl: at 14.4s, estimator arima's best error=1.8560, best estimator prophet's best error=1.5681
|
||||
[flaml.automl: 11-15 18:45:04] {1826} INFO - iteration 6, current learner sarimax
|
||||
INFO:flaml.automl:iteration 6, current learner sarimax
|
||||
[flaml.automl: 11-15 18:45:04] {2029} INFO - at 14.7s, estimator sarimax's best error=2.3515, best estimator prophet's best error=1.5681
|
||||
INFO:flaml.automl: at 14.7s, estimator sarimax's best error=2.3515, best estimator prophet's best error=1.5681
|
||||
[flaml.automl: 11-15 18:45:04] {1826} INFO - iteration 7, current learner sarimax
|
||||
INFO:flaml.automl:iteration 7, current learner sarimax
|
||||
[flaml.automl: 11-15 18:45:04] {2029} INFO - at 15.0s, estimator sarimax's best error=1.6371, best estimator prophet's best error=1.5681
|
||||
INFO:flaml.automl: at 15.0s, estimator sarimax's best error=1.6371, best estimator prophet's best error=1.5681
|
||||
[flaml.automl: 11-15 18:45:05] {2242} INFO - retrain prophet for 0.5s
|
||||
INFO:flaml.automl:retrain prophet for 0.5s
|
||||
[flaml.automl: 11-15 18:45:05] {2247} INFO - retrained model: <prophet.forecaster.Prophet object at 0x7f042ba1da50>
|
||||
INFO:flaml.automl:retrained model: <prophet.forecaster.Prophet object at 0x7f042ba1da50>
|
||||
[flaml.automl: 11-15 18:45:05] {1608} INFO - fit succeeded
|
||||
INFO:flaml.automl:fit succeeded
|
||||
[flaml.automl: 11-15 18:45:05] {1610} INFO - Time taken to find the best model: 10.450132608413696
|
||||
INFO:flaml.automl:Time taken to find the best model: 10.450132608413696
|
||||
0 0.384715
|
||||
1 0.191349
|
||||
2 0.372324
|
||||
3 0.814549
|
||||
4 0.269616
|
||||
5 0.470667
|
||||
6 0.603665
|
||||
7 0.256773
|
||||
8 0.408787
|
||||
9 0.663065
|
||||
10 0.619943
|
||||
11 0.090284
|
||||
Name: yhat, dtype: float64
|
||||
```python
|
||||
[flaml.automl: 01-21 08:01:20] {2018} INFO - task = ts_forecast
|
||||
[flaml.automl: 01-21 08:01:20] {2020} INFO - Data split method: time
|
||||
[flaml.automl: 01-21 08:01:20] {2024} INFO - Evaluation method: holdout
|
||||
[flaml.automl: 01-21 08:01:20] {2124} INFO - Minimizing error metric: mape
|
||||
[flaml.automl: 01-21 08:01:21] {2181} INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'xgboost', 'extra_tree', 'xgb_limitdepth', 'prophet', 'arima', 'sarimax']
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 0, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2547} INFO - Estimated sufficient time budget=1429s. Estimated necessary time budget=1s.
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 1, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 2, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 0.9s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 3, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 4, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 5, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9811, best estimator lgbm's best error=0.9811
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 6, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9652, best estimator lgbm's best error=0.9652
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 7, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 8, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:21] {2594} INFO - at 1.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:21] {2434} INFO - iteration 9, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 10, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 11, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 12, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 13, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 14, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 15, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 16, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 17, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 18, current learner rf
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator rf's best error=1.0994, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 19, current learner rf
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.2s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 20, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0271, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 21, current learner rf
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 22, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 23, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 24, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 25, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.3s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 26, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 27, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 28, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=1.0130, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 29, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.4s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 30, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 31, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 32, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 33, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 34, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 35, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.5s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 36, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 37, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 38, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 39, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 40, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.6s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 41, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 42, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 43, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 44, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.7s, estimator xgb_limitdepth's best error=1.5815, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 45, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 46, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 47, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.8s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 48, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 49, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 50, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 51, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 1.9s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 52, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 53, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 54, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 55, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 56, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 57, current learner rf
|
||||
[flaml.automl: 01-21 08:01:22] {2594} INFO - at 2.0s, estimator rf's best error=1.0848, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:22] {2434} INFO - iteration 58, current learner xgboost
|
||||
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator xgboost's best error=1.0015, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 59, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 60, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 61, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator extra_tree's best error=0.9499, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 62, current learner lgbm
|
||||
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.1s, estimator lgbm's best error=0.9466, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 63, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:23] {2594} INFO - at 2.2s, estimator xgb_limitdepth's best error=0.9683, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:23] {2434} INFO - iteration 64, current learner prophet
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.2s, estimator prophet's best error=1.5706, best estimator lgbm's best error=0.9466
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 65, current learner arima
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.2s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 66, current learner arima
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.4s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 67, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.4s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 68, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.5s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 69, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 70, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 71, current learner arima
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 72, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 73, current learner arima
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.7s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 74, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.7s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 75, current learner arima
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.8s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 76, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 4.9s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 77, current learner arima
|
||||
[flaml.automl: 01-21 08:01:25] {2594} INFO - at 5.0s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:25] {2434} INFO - iteration 78, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 79, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 80, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 81, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:26] {2594} INFO - at 5.1s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:26] {2434} INFO - iteration 82, current learner prophet
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator prophet's best error=1.4076, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 83, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 84, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 85, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.6s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 86, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.8s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 87, current learner arima
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.8s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 88, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.9s, estimator sarimax's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 89, current learner arima
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 6.9s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 90, current learner arima
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator arima's best error=0.5693, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 91, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 92, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:27] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.9683, best estimator arima's best error=0.5693
|
||||
[flaml.automl: 01-21 08:01:27] {2434} INFO - iteration 93, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.0s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600
|
||||
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 94, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.1s, estimator xgb_limitdepth's best error=0.9683, best estimator sarimax's best error=0.5600
|
||||
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 95, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600
|
||||
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 96, current learner arima
|
||||
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator arima's best error=0.5693, best estimator sarimax's best error=0.5600
|
||||
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 97, current learner arima
|
||||
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.2s, estimator arima's best error=0.5693, best estimator sarimax's best error=0.5600
|
||||
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 98, current learner extra_tree
|
||||
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator extra_tree's best error=0.9499, best estimator sarimax's best error=0.5600
|
||||
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 99, current learner sarimax
|
||||
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator sarimax's best error=0.5600, best estimator sarimax's best error=0.5600
|
||||
[flaml.automl: 01-21 08:01:28] {2434} INFO - iteration 100, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 08:01:28] {2594} INFO - at 7.3s, estimator xgb_limitdepth's best error=0.9683, best estimator sarimax's best error=0.5600
|
||||
```
|
||||
|
||||
### Multivariate time series
|
||||
@ -131,62 +277,142 @@ automl.fit(dataframe=train_df, # training data
|
||||
#### Sample output
|
||||
|
||||
```
|
||||
[flaml.automl: 11-15 18:54:12] {1485} INFO - Data split method: time
|
||||
INFO:flaml.automl:Data split method: time
|
||||
[flaml.automl: 11-15 18:54:12] {1489} INFO - Evaluation method: holdout
|
||||
INFO:flaml.automl:Evaluation method: holdout
|
||||
[flaml.automl: 11-15 18:54:13] {1540} INFO - Minimizing error metric: mape
|
||||
INFO:flaml.automl:Minimizing error metric: mape
|
||||
[flaml.automl: 11-15 18:54:13] {1577} INFO - List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax']
|
||||
INFO:flaml.automl:List of ML learners in AutoML Run: ['prophet', 'arima', 'sarimax']
|
||||
[flaml.automl: 11-15 18:54:13] {1826} INFO - iteration 0, current learner prophet
|
||||
INFO:flaml.automl:iteration 0, current learner prophet
|
||||
[flaml.automl: 11-15 18:54:15] {1944} INFO - Estimated sufficient time budget=25297s. Estimated necessary time budget=25s.
|
||||
INFO:flaml.automl:Estimated sufficient time budget=25297s. Estimated necessary time budget=25s.
|
||||
[flaml.automl: 11-15 18:54:15] {2029} INFO - at 2.6s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008
|
||||
INFO:flaml.automl: at 2.6s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 11-15 18:54:15] {1826} INFO - iteration 1, current learner prophet
|
||||
INFO:flaml.automl:iteration 1, current learner prophet
|
||||
[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.2s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008
|
||||
INFO:flaml.automl: at 5.2s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 2, current learner arima
|
||||
INFO:flaml.automl:iteration 2, current learner arima
|
||||
[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.5s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
INFO:flaml.automl: at 5.5s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 3, current learner arima
|
||||
INFO:flaml.automl:iteration 3, current learner arima
|
||||
[flaml.automl: 11-15 18:54:18] {2029} INFO - at 5.6s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
INFO:flaml.automl: at 5.6s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 11-15 18:54:18] {1826} INFO - iteration 4, current learner prophet
|
||||
INFO:flaml.automl:iteration 4, current learner prophet
|
||||
[flaml.automl: 11-15 18:54:21] {2029} INFO - at 8.1s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005
|
||||
INFO:flaml.automl: at 8.1s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 11-15 18:54:21] {1826} INFO - iteration 5, current learner arima
|
||||
INFO:flaml.automl:iteration 5, current learner arima
|
||||
[flaml.automl: 11-15 18:54:21] {2029} INFO - at 8.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005
|
||||
INFO:flaml.automl: at 8.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 11-15 18:54:21] {1826} INFO - iteration 6, current learner arima
|
||||
INFO:flaml.automl:iteration 6, current learner arima
|
||||
[flaml.automl: 11-15 18:54:22] {2029} INFO - at 9.7s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005
|
||||
INFO:flaml.automl: at 9.7s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 11-15 18:54:22] {1826} INFO - iteration 7, current learner sarimax
|
||||
INFO:flaml.automl:iteration 7, current learner sarimax
|
||||
[flaml.automl: 11-15 18:54:23] {2029} INFO - at 10.1s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005
|
||||
INFO:flaml.automl: at 10.1s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 11-15 18:54:23] {2242} INFO - retrain prophet for 0.9s
|
||||
INFO:flaml.automl:retrain prophet for 0.9s
|
||||
[flaml.automl: 11-15 18:54:23] {2247} INFO - retrained model: <prophet.forecaster.Prophet object at 0x7f0418e21f50>
|
||||
INFO:flaml.automl:retrained model: <prophet.forecaster.Prophet object at 0x7f0418e21f50>
|
||||
[flaml.automl: 11-15 18:54:23] {1608} INFO - fit succeeded
|
||||
INFO:flaml.automl:fit succeeded
|
||||
[flaml.automl: 11-15 18:54:23] {1610} INFO - Time taken to find the best model: 8.118467330932617
|
||||
INFO:flaml.automl:Time taken to find the best model: 8.118467330932617
|
||||
[flaml.automl: 11-15 18:54:23] {1624} WARNING - Time taken to find the best model is 81% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.
|
||||
WARNING:flaml.automl:Time taken to find the best model is 81% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.
|
||||
[flaml.automl: 01-21 07:54:04] {2018} INFO - task = ts_forecast
|
||||
[flaml.automl: 01-21 07:54:04] {2020} INFO - Data split method: time
|
||||
[flaml.automl: 01-21 07:54:04] {2024} INFO - Evaluation method: holdout
|
||||
[flaml.automl: 01-21 07:54:04] {2124} INFO - Minimizing error metric: mape
|
||||
Importing plotly failed. Interactive plots will not work.
|
||||
[flaml.automl: 01-21 07:54:04] {2181} INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'xgboost', 'extra_tree', 'xgb_limitdepth', 'prophet', 'arima', 'sarimax']
|
||||
[flaml.automl: 01-21 07:54:04] {2434} INFO - iteration 0, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2547} INFO - Estimated sufficient time budget=2145s. Estimated necessary time budget=2s.
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 0.9s, estimator lgbm's best error=0.0621, best estimator lgbm's best error=0.0621
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 1, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0574, best estimator lgbm's best error=0.0574
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 2, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0464, best estimator lgbm's best error=0.0464
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 3, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0464, best estimator lgbm's best error=0.0464
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 4, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.0s, estimator lgbm's best error=0.0365, best estimator lgbm's best error=0.0365
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 5, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 6, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 7, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.1s, estimator lgbm's best error=0.0192, best estimator lgbm's best error=0.0192
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 8, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0110, best estimator lgbm's best error=0.0110
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 9, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0110, best estimator lgbm's best error=0.0110
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 10, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.2s, estimator lgbm's best error=0.0036, best estimator lgbm's best error=0.0036
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 11, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.4s, estimator lgbm's best error=0.0023, best estimator lgbm's best error=0.0023
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 12, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.4s, estimator lgbm's best error=0.0023, best estimator lgbm's best error=0.0023
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 13, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.5s, estimator lgbm's best error=0.0021, best estimator lgbm's best error=0.0021
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 14, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.6s, estimator lgbm's best error=0.0021, best estimator lgbm's best error=0.0021
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 15, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.7s, estimator lgbm's best error=0.0020, best estimator lgbm's best error=0.0020
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 16, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:05] {2594} INFO - at 1.8s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:05] {2434} INFO - iteration 17, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 1.9s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 18, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.0s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 19, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 20, current learner rf
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator rf's best error=0.0228, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 21, current learner rf
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.1s, estimator rf's best error=0.0210, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 22, current learner xgboost
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.6738, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 23, current learner xgboost
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.6738, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 24, current learner xgboost
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.2s, estimator xgboost's best error=0.1717, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 25, current learner xgboost
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0249, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 26, current learner xgboost
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0249, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 27, current learner xgboost
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.3s, estimator xgboost's best error=0.0242, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 28, current learner extra_tree
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.4s, estimator extra_tree's best error=0.0245, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 29, current learner extra_tree
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.4s, estimator extra_tree's best error=0.0160, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 30, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.5s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 31, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 32, current learner rf
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator rf's best error=0.0210, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 33, current learner extra_tree
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.6s, estimator extra_tree's best error=0.0160, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 34, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.8s, estimator lgbm's best error=0.0017, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 35, current learner extra_tree
|
||||
[flaml.automl: 01-21 07:54:06] {2594} INFO - at 2.8s, estimator extra_tree's best error=0.0158, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:06] {2434} INFO - iteration 36, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.8s, estimator xgb_limitdepth's best error=0.0447, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 37, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.9s, estimator xgb_limitdepth's best error=0.0447, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 38, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 2.9s, estimator xgb_limitdepth's best error=0.0029, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 39, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.0s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 40, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.1s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 41, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.1s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 42, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:07] {2594} INFO - at 3.3s, estimator xgb_limitdepth's best error=0.0018, best estimator lgbm's best error=0.0017
|
||||
[flaml.automl: 01-21 07:54:07] {2434} INFO - iteration 43, current learner prophet
|
||||
[flaml.automl: 01-21 07:54:09] {2594} INFO - at 5.5s, estimator prophet's best error=0.0008, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:09] {2434} INFO - iteration 44, current learner arima
|
||||
[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.1s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 45, current learner sarimax
|
||||
[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.4s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 46, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.5s, estimator lgbm's best error=0.0017, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 47, current learner sarimax
|
||||
[flaml.automl: 01-21 07:54:10] {2594} INFO - at 6.6s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:10] {2434} INFO - iteration 48, current learner sarimax
|
||||
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 6.9s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 49, current learner arima
|
||||
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 6.9s, estimator arima's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 50, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.0s, estimator xgb_limitdepth's best error=0.0018, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 51, current learner sarimax
|
||||
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.5s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 52, current learner xgboost
|
||||
[flaml.automl: 01-21 07:54:11] {2594} INFO - at 7.6s, estimator xgboost's best error=0.0242, best estimator prophet's best error=0.0008
|
||||
[flaml.automl: 01-21 07:54:11] {2434} INFO - iteration 53, current learner prophet
|
||||
[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.3s, estimator prophet's best error=0.0005, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 54, current learner sarimax
|
||||
[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.4s, estimator sarimax's best error=0.0047, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 55, current learner xgb_limitdepth
|
||||
[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.8s, estimator xgb_limitdepth's best error=0.0018, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 56, current learner xgboost
|
||||
[flaml.automl: 01-21 07:54:13] {2594} INFO - at 9.8s, estimator xgboost's best error=0.0242, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 01-21 07:54:13] {2434} INFO - iteration 57, current learner lgbm
|
||||
[flaml.automl: 01-21 07:54:14] {2594} INFO - at 9.9s, estimator lgbm's best error=0.0017, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 01-21 07:54:14] {2434} INFO - iteration 58, current learner rf
|
||||
[flaml.automl: 01-21 07:54:14] {2594} INFO - at 10.0s, estimator rf's best error=0.0146, best estimator prophet's best error=0.0005
|
||||
[flaml.automl: 01-21 07:54:14] {2824} INFO - retrain prophet for 0.6s
|
||||
[flaml.automl: 01-21 07:54:14] {2831} INFO - retrained model: <prophet.forecaster.Prophet object at 0x7fb68ea65d60>
|
||||
[flaml.automl: 01-21 07:54:14] {2210} INFO - fit succeeded
|
||||
[flaml.automl: 01-21 07:54:14] {2211} INFO - Time taken to find the best model: 9.339771270751953
|
||||
[flaml.automl: 01-21 07:54:14] {2222} WARNING - Time taken to find the best model is 93% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.
|
||||
```
|
||||
|
||||
#### Compute and plot predictions
|
||||
|
||||
The example plotting code requires matplotlib.
|
||||
|
||||
```python
|
||||
flaml_y_pred = automl.predict(X_test)
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user