yujunjun/autogen
1
0
Fork 0
mirror of https://github.com/microsoft/autogen.git synced 2025-10-31 09:50:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix automl settings in scikit-learn pipeline integration example (#602)

Browse Source 
* Added test directory and core file to gitignore.

Closes #601.

* Fixed pipeline fit parameters.

Closes #600.

* Reverted changes to gitignore.
This commit is contained in:
Zvi Baratz 2022-06-23 05:44:14 +03:00 committed by GitHub
parent c45741a67b
commit fefdb48ef3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 211 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

396
notebook/integrate_sklearn.ipynb
View File

@ -93,14 +93,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"load dataset from ./openml_ds1169.pkl\n", "download dataset from openml\n",
"Dataset name: airlines\n", "Dataset name: airlines\n",
"X_train.shape: (404537, 7), y_train.shape: (404537,);\n", "X_train.shape: (404537, 7), y_train.shape: (404537,);\n",
"X_test.shape: (134846, 7), y_test.shape: (134846,)\n" "X_test.shape: (134846, 7), y_test.shape: (134846,)\n"
@ -115,7 +115,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -124,7 +124,7 @@
"array([ 12., 2648., 4., 15., 4., 450., 67.], dtype=float32)" "array([ 12., 2648., 4., 15., 4., 450., 67.], dtype=float32)"
] ]
}, },
"execution_count": 5, "execution_count": 2,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -142,29 +142,74 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "data": {
"text/html": [ "text/html": [
"<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"b91d1bdf-ccb8-4fa5-a2d0-67a3538c0afc\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"b91d1bdf-ccb8-4fa5-a2d0-67a3538c0afc\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[('imputuer', SimpleImputer()),\n", "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;imputuer&#x27;, SimpleImputer()),\n",
" ('standardizer', StandardScaler()),\n", " (&#x27;standardizer&#x27;, StandardScaler()),\n",
" ('automl', <flaml.automl.AutoML object at 0x7f046d56fb50>)])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a8311733-9e55-4c0c-9c2a-6b9ba6227596\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a8311733-9e55-4c0c-9c2a-6b9ba6227596\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"52580e54-89ab-4fb7-83a1-ae13962854bb\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"52580e54-89ab-4fb7-83a1-ae13962854bb\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"b9fe5397-bf24-491d-a938-c39a780e1ac0\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"b9fe5397-bf24-491d-a938-c39a780e1ac0\">AutoML</label><div class=\"sk-toggleable__content\"><pre><flaml.automl.AutoML object at 0x7f046d56fb50></pre></div></div></div></div></div></div></div>" " (&#x27;automl&#x27;,\n",
" AutoML(append_log=False, auto_augment=True, custom_hp={},\n",
" early_stop=False, ensemble=False, estimator_list=&#x27;auto&#x27;,\n",
" eval_method=&#x27;auto&#x27;, fit_kwargs_by_estimator={},\n",
" hpo_method=&#x27;auto&#x27;, keep_search_state=False,\n",
" learner_selector=&#x27;sample&#x27;, log_file_name=&#x27;&#x27;,\n",
" log_training_metric=False, log_type=&#x27;better&#x27;,\n",
" max_iter=None, mem_thres=4294967296, metric=&#x27;auto&#x27;,\n",
" metric_constraints=[], min_sample_size=10000,\n",
" model_history=False, n_concurrent_trials=1, n_jobs=-1,\n",
" n_splits=5, pred_time_limit=inf, retrain_full=True,\n",
" sample=True, split_ratio=0.1, split_type=&#x27;auto&#x27;,\n",
" starting_points=&#x27;static&#x27;, task=&#x27;classification&#x27;, ...))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;imputuer&#x27;, SimpleImputer()),\n",
" (&#x27;standardizer&#x27;, StandardScaler()),\n",
" (&#x27;automl&#x27;,\n",
" AutoML(append_log=False, auto_augment=True, custom_hp={},\n",
" early_stop=False, ensemble=False, estimator_list=&#x27;auto&#x27;,\n",
" eval_method=&#x27;auto&#x27;, fit_kwargs_by_estimator={},\n",
" hpo_method=&#x27;auto&#x27;, keep_search_state=False,\n",
" learner_selector=&#x27;sample&#x27;, log_file_name=&#x27;&#x27;,\n",
" log_training_metric=False, log_type=&#x27;better&#x27;,\n",
" max_iter=None, mem_thres=4294967296, metric=&#x27;auto&#x27;,\n",
" metric_constraints=[], min_sample_size=10000,\n",
" model_history=False, n_concurrent_trials=1, n_jobs=-1,\n",
" n_splits=5, pred_time_limit=inf, retrain_full=True,\n",
" sample=True, split_ratio=0.1, split_type=&#x27;auto&#x27;,\n",
" starting_points=&#x27;static&#x27;, task=&#x27;classification&#x27;, ...))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">AutoML</label><div class=\"sk-toggleable__content\"><pre>AutoML(append_log=False, auto_augment=True, custom_hp={}, early_stop=False,\n",
" ensemble=False, estimator_list=&#x27;auto&#x27;, eval_method=&#x27;auto&#x27;,\n",
" fit_kwargs_by_estimator={}, hpo_method=&#x27;auto&#x27;, keep_search_state=False,\n",
" learner_selector=&#x27;sample&#x27;, log_file_name=&#x27;&#x27;, log_training_metric=False,\n",
" log_type=&#x27;better&#x27;, max_iter=None, mem_thres=4294967296, metric=&#x27;auto&#x27;,\n",
" metric_constraints=[], min_sample_size=10000, model_history=False,\n",
" n_concurrent_trials=1, n_jobs=-1, n_splits=5, pred_time_limit=inf,\n",
" retrain_full=True, sample=True, split_ratio=0.1, split_type=&#x27;auto&#x27;,\n",
" starting_points=&#x27;static&#x27;, task=&#x27;classification&#x27;, ...)</pre></div></div></div></div></div></div></div>"
], ],
"text/plain": [ "text/plain": [
"Pipeline(steps=[('imputuer', SimpleImputer()),\n", "Pipeline(steps=[('imputuer', SimpleImputer()),\n",
" ('standardizer', StandardScaler()),\n", " ('standardizer', StandardScaler()),\n",
" ('automl', <flaml.automl.AutoML object at 0x7f046d56fb50>)])" " ('automl',\n",
" AutoML(append_log=False, auto_augment=True, custom_hp={},\n",
" early_stop=False, ensemble=False, estimator_list='auto',\n",
" eval_method='auto', fit_kwargs_by_estimator={},\n",
" hpo_method='auto', keep_search_state=False,\n",
" learner_selector='sample', log_file_name='',\n",
" log_training_metric=False, log_type='better',\n",
" max_iter=None, mem_thres=4294967296, metric='auto',\n",
" metric_constraints=[], min_sample_size=10000,\n",
" model_history=False, n_concurrent_trials=1, n_jobs=-1,\n",
" n_splits=5, pred_time_limit=inf, retrain_full=True,\n",
" sample=True, split_ratio=0.1, split_type='auto',\n",
" starting_points='static', task='classification', ...))])"
] ]
}, },
"execution_count": 6, "execution_count": 3,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
], ],
"source": [ "source": [
"import sklearn\n",
"from sklearn import set_config\n", "from sklearn import set_config\n",
"from sklearn.pipeline import Pipeline\n", "from sklearn.pipeline import Pipeline\n",
"from sklearn.impute import SimpleImputer\n", "from sklearn.impute import SimpleImputer\n",
@ -195,217 +240,177 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"settings = {\n", "automl_settings = {\n",
" \"time_budget\": 60, # total running time in seconds\n", " \"time_budget\": 60, # total running time in seconds\n",
" \"metric\": 'accuracy', # primary metrics can be chosen from: ['accuracy','roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'f1','log_loss','mae','mse','r2']\n", " \"metric\": 'accuracy', # primary metrics can be chosen from: ['accuracy','roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'f1','log_loss','mae','mse','r2']\n",
" \"task\": 'classification', # task type \n", " \"task\": 'classification', # task type \n",
" \"estimator_list\":['xgboost','catboost','lgbm'],\n", " \"estimator_list\": ['xgboost','catboost','lgbm'],\n",
" \"log_file_name\": 'airlines_experiment.log', # flaml log file\n", " \"log_file_name\": 'airlines_experiment.log', # flaml log file\n",
"}" "}\n",
"pipeline_settings = {f\"automl__{key}\": value for key, value in automl_settings.items()}"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"[flaml.automl: 08-22 21:32:13] {1130} INFO - Evaluation method: holdout\n", "[flaml.automl: 06-22 08:01:43] {2390} INFO - task = classification\n",
"[flaml.automl: 08-22 21:32:14] {624} INFO - Using StratifiedKFold\n", "[flaml.automl: 06-22 08:01:43] {2392} INFO - Data split method: stratified\n",
"[flaml.automl: 08-22 21:32:14] {1155} INFO - Minimizing error metric: 1-accuracy\n", "[flaml.automl: 06-22 08:01:43] {2396} INFO - Evaluation method: holdout\n",
"[flaml.automl: 08-22 21:32:14] {1175} INFO - List of ML learners in AutoML Run: ['xgboost', 'catboost', 'lgbm']\n", "[flaml.automl: 06-22 08:01:44] {2465} INFO - Minimizing error metric: 1-accuracy\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 0, current learner xgboost\n", "[flaml.automl: 06-22 08:01:44] {2605} INFO - List of ML learners in AutoML Run: ['xgboost', 'catboost', 'lgbm']\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 0.5s,\tbest xgboost's error=0.3755,\tbest xgboost's error=0.3755\n", "[flaml.automl: 06-22 08:01:44] {2897} INFO - iteration 0, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 1, current learner xgboost\n", "[flaml.automl: 06-22 08:01:44] {3025} INFO - Estimated sufficient time budget=105341s. Estimated necessary time budget=116s.\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 0.6s,\tbest xgboost's error=0.3755,\tbest xgboost's error=0.3755\n", "[flaml.automl: 06-22 08:01:44] {3072} INFO - at 0.7s,\testimator xgboost's best error=0.3755,\tbest estimator xgboost's best error=0.3755\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 2, current learner xgboost\n", "[flaml.automl: 06-22 08:01:44] {2897} INFO - iteration 1, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 0.6s,\tbest xgboost's error=0.3755,\tbest xgboost's error=0.3755\n", "[flaml.automl: 06-22 08:01:44] {3072} INFO - at 0.9s,\testimator lgbm's best error=0.3814,\tbest estimator xgboost's best error=0.3755\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 3, current learner xgboost\n", "[flaml.automl: 06-22 08:01:44] {2897} INFO - iteration 2, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 0.7s,\tbest xgboost's error=0.3755,\tbest xgboost's error=0.3755\n", "[flaml.automl: 06-22 08:01:45] {3072} INFO - at 1.3s,\testimator xgboost's best error=0.3755,\tbest estimator xgboost's best error=0.3755\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 4, current learner xgboost\n", "[flaml.automl: 06-22 08:01:45] {2897} INFO - iteration 3, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 0.7s,\tbest xgboost's error=0.3679,\tbest xgboost's error=0.3679\n", "[flaml.automl: 06-22 08:01:45] {3072} INFO - at 1.5s,\testimator lgbm's best error=0.3814,\tbest estimator xgboost's best error=0.3755\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 5, current learner lgbm\n", "[flaml.automl: 06-22 08:01:45] {2897} INFO - iteration 4, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 0.8s,\tbest lgbm's error=0.3811,\tbest xgboost's error=0.3679\n", "[flaml.automl: 06-22 08:01:45] {3072} INFO - at 1.8s,\testimator xgboost's best error=0.3755,\tbest estimator xgboost's best error=0.3755\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 6, current learner xgboost\n", "[flaml.automl: 06-22 08:01:45] {2897} INFO - iteration 5, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 0.8s,\tbest xgboost's error=0.3679,\tbest xgboost's error=0.3679\n", "[flaml.automl: 06-22 08:01:45] {3072} INFO - at 2.0s,\testimator lgbm's best error=0.3755,\tbest estimator xgboost's best error=0.3755\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 7, current learner xgboost\n", "[flaml.automl: 06-22 08:01:45] {2897} INFO - iteration 6, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 0.9s,\tbest xgboost's error=0.3679,\tbest xgboost's error=0.3679\n", "[flaml.automl: 06-22 08:01:46] {3072} INFO - at 2.3s,\testimator xgboost's best error=0.3724,\tbest estimator xgboost's best error=0.3724\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 8, current learner xgboost\n", "[flaml.automl: 06-22 08:01:46] {2897} INFO - iteration 7, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:14] {1515} INFO - at 1.0s,\tbest xgboost's error=0.3679,\tbest xgboost's error=0.3679\n", "[flaml.automl: 06-22 08:01:46] {3072} INFO - at 2.6s,\testimator xgboost's best error=0.3724,\tbest estimator xgboost's best error=0.3724\n",
"[flaml.automl: 08-22 21:32:14] {1358} INFO - iteration 9, current learner lgbm\n", "[flaml.automl: 06-22 08:01:46] {2897} INFO - iteration 8, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:15] {1515} INFO - at 1.1s,\tbest lgbm's error=0.3811,\tbest xgboost's error=0.3679\n", "[flaml.automl: 06-22 08:01:47] {3072} INFO - at 3.1s,\testimator xgboost's best error=0.3657,\tbest estimator xgboost's best error=0.3657\n",
"[flaml.automl: 08-22 21:32:15] {1358} INFO - iteration 10, current learner lgbm\n", "[flaml.automl: 06-22 08:01:47] {2897} INFO - iteration 9, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:15] {1515} INFO - at 1.1s,\tbest lgbm's error=0.3755,\tbest xgboost's error=0.3679\n", "[flaml.automl: 06-22 08:01:47] {3072} INFO - at 3.6s,\testimator xgboost's best error=0.3657,\tbest estimator xgboost's best error=0.3657\n",
"[flaml.automl: 08-22 21:32:15] {1358} INFO - iteration 11, current learner xgboost\n", "[flaml.automl: 06-22 08:01:47] {2897} INFO - iteration 10, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:15] {1515} INFO - at 1.2s,\tbest xgboost's error=0.3637,\tbest xgboost's error=0.3637\n", "[flaml.automl: 06-22 08:01:48] {3072} INFO - at 4.8s,\testimator xgboost's best error=0.3592,\tbest estimator xgboost's best error=0.3592\n",
"[flaml.automl: 08-22 21:32:15] {1358} INFO - iteration 12, current learner xgboost\n", "[flaml.automl: 06-22 08:01:48] {2897} INFO - iteration 11, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:15] {1515} INFO - at 1.4s,\tbest xgboost's error=0.3594,\tbest xgboost's error=0.3594\n", "[flaml.automl: 06-22 08:01:50] {3072} INFO - at 6.8s,\testimator xgboost's best error=0.3580,\tbest estimator xgboost's best error=0.3580\n",
"[flaml.automl: 08-22 21:32:15] {1358} INFO - iteration 13, current learner xgboost\n", "[flaml.automl: 06-22 08:01:50] {2897} INFO - iteration 12, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:15] {1515} INFO - at 1.5s,\tbest xgboost's error=0.3594,\tbest xgboost's error=0.3594\n", "[flaml.automl: 06-22 08:01:51] {3072} INFO - at 8.1s,\testimator xgboost's best error=0.3580,\tbest estimator xgboost's best error=0.3580\n",
"[flaml.automl: 08-22 21:32:15] {1358} INFO - iteration 14, current learner xgboost\n", "[flaml.automl: 06-22 08:01:51] {2897} INFO - iteration 13, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:15] {1515} INFO - at 1.7s,\tbest xgboost's error=0.3591,\tbest xgboost's error=0.3591\n", "[flaml.automl: 06-22 08:01:52] {3072} INFO - at 8.4s,\testimator lgbm's best error=0.3644,\tbest estimator xgboost's best error=0.3580\n",
"[flaml.automl: 08-22 21:32:15] {1358} INFO - iteration 15, current learner lgbm\n", "[flaml.automl: 06-22 08:01:52] {2897} INFO - iteration 14, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:15] {1515} INFO - at 1.7s,\tbest lgbm's error=0.3647,\tbest xgboost's error=0.3591\n", "[flaml.automl: 06-22 08:01:52] {3072} INFO - at 8.7s,\testimator lgbm's best error=0.3644,\tbest estimator xgboost's best error=0.3580\n",
"[flaml.automl: 08-22 21:32:15] {1358} INFO - iteration 16, current learner xgboost\n", "[flaml.automl: 06-22 08:01:52] {2897} INFO - iteration 15, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:15] {1515} INFO - at 2.0s,\tbest xgboost's error=0.3585,\tbest xgboost's error=0.3585\n", "[flaml.automl: 06-22 08:01:53] {3072} INFO - at 9.3s,\testimator lgbm's best error=0.3644,\tbest estimator xgboost's best error=0.3580\n",
"[flaml.automl: 08-22 21:32:15] {1358} INFO - iteration 17, current learner lgbm\n", "[flaml.automl: 06-22 08:01:53] {2897} INFO - iteration 16, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:16] {1515} INFO - at 2.0s,\tbest lgbm's error=0.3647,\tbest xgboost's error=0.3585\n", "[flaml.automl: 06-22 08:01:56] {3072} INFO - at 12.1s,\testimator xgboost's best error=0.3559,\tbest estimator xgboost's best error=0.3559\n",
"[flaml.automl: 08-22 21:32:16] {1358} INFO - iteration 18, current learner lgbm\n", "[flaml.automl: 06-22 08:01:56] {2897} INFO - iteration 17, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:16] {1515} INFO - at 2.1s,\tbest lgbm's error=0.3629,\tbest xgboost's error=0.3585\n", "[flaml.automl: 06-22 08:01:56] {3072} INFO - at 12.6s,\testimator lgbm's best error=0.3604,\tbest estimator xgboost's best error=0.3559\n",
"[flaml.automl: 08-22 21:32:16] {1358} INFO - iteration 19, current learner xgboost\n", "[flaml.automl: 06-22 08:01:56] {2897} INFO - iteration 18, current learner catboost\n",
"[flaml.automl: 08-22 21:32:16] {1515} INFO - at 2.3s,\tbest xgboost's error=0.3553,\tbest xgboost's error=0.3553\n", "[flaml.automl: 06-22 08:01:56] {3072} INFO - at 13.0s,\testimator catboost's best error=0.3615,\tbest estimator xgboost's best error=0.3559\n",
"[flaml.automl: 08-22 21:32:16] {1358} INFO - iteration 20, current learner xgboost\n", "[flaml.automl: 06-22 08:01:56] {2897} INFO - iteration 19, current learner catboost\n",
"[flaml.automl: 08-22 21:32:16] {1515} INFO - at 2.6s,\tbest xgboost's error=0.3553,\tbest xgboost's error=0.3553\n", "[flaml.automl: 06-22 08:01:57] {3072} INFO - at 13.7s,\testimator catboost's best error=0.3615,\tbest estimator xgboost's best error=0.3559\n",
"[flaml.automl: 08-22 21:32:16] {1358} INFO - iteration 21, current learner xgboost\n", "[flaml.automl: 06-22 08:01:57] {2897} INFO - iteration 20, current learner catboost\n",
"[flaml.automl: 08-22 21:32:16] {1515} INFO - at 2.7s,\tbest xgboost's error=0.3553,\tbest xgboost's error=0.3553\n", "[flaml.automl: 06-22 08:01:57] {3072} INFO - at 13.9s,\testimator catboost's best error=0.3615,\tbest estimator xgboost's best error=0.3559\n",
"[flaml.automl: 08-22 21:32:16] {1358} INFO - iteration 22, current learner lgbm\n", "[flaml.automl: 06-22 08:01:57] {2897} INFO - iteration 21, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:16] {1515} INFO - at 2.8s,\tbest lgbm's error=0.3629,\tbest xgboost's error=0.3553\n", "[flaml.automl: 06-22 08:01:59] {3072} INFO - at 15.7s,\testimator xgboost's best error=0.3559,\tbest estimator xgboost's best error=0.3559\n",
"[flaml.automl: 08-22 21:32:16] {1358} INFO - iteration 23, current learner lgbm\n", "[flaml.automl: 06-22 08:01:59] {2897} INFO - iteration 22, current learner catboost\n",
"[flaml.automl: 08-22 21:32:16] {1515} INFO - at 2.9s,\tbest lgbm's error=0.3629,\tbest xgboost's error=0.3553\n", "[flaml.automl: 06-22 08:02:00] {3072} INFO - at 16.5s,\testimator catboost's best error=0.3489,\tbest estimator catboost's best error=0.3489\n",
"[flaml.automl: 08-22 21:32:16] {1358} INFO - iteration 24, current learner xgboost\n", "[flaml.automl: 06-22 08:02:00] {2897} INFO - iteration 23, current learner catboost\n",
"[flaml.automl: 08-22 21:32:17] {1515} INFO - at 3.1s,\tbest xgboost's error=0.3520,\tbest xgboost's error=0.3520\n", "[flaml.automl: 06-22 08:02:02] {3072} INFO - at 18.9s,\testimator catboost's best error=0.3489,\tbest estimator catboost's best error=0.3489\n",
"[flaml.automl: 08-22 21:32:17] {1358} INFO - iteration 25, current learner xgboost\n", "[flaml.automl: 06-22 08:02:02] {2897} INFO - iteration 24, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:17] {1515} INFO - at 3.3s,\tbest xgboost's error=0.3520,\tbest xgboost's error=0.3520\n", "[flaml.automl: 06-22 08:02:03] {3072} INFO - at 19.2s,\testimator lgbm's best error=0.3604,\tbest estimator catboost's best error=0.3489\n",
"[flaml.automl: 08-22 21:32:17] {1358} INFO - iteration 26, current learner lgbm\n", "[flaml.automl: 06-22 08:02:03] {2897} INFO - iteration 25, current learner catboost\n",
"[flaml.automl: 08-22 21:32:17] {1515} INFO - at 3.4s,\tbest lgbm's error=0.3573,\tbest xgboost's error=0.3520\n", "[flaml.automl: 06-22 08:02:03] {3072} INFO - at 20.0s,\testimator catboost's best error=0.3472,\tbest estimator catboost's best error=0.3472\n",
"[flaml.automl: 08-22 21:32:17] {1358} INFO - iteration 27, current learner lgbm\n", "[flaml.automl: 06-22 08:02:03] {2897} INFO - iteration 26, current learner catboost\n",
"[flaml.automl: 08-22 21:32:17] {1515} INFO - at 3.5s,\tbest lgbm's error=0.3573,\tbest xgboost's error=0.3520\n", "[flaml.automl: 06-22 08:02:06] {3072} INFO - at 22.2s,\testimator catboost's best error=0.3472,\tbest estimator catboost's best error=0.3472\n",
"[flaml.automl: 08-22 21:32:17] {1358} INFO - iteration 28, current learner xgboost\n", "[flaml.automl: 06-22 08:02:06] {2897} INFO - iteration 27, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:17] {1515} INFO - at 3.9s,\tbest xgboost's error=0.3520,\tbest xgboost's error=0.3520\n", "[flaml.automl: 06-22 08:02:06] {3072} INFO - at 22.6s,\testimator lgbm's best error=0.3604,\tbest estimator catboost's best error=0.3472\n",
"[flaml.automl: 08-22 21:32:17] {1358} INFO - iteration 29, current learner xgboost\n", "[flaml.automl: 06-22 08:02:06] {2897} INFO - iteration 28, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:18] {1515} INFO - at 4.1s,\tbest xgboost's error=0.3520,\tbest xgboost's error=0.3520\n", "[flaml.automl: 06-22 08:02:06] {3072} INFO - at 22.9s,\testimator lgbm's best error=0.3604,\tbest estimator catboost's best error=0.3472\n",
"[flaml.automl: 08-22 21:32:18] {1358} INFO - iteration 30, current learner xgboost\n", "[flaml.automl: 06-22 08:02:06] {2897} INFO - iteration 29, current learner catboost\n",
"[flaml.automl: 08-22 21:32:18] {1515} INFO - at 4.8s,\tbest xgboost's error=0.3485,\tbest xgboost's error=0.3485\n", "[flaml.automl: 06-22 08:02:07] {3072} INFO - at 23.6s,\testimator catboost's best error=0.3472,\tbest estimator catboost's best error=0.3472\n",
"[flaml.automl: 08-22 21:32:18] {1358} INFO - iteration 31, current learner lgbm\n", "[flaml.automl: 06-22 08:02:07] {2897} INFO - iteration 30, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:19] {1515} INFO - at 5.2s,\tbest lgbm's error=0.3573,\tbest xgboost's error=0.3485\n", "[flaml.automl: 06-22 08:02:09] {3072} INFO - at 25.4s,\testimator xgboost's best error=0.3548,\tbest estimator catboost's best error=0.3472\n",
"[flaml.automl: 08-22 21:32:19] {1358} INFO - iteration 32, current learner xgboost\n", "[flaml.automl: 06-22 08:02:09] {2897} INFO - iteration 31, current learner catboost\n",
"[flaml.automl: 08-22 21:32:19] {1515} INFO - at 5.7s,\tbest xgboost's error=0.3485,\tbest xgboost's error=0.3485\n", "[flaml.automl: 06-22 08:02:16] {3072} INFO - at 32.3s,\testimator catboost's best error=0.3388,\tbest estimator catboost's best error=0.3388\n",
"[flaml.automl: 08-22 21:32:19] {1358} INFO - iteration 33, current learner xgboost\n", "[flaml.automl: 06-22 08:02:16] {2897} INFO - iteration 32, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:20] {1515} INFO - at 6.6s,\tbest xgboost's error=0.3485,\tbest xgboost's error=0.3485\n", "[flaml.automl: 06-22 08:02:16] {3072} INFO - at 32.7s,\testimator lgbm's best error=0.3604,\tbest estimator catboost's best error=0.3388\n",
"[flaml.automl: 08-22 21:32:20] {1358} INFO - iteration 34, current learner lgbm\n", "[flaml.automl: 06-22 08:02:16] {2897} INFO - iteration 33, current learner catboost\n",
"[flaml.automl: 08-22 21:32:20] {1515} INFO - at 6.9s,\tbest lgbm's error=0.3481,\tbest lgbm's error=0.3481\n", "[flaml.automl: 06-22 08:02:22] {3072} INFO - at 38.5s,\testimator catboost's best error=0.3388,\tbest estimator catboost's best error=0.3388\n",
"[flaml.automl: 08-22 21:32:20] {1358} INFO - iteration 35, current learner lgbm\n", "[flaml.automl: 06-22 08:02:22] {2897} INFO - iteration 34, current learner catboost\n",
"[flaml.automl: 08-22 21:32:21] {1515} INFO - at 7.2s,\tbest lgbm's error=0.3481,\tbest lgbm's error=0.3481\n", "[flaml.automl: 06-22 08:02:43] {3072} INFO - at 59.6s,\testimator catboost's best error=0.3388,\tbest estimator catboost's best error=0.3388\n",
"[flaml.automl: 08-22 21:32:21] {1358} INFO - iteration 36, current learner lgbm\n", "[flaml.automl: 06-22 08:02:46] {3336} INFO - retrain catboost for 2.8s\n",
"[flaml.automl: 08-22 21:32:21] {1515} INFO - at 7.4s,\tbest lgbm's error=0.3481,\tbest lgbm's error=0.3481\n", "[flaml.automl: 06-22 08:02:46] {3343} INFO - retrained model: <catboost.core.CatBoostClassifier object at 0x7fbeeb3859d0>\n",
"[flaml.automl: 08-22 21:32:21] {1358} INFO - iteration 37, current learner xgboost\n", "[flaml.automl: 06-22 08:02:46] {2636} INFO - fit succeeded\n",
"[flaml.automl: 08-22 21:32:22] {1515} INFO - at 8.2s,\tbest xgboost's error=0.3485,\tbest lgbm's error=0.3481\n", "[flaml.automl: 06-22 08:02:46] {2637} INFO - Time taken to find the best model: 32.311296463012695\n"
"[flaml.automl: 08-22 21:32:22] {1358} INFO - iteration 38, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:22] {1515} INFO - at 8.5s,\tbest lgbm's error=0.3481,\tbest lgbm's error=0.3481\n",
"[flaml.automl: 08-22 21:32:22] {1358} INFO - iteration 39, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:22] {1515} INFO - at 8.8s,\tbest lgbm's error=0.3481,\tbest lgbm's error=0.3481\n",
"[flaml.automl: 08-22 21:32:22] {1358} INFO - iteration 40, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:23] {1515} INFO - at 9.7s,\tbest xgboost's error=0.3485,\tbest lgbm's error=0.3481\n",
"[flaml.automl: 08-22 21:32:23] {1358} INFO - iteration 41, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:25] {1515} INFO - at 11.7s,\tbest lgbm's error=0.3481,\tbest lgbm's error=0.3481\n",
"[flaml.automl: 08-22 21:32:25] {1358} INFO - iteration 42, current learner catboost\n",
"[flaml.automl: 08-22 21:32:26] {1515} INFO - at 12.2s,\tbest catboost's error=0.3647,\tbest lgbm's error=0.3481\n",
"[flaml.automl: 08-22 21:32:26] {1358} INFO - iteration 43, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:28] {1515} INFO - at 14.4s,\tbest lgbm's error=0.3427,\tbest lgbm's error=0.3427\n",
"[flaml.automl: 08-22 21:32:28] {1358} INFO - iteration 44, current learner catboost\n",
"[flaml.automl: 08-22 21:32:28] {1515} INFO - at 14.6s,\tbest catboost's error=0.3647,\tbest lgbm's error=0.3427\n",
"[flaml.automl: 08-22 21:32:28] {1358} INFO - iteration 45, current learner catboost\n",
"[flaml.automl: 08-22 21:32:28] {1515} INFO - at 14.8s,\tbest catboost's error=0.3601,\tbest lgbm's error=0.3427\n",
"[flaml.automl: 08-22 21:32:28] {1358} INFO - iteration 46, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:30] {1515} INFO - at 16.9s,\tbest lgbm's error=0.3427,\tbest lgbm's error=0.3427\n",
"[flaml.automl: 08-22 21:32:30] {1358} INFO - iteration 47, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:34] {1515} INFO - at 21.0s,\tbest xgboost's error=0.3332,\tbest xgboost's error=0.3332\n",
"[flaml.automl: 08-22 21:32:34] {1358} INFO - iteration 48, current learner catboost\n",
"[flaml.automl: 08-22 21:32:35] {1515} INFO - at 21.1s,\tbest catboost's error=0.3601,\tbest xgboost's error=0.3332\n",
"[flaml.automl: 08-22 21:32:35] {1358} INFO - iteration 49, current learner lgbm\n",
"[flaml.automl: 08-22 21:32:37] {1515} INFO - at 23.2s,\tbest lgbm's error=0.3409,\tbest xgboost's error=0.3332\n",
"[flaml.automl: 08-22 21:32:37] {1358} INFO - iteration 50, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:38] {1515} INFO - at 24.6s,\tbest xgboost's error=0.3332,\tbest xgboost's error=0.3332\n",
"[flaml.automl: 08-22 21:32:38] {1358} INFO - iteration 51, current learner xgboost\n",
"[flaml.automl: 08-22 21:32:53] {1515} INFO - at 40.0s,\tbest xgboost's error=0.3279,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:32:53] {1358} INFO - iteration 52, current learner xgboost\n",
"[flaml.automl: 08-22 21:33:01] {1515} INFO - at 47.6s,\tbest xgboost's error=0.3279,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:01] {1358} INFO - iteration 53, current learner catboost\n",
"[flaml.automl: 08-22 21:33:01] {1515} INFO - at 47.7s,\tbest catboost's error=0.3601,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:01] {1358} INFO - iteration 54, current learner catboost\n",
"[flaml.automl: 08-22 21:33:02] {1515} INFO - at 48.2s,\tbest catboost's error=0.3601,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:02] {1358} INFO - iteration 55, current learner catboost\n",
"[flaml.automl: 08-22 21:33:02] {1515} INFO - at 48.5s,\tbest catboost's error=0.3552,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:02] {1358} INFO - iteration 56, current learner catboost\n",
"[flaml.automl: 08-22 21:33:02] {1515} INFO - at 48.7s,\tbest catboost's error=0.3552,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:02] {1358} INFO - iteration 57, current learner catboost\n",
"[flaml.automl: 08-22 21:33:02] {1515} INFO - at 49.0s,\tbest catboost's error=0.3552,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:02] {1358} INFO - iteration 58, current learner catboost\n",
"[flaml.automl: 08-22 21:33:03] {1515} INFO - at 49.1s,\tbest catboost's error=0.3552,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:03] {1358} INFO - iteration 59, current learner catboost\n",
"[flaml.automl: 08-22 21:33:03] {1515} INFO - at 49.4s,\tbest catboost's error=0.3552,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:03] {1358} INFO - iteration 60, current learner catboost\n",
"[flaml.automl: 08-22 21:33:06] {1515} INFO - at 52.2s,\tbest catboost's error=0.3453,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:06] {1358} INFO - iteration 61, current learner catboost\n",
"[flaml.automl: 08-22 21:33:07] {1515} INFO - at 53.9s,\tbest catboost's error=0.3453,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:07] {1358} INFO - iteration 62, current learner catboost\n",
"[flaml.automl: 08-22 21:33:09] {1515} INFO - at 55.3s,\tbest catboost's error=0.3453,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:09] {1358} INFO - iteration 63, current learner catboost\n",
"[flaml.automl: 08-22 21:33:10] {1515} INFO - at 56.4s,\tbest catboost's error=0.3453,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:10] {1358} INFO - iteration 64, current learner catboost\n",
"[flaml.automl: 08-22 21:33:11] {1515} INFO - at 57.5s,\tbest catboost's error=0.3453,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:11] {1358} INFO - iteration 65, current learner lgbm\n",
"[flaml.automl: 08-22 21:33:13] {1515} INFO - at 59.8s,\tbest lgbm's error=0.3409,\tbest xgboost's error=0.3279\n",
"[flaml.automl: 08-22 21:33:13] {1592} INFO - selected model: XGBClassifier(base_score=0.5, booster='gbtree',\n",
" colsample_bylevel=0.810466508891351, colsample_bynode=1,\n",
" colsample_bytree=0.8005378817953572, gamma=0, gpu_id=-1,\n",
" grow_policy='lossguide', importance_type='gain',\n",
" interaction_constraints='', learning_rate=0.06234183309508761,\n",
" max_delta_step=0, max_depth=0, max_leaves=1797,\n",
" min_child_weight=0.07275175679381725, missing=nan,\n",
" monotone_constraints='()', n_estimators=63, n_jobs=-1,\n",
" num_parallel_tree=1, random_state=0, reg_alpha=0.5768305704485758,\n",
" reg_lambda=6.867180836557797, scale_pos_weight=1,\n",
" subsample=0.9814772488195874, tree_method='hist',\n",
" use_label_encoder=False, validate_parameters=1, verbosity=0)\n",
"[flaml.automl: 08-22 21:33:26] {1633} INFO - retrain xgboost for 13.0s\n",
"[flaml.automl: 08-22 21:33:26] {1636} INFO - retrained model: XGBClassifier(base_score=0.5, booster='gbtree',\n",
" colsample_bylevel=0.810466508891351, colsample_bynode=1,\n",
" colsample_bytree=0.8005378817953572, gamma=0, gpu_id=-1,\n",
" grow_policy='lossguide', importance_type='gain',\n",
" interaction_constraints='', learning_rate=0.06234183309508761,\n",
" max_delta_step=0, max_depth=0, max_leaves=1797,\n",
" min_child_weight=0.07275175679381725, missing=nan,\n",
" monotone_constraints='()', n_estimators=63, n_jobs=-1,\n",
" num_parallel_tree=1, random_state=0, reg_alpha=0.5768305704485758,\n",
" reg_lambda=6.867180836557797, scale_pos_weight=1,\n",
" subsample=0.9814772488195874, tree_method='hist',\n",
" use_label_encoder=False, validate_parameters=1, verbosity=0)\n",
"[flaml.automl: 08-22 21:33:26] {1199} INFO - fit succeeded\n",
"[flaml.automl: 08-22 21:33:26] {1200} INFO - Time taken to find the best model: 40.023393869400024\n"
] ]
}, },
{ {
"data": { "data": {
"text/html": [ "text/html": [
"<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"b994edf1-5e76-4cd3-b719-4a204af673dc\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"b994edf1-5e76-4cd3-b719-4a204af673dc\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[('imputuer', SimpleImputer()),\n", "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;imputuer&#x27;, SimpleImputer()),\n",
" ('standardizer', StandardScaler()),\n", " (&#x27;standardizer&#x27;, StandardScaler()),\n",
" ('automl', <flaml.automl.AutoML object at 0x7f046d56fb50>)])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"c94ee64a-d8b1-4cbb-aeca-952bf6963c13\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"c94ee64a-d8b1-4cbb-aeca-952bf6963c13\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"6a28d11a-19e2-4243-8b85-e3ba5f6f2a7e\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"6a28d11a-19e2-4243-8b85-e3ba5f6f2a7e\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"03dcbe59-a8be-4f09-a944-115d90939f81\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"03dcbe59-a8be-4f09-a944-115d90939f81\">AutoML</label><div class=\"sk-toggleable__content\"><pre><flaml.automl.AutoML object at 0x7f046d56fb50></pre></div></div></div></div></div></div></div>" " (&#x27;automl&#x27;,\n",
" AutoML(append_log=False, auto_augment=True, custom_hp={},\n",
" early_stop=False, ensemble=False, estimator_list=&#x27;auto&#x27;,\n",
" eval_method=&#x27;auto&#x27;, fit_kwargs_by_estimator={},\n",
" hpo_method=&#x27;auto&#x27;, keep_search_state=False,\n",
" learner_selector=&#x27;sample&#x27;, log_file_name=&#x27;&#x27;,\n",
" log_training_metric=False, log_type=&#x27;better&#x27;,\n",
" max_iter=None, mem_thres=4294967296, metric=&#x27;auto&#x27;,\n",
" metric_constraints=[], min_sample_size=10000,\n",
" model_history=False, n_concurrent_trials=1, n_jobs=-1,\n",
" n_splits=5, pred_time_limit=inf, retrain_full=True,\n",
" sample=True, split_ratio=0.1, split_type=&#x27;auto&#x27;,\n",
" starting_points=&#x27;static&#x27;, task=&#x27;classification&#x27;, ...))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;imputuer&#x27;, SimpleImputer()),\n",
" (&#x27;standardizer&#x27;, StandardScaler()),\n",
" (&#x27;automl&#x27;,\n",
" AutoML(append_log=False, auto_augment=True, custom_hp={},\n",
" early_stop=False, ensemble=False, estimator_list=&#x27;auto&#x27;,\n",
" eval_method=&#x27;auto&#x27;, fit_kwargs_by_estimator={},\n",
" hpo_method=&#x27;auto&#x27;, keep_search_state=False,\n",
" learner_selector=&#x27;sample&#x27;, log_file_name=&#x27;&#x27;,\n",
" log_training_metric=False, log_type=&#x27;better&#x27;,\n",
" max_iter=None, mem_thres=4294967296, metric=&#x27;auto&#x27;,\n",
" metric_constraints=[], min_sample_size=10000,\n",
" model_history=False, n_concurrent_trials=1, n_jobs=-1,\n",
" n_splits=5, pred_time_limit=inf, retrain_full=True,\n",
" sample=True, split_ratio=0.1, split_type=&#x27;auto&#x27;,\n",
" starting_points=&#x27;static&#x27;, task=&#x27;classification&#x27;, ...))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">AutoML</label><div class=\"sk-toggleable__content\"><pre>AutoML(append_log=False, auto_augment=True, custom_hp={}, early_stop=False,\n",
" ensemble=False, estimator_list=&#x27;auto&#x27;, eval_method=&#x27;auto&#x27;,\n",
" fit_kwargs_by_estimator={}, hpo_method=&#x27;auto&#x27;, keep_search_state=False,\n",
" learner_selector=&#x27;sample&#x27;, log_file_name=&#x27;&#x27;, log_training_metric=False,\n",
" log_type=&#x27;better&#x27;, max_iter=None, mem_thres=4294967296, metric=&#x27;auto&#x27;,\n",
" metric_constraints=[], min_sample_size=10000, model_history=False,\n",
" n_concurrent_trials=1, n_jobs=-1, n_splits=5, pred_time_limit=inf,\n",
" retrain_full=True, sample=True, split_ratio=0.1, split_type=&#x27;auto&#x27;,\n",
" starting_points=&#x27;static&#x27;, task=&#x27;classification&#x27;, ...)</pre></div></div></div></div></div></div></div>"
], ],
"text/plain": [ "text/plain": [
"Pipeline(steps=[('imputuer', SimpleImputer()),\n", "Pipeline(steps=[('imputuer', SimpleImputer()),\n",
" ('standardizer', StandardScaler()),\n", " ('standardizer', StandardScaler()),\n",
" ('automl', <flaml.automl.AutoML object at 0x7f046d56fb50>)])" " ('automl',\n",
" AutoML(append_log=False, auto_augment=True, custom_hp={},\n",
" early_stop=False, ensemble=False, estimator_list='auto',\n",
" eval_method='auto', fit_kwargs_by_estimator={},\n",
" hpo_method='auto', keep_search_state=False,\n",
" learner_selector='sample', log_file_name='',\n",
" log_training_metric=False, log_type='better',\n",
" max_iter=None, mem_thres=4294967296, metric='auto',\n",
" metric_constraints=[], min_sample_size=10000,\n",
" model_history=False, n_concurrent_trials=1, n_jobs=-1,\n",
" n_splits=5, pred_time_limit=inf, retrain_full=True,\n",
" sample=True, split_ratio=0.1, split_type='auto',\n",
" starting_points='static', task='classification', ...))])"
] ]
}, },
"execution_count": 8, "execution_count": 5,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
], ],
"source": [ "source": [
"automl_pipeline.fit(X_train, y_train, \n", "automl_pipeline.fit(X_train, y_train, **pipeline_settings)"
" automl__time_budget=settings['time_budget'],\n",
" automl__metric=settings['metric'],\n",
" automl__estimator_list=settings['estimator_list'],\n",
" automl__log_training_metric=True)"
] ]
}, },
{ {
@ -500,11 +505,9 @@
} }
], ],
"metadata": { "metadata": {
"interpreter": {
"hash": "0cfea3304185a9579d09e0953576b57c8581e46e6ebc6dfeb681bc5a511f7544"
},
"kernelspec": { "kernelspec": {
"display_name": "Python 3.8.0 64-bit ('blend': conda)", "display_name": "Python 3.9.12 64-bit",
"language": "python",
"name": "python3" "name": "python3"
}, },
"language_info": { "language_info": {
@ -517,7 +520,12 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.0" "version": "3.9.12"
},
"vscode": {
"interpreter": {
"hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
}
} }
}, },
"nbformat": 4, "nbformat": 4,

17
website/docs/Examples/Integrate - Scikit-learn Pipeline.md
View File

@ -37,16 +37,17 @@ automl_pipeline
### Run AutoML in the pipeline ### Run AutoML in the pipeline
```python ```python
settings = { automl_settings = {
"time_budget": 60, # total running time in seconds "time_budget": 60, # total running time in seconds
"metric": 'accuracy', # primary metrics can be chosen from: ['accuracy','roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'f1','log_loss','mae','mse','r2'] "metric": "accuracy", # primary metrics can be chosen from: ['accuracy','roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'f1','log_loss','mae','mse','r2']
"task": 'classification', # task type "task": "classification", # task type
"estimator_list":['xgboost','catboost','lgbm'], "estimator_list": ["xgboost", "catboost", "lgbm"],
"log_file_name": 'airlines_experiment.log', # flaml log file "log_file_name": "airlines_experiment.log", # flaml log file
} }
automl_pipeline.fit(X_train, y_train, pipeline_settings = {
automl__time_budget=60, f"automl__{key}": value for key, value in automl_settings.items()
automl__metric="accuracy") }
automl_pipeline.fit(X_train, y_train, **pipeline_settings)
``` ```
### Get the automl object from the pipeline ### Get the automl object from the pipeline