mirror of
https://github.com/microsoft/autogen.git
synced 2025-10-20 20:39:37 +00:00
27b2712016
* Refactor into automl subpackage Moved some of the packages into an automl subpackage to tidy before the task-based refactor. This is in response to discussions with the group and a comment on the first task-based PR. Only changes here are moving subpackages and modules into the new automl, fixing imports to work with this structure and fixing some dependencies in setup.py. * Fix doc building post automl subpackage refactor * Fix broken links in website post automl subpackage refactor * Fix broken links in website post automl subpackage refactor * Remove vw from test deps as this is breaking the build * Move default back to the top-level I'd moved this to automl as that's where it's used internally, but had missed that this is actually part of the public interface so makes sense to live where it was. * Re-add top level modules with deprecation warnings flaml.data, flaml.ml and flaml.model are re-added to the top level, being re-exported from flaml.automl for backwards compatability. Adding a deprecation warning so that we can have a planned removal later. * Fix model.py line-endings * WIP * WIP - Notes below Got to the point where the methods from AutoML are pulled to GenericTask. Started removing private markers and removing the passing of automl to these methods. Done with decide_split_type, started on prepare_data. Need to do the others after * Re-add generic_task * Fix tests: add Task.__str__ * Fix tests: test for ray.ObjectRef * Hotwire TS_Sklearn wrapper to fix test fail * Remove unused data size field from Task * Fix import for CLASSIFICATION in notebook * Update flaml/automl/data.py Co-authored-by: Chi Wang <wang.chi@microsoft.com> * Fix review comments * Fix task -> str in custom learner constructor * Remove unused CLASSIFICATION imports * Hotwire TS_Sklearn wrapper to fix test fail by setting optimizer_for_horizon == False * Revert changes to the automl_classification and pin FLAML version * Fix imports in reverted notebook * Fix FLAML version in automl notebooks * Fix ml.py line endings * Fix CLASSIFICATION task import in automl_classification notebook * Uncomment pip install in notebook and revert import Not convinced this will work because of installing an older version of the package into the environment in which we're running the tests, but let's see. * Revert c6a5dd1a0 * Revert "Revert c6a5dd1a0" This reverts commit e55e35adea03993de87b23f092b14c6af623d487. * Black format model.py * Bump version to 1.1.2 in automl_xgboost * Add docstrings to the Task ABC * Fix import in custom_learner * fix 'optimize_for_horizon' for ts_sklearn * remove debugging print statements * Check for is_forecast() before is_classification() in decide_split_type * Attempt to fix formatting fail * Another attempt to fix formatting fail * And another attempt to fix formatting fail * Add type annotations for task arg in signatures and docstrings * Fix formatting * Fix linting --------- Co-authored-by: Qingyun Wu <qingyun.wu@psu.edu> Co-authored-by: EgorKraevTransferwise <egor.kraev@transferwise.com> Co-authored-by: Chi Wang <wang.chi@microsoft.com> Co-authored-by: Kevin Chen <chenkevin.8787@gmail.com>
471 lines
18 KiB
Python
471 lines
18 KiB
Python
import unittest
|
|
import numpy as np
|
|
import scipy.sparse
|
|
from sklearn.datasets import load_iris, load_wine
|
|
from flaml import AutoML
|
|
from flaml.automl.data import get_output_from_log
|
|
from flaml.automl.training_log import training_log_reader
|
|
from flaml.tune.spark.utils import check_spark
|
|
import os
|
|
|
|
spark_available, _ = check_spark()
|
|
skip_spark = not spark_available
|
|
|
|
os.environ["FLAML_MAX_CONCURRENT"] = "2"
|
|
|
|
# To solve pylint issue, we put code for customizing mylearner in a separate file
|
|
if os.path.exists(os.path.join(os.getcwd(), "test", "spark", "custom_mylearner.py")):
|
|
try:
|
|
from test.spark.custom_mylearner import *
|
|
from flaml.tune.spark.mylearner import (
|
|
MyRegularizedGreedyForest,
|
|
custom_metric,
|
|
MyLargeLGBM,
|
|
MyLargeXGB,
|
|
)
|
|
|
|
skip_my_learner = False
|
|
except ImportError:
|
|
skip_my_learner = True
|
|
else:
|
|
skip_my_learner = True
|
|
|
|
|
|
class TestMultiClass(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
if skip_spark:
|
|
self.skipTest("Spark is not installed. Skip all spark tests.")
|
|
|
|
@unittest.skipIf(
|
|
skip_my_learner,
|
|
"Please run pytest in the root directory of FLAML, i.e., the directory that contains the setup.py file",
|
|
)
|
|
def test_custom_learner(self):
|
|
automl = AutoML()
|
|
automl.add_learner(learner_name="RGF", learner_class=MyRegularizedGreedyForest)
|
|
X_train, y_train = load_wine(return_X_y=True)
|
|
settings = {
|
|
"time_budget": 8, # total running time in seconds
|
|
"estimator_list": ["RGF", "lgbm", "rf", "xgboost"],
|
|
"task": "classification", # task type
|
|
"sample": True, # whether to subsample training data
|
|
"log_file_name": "test/wine.log",
|
|
"log_training_metric": True, # whether to log training metric
|
|
"n_jobs": 1,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
"verbose": 4,
|
|
}
|
|
automl.fit(X_train=X_train, y_train=y_train, **settings)
|
|
# print the best model found for RGF
|
|
print(automl.best_model_for_estimator("RGF"))
|
|
|
|
MyRegularizedGreedyForest.search_space = lambda data_size, task: {}
|
|
automl.fit(X_train=X_train, y_train=y_train, **settings)
|
|
|
|
@unittest.skipIf(
|
|
skip_my_learner,
|
|
"Please run pytest in the root directory of FLAML, i.e., the directory that contains the setup.py file",
|
|
)
|
|
def test_custom_metric(self):
|
|
df, y = load_iris(return_X_y=True, as_frame=True)
|
|
df["label"] = y
|
|
automl_experiment = AutoML()
|
|
automl_settings = {
|
|
"dataframe": df,
|
|
"label": "label",
|
|
"time_budget": 5,
|
|
"eval_method": "cv",
|
|
"metric": custom_metric,
|
|
"task": "classification",
|
|
"log_file_name": "test/iris_custom.log",
|
|
"log_training_metric": True,
|
|
"log_type": "all",
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"sample_weight": np.ones(len(y)),
|
|
"pred_time_limit": 1e-5,
|
|
# "ensemble": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
automl_experiment.fit(**automl_settings)
|
|
print(automl_experiment.classes_)
|
|
print(automl_experiment.model)
|
|
print(automl_experiment.config_history)
|
|
print(automl_experiment.best_model_for_estimator("rf"))
|
|
print(automl_experiment.best_iteration)
|
|
print(automl_experiment.best_estimator)
|
|
automl_experiment = AutoML()
|
|
estimator = automl_experiment.get_estimator_from_log(
|
|
automl_settings["log_file_name"], record_id=0, task="multiclass"
|
|
)
|
|
print(estimator)
|
|
(
|
|
time_history,
|
|
best_valid_loss_history,
|
|
valid_loss_history,
|
|
config_history,
|
|
metric_history,
|
|
) = get_output_from_log(
|
|
filename=automl_settings["log_file_name"], time_budget=6
|
|
)
|
|
print(metric_history)
|
|
|
|
def test_classification(self, as_frame=False):
|
|
automl_experiment = AutoML()
|
|
automl_settings = {
|
|
"time_budget": 4,
|
|
"metric": "accuracy",
|
|
"task": "classification",
|
|
"log_file_name": "test/iris.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True, as_frame=as_frame)
|
|
if as_frame:
|
|
# test drop column
|
|
X_train.columns = range(X_train.shape[1])
|
|
X_train[X_train.shape[1]] = np.zeros(len(y_train))
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
print(automl_experiment.classes_)
|
|
print(automl_experiment.predict(X_train)[:5])
|
|
print(automl_experiment.model)
|
|
print(automl_experiment.config_history)
|
|
print(automl_experiment.best_model_for_estimator("catboost"))
|
|
print(automl_experiment.best_iteration)
|
|
print(automl_experiment.best_estimator)
|
|
del automl_settings["metric"]
|
|
del automl_settings["model_history"]
|
|
del automl_settings["log_training_metric"]
|
|
automl_experiment = AutoML(task="classification")
|
|
duration = automl_experiment.retrain_from_log(
|
|
log_file_name=automl_settings["log_file_name"],
|
|
X_train=X_train,
|
|
y_train=y_train,
|
|
train_full=True,
|
|
record_id=0,
|
|
)
|
|
print(duration)
|
|
print(automl_experiment.model)
|
|
print(automl_experiment.predict_proba(X_train)[:5])
|
|
|
|
def test_micro_macro_f1(self):
|
|
automl_experiment_micro = AutoML()
|
|
automl_experiment_macro = AutoML()
|
|
automl_settings = {
|
|
"time_budget": 2,
|
|
"task": "classification",
|
|
"log_file_name": "test/micro_macro_f1.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True)
|
|
automl_experiment_micro.fit(
|
|
X_train=X_train, y_train=y_train, metric="micro_f1", **automl_settings
|
|
)
|
|
automl_experiment_macro.fit(
|
|
X_train=X_train, y_train=y_train, metric="macro_f1", **automl_settings
|
|
)
|
|
estimator = automl_experiment_macro.model
|
|
y_pred = estimator.predict(X_train)
|
|
y_pred_proba = estimator.predict_proba(X_train)
|
|
from flaml.automl.ml import norm_confusion_matrix, multi_class_curves
|
|
|
|
print(norm_confusion_matrix(y_train, y_pred))
|
|
from sklearn.metrics import roc_curve, precision_recall_curve
|
|
|
|
print(multi_class_curves(y_train, y_pred_proba, roc_curve))
|
|
print(multi_class_curves(y_train, y_pred_proba, precision_recall_curve))
|
|
|
|
def test_roc_auc_ovr(self):
|
|
automl_experiment = AutoML()
|
|
X_train, y_train = load_iris(return_X_y=True)
|
|
automl_settings = {
|
|
"time_budget": 1,
|
|
"metric": "roc_auc_ovr",
|
|
"task": "classification",
|
|
"log_file_name": "test/roc_auc_ovr.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"sample_weight": np.ones(len(y_train)),
|
|
"eval_method": "holdout",
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
|
|
def test_roc_auc_ovo(self):
|
|
automl_experiment = AutoML()
|
|
automl_settings = {
|
|
"time_budget": 1,
|
|
"metric": "roc_auc_ovo",
|
|
"task": "classification",
|
|
"log_file_name": "test/roc_auc_ovo.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True)
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
|
|
def test_roc_auc_ovr_weighted(self):
|
|
automl_experiment = AutoML()
|
|
automl_settings = {
|
|
"time_budget": 1,
|
|
"metric": "roc_auc_ovr_weighted",
|
|
"task": "classification",
|
|
"log_file_name": "test/roc_auc_weighted.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True)
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
|
|
def test_roc_auc_ovo_weighted(self):
|
|
automl_experiment = AutoML()
|
|
automl_settings = {
|
|
"time_budget": 1,
|
|
"metric": "roc_auc_ovo_weighted",
|
|
"task": "classification",
|
|
"log_file_name": "test/roc_auc_weighted.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True)
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
|
|
def test_sparse_matrix_classification(self):
|
|
automl_experiment = AutoML()
|
|
automl_settings = {
|
|
"time_budget": 2,
|
|
"metric": "auto",
|
|
"task": "classification",
|
|
"log_file_name": "test/sparse_classification.log",
|
|
"split_type": "uniform",
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train = scipy.sparse.random(1554, 21, dtype=int)
|
|
y_train = np.random.randint(3, size=1554)
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
print(automl_experiment.classes_)
|
|
print(automl_experiment.predict_proba(X_train))
|
|
print(automl_experiment.model)
|
|
print(automl_experiment.config_history)
|
|
print(automl_experiment.best_model_for_estimator("extra_tree"))
|
|
print(automl_experiment.best_iteration)
|
|
print(automl_experiment.best_estimator)
|
|
|
|
@unittest.skipIf(
|
|
skip_my_learner,
|
|
"Please run pytest in the root directory of FLAML, i.e., the directory that contains the setup.py file",
|
|
)
|
|
def _test_memory_limit(self):
|
|
automl_experiment = AutoML()
|
|
automl_experiment.add_learner(
|
|
learner_name="large_lgbm", learner_class=MyLargeLGBM
|
|
)
|
|
automl_settings = {
|
|
"time_budget": -1,
|
|
"task": "classification",
|
|
"log_file_name": "test/classification_oom.log",
|
|
"estimator_list": ["large_lgbm"],
|
|
"log_type": "all",
|
|
"hpo_method": "random",
|
|
"free_mem_ratio": 0.2,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True, as_frame=True)
|
|
|
|
automl_experiment.fit(
|
|
X_train=X_train, y_train=y_train, max_iter=1, **automl_settings
|
|
)
|
|
print(automl_experiment.model)
|
|
|
|
@unittest.skipIf(
|
|
skip_my_learner,
|
|
"Please run pytest in the root directory of FLAML, i.e., the directory that contains the setup.py file",
|
|
)
|
|
def test_time_limit(self):
|
|
automl_experiment = AutoML()
|
|
automl_experiment.add_learner(
|
|
learner_name="large_lgbm", learner_class=MyLargeLGBM
|
|
)
|
|
automl_experiment.add_learner(
|
|
learner_name="large_xgb", learner_class=MyLargeXGB
|
|
)
|
|
automl_settings = {
|
|
"time_budget": 0.5,
|
|
"task": "classification",
|
|
"log_file_name": "test/classification_timeout.log",
|
|
"estimator_list": ["catboost"],
|
|
"log_type": "all",
|
|
"hpo_method": "random",
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True, as_frame=True)
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
print(automl_experiment.model.params)
|
|
automl_settings["estimator_list"] = ["large_xgb"]
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
print(automl_experiment.model)
|
|
automl_settings["estimator_list"] = ["large_lgbm"]
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
print(automl_experiment.model)
|
|
|
|
def test_fit_w_starting_point(self, as_frame=True):
|
|
automl_experiment = AutoML()
|
|
automl_settings = {
|
|
"time_budget": 3,
|
|
"metric": "accuracy",
|
|
"task": "classification",
|
|
"log_file_name": "test/iris.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True, as_frame=as_frame)
|
|
if as_frame:
|
|
# test drop column
|
|
X_train.columns = range(X_train.shape[1])
|
|
X_train[X_train.shape[1]] = np.zeros(len(y_train))
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
automl_val_accuracy = 1.0 - automl_experiment.best_loss
|
|
print("Best ML leaner:", automl_experiment.best_estimator)
|
|
print("Best hyperparmeter config:", automl_experiment.best_config)
|
|
print("Best accuracy on validation data: {0:.4g}".format(automl_val_accuracy))
|
|
print(
|
|
"Training duration of best run: {0:.4g} s".format(
|
|
automl_experiment.best_config_train_time
|
|
)
|
|
)
|
|
|
|
starting_points = automl_experiment.best_config_per_estimator
|
|
print("starting_points", starting_points)
|
|
print("loss of the starting_points", automl_experiment.best_loss_per_estimator)
|
|
automl_settings_resume = {
|
|
"time_budget": 2,
|
|
"metric": "accuracy",
|
|
"task": "classification",
|
|
"log_file_name": "test/iris_resume.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"log_type": "all",
|
|
"starting_points": starting_points,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
new_automl_experiment = AutoML()
|
|
new_automl_experiment.fit(
|
|
X_train=X_train, y_train=y_train, **automl_settings_resume
|
|
)
|
|
|
|
new_automl_val_accuracy = 1.0 - new_automl_experiment.best_loss
|
|
print("Best ML leaner:", new_automl_experiment.best_estimator)
|
|
print("Best hyperparmeter config:", new_automl_experiment.best_config)
|
|
print(
|
|
"Best accuracy on validation data: {0:.4g}".format(new_automl_val_accuracy)
|
|
)
|
|
print(
|
|
"Training duration of best run: {0:.4g} s".format(
|
|
new_automl_experiment.best_config_train_time
|
|
)
|
|
)
|
|
|
|
def test_fit_w_starting_points_list(self, as_frame=True):
|
|
automl_experiment = AutoML()
|
|
automl_settings = {
|
|
"time_budget": 3,
|
|
"metric": "accuracy",
|
|
"task": "classification",
|
|
"log_file_name": "test/iris.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"model_history": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
X_train, y_train = load_iris(return_X_y=True, as_frame=as_frame)
|
|
if as_frame:
|
|
# test drop column
|
|
X_train.columns = range(X_train.shape[1])
|
|
X_train[X_train.shape[1]] = np.zeros(len(y_train))
|
|
automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings)
|
|
automl_val_accuracy = 1.0 - automl_experiment.best_loss
|
|
print("Best ML leaner:", automl_experiment.best_estimator)
|
|
print("Best hyperparmeter config:", automl_experiment.best_config)
|
|
print("Best accuracy on validation data: {0:.4g}".format(automl_val_accuracy))
|
|
print(
|
|
"Training duration of best run: {0:.4g} s".format(
|
|
automl_experiment.best_config_train_time
|
|
)
|
|
)
|
|
|
|
starting_points = {}
|
|
log_file_name = automl_settings["log_file_name"]
|
|
with training_log_reader(log_file_name) as reader:
|
|
sample_size = 1000
|
|
for record in reader.records():
|
|
config = record.config
|
|
config["FLAML_sample_size"] = sample_size
|
|
sample_size += 1000
|
|
learner = record.learner
|
|
if learner not in starting_points:
|
|
starting_points[learner] = []
|
|
starting_points[learner].append(config)
|
|
max_iter = sum([len(s) for k, s in starting_points.items()])
|
|
automl_settings_resume = {
|
|
"time_budget": 2,
|
|
"metric": "accuracy",
|
|
"task": "classification",
|
|
"log_file_name": "test/iris_resume_all.log",
|
|
"log_training_metric": True,
|
|
"n_jobs": 1,
|
|
"max_iter": max_iter,
|
|
"model_history": True,
|
|
"log_type": "all",
|
|
"starting_points": starting_points,
|
|
"append_log": True,
|
|
"n_concurrent_trials": 2,
|
|
"use_spark": True,
|
|
}
|
|
new_automl_experiment = AutoML()
|
|
new_automl_experiment.fit(
|
|
X_train=X_train, y_train=y_train, **automl_settings_resume
|
|
)
|
|
|
|
new_automl_val_accuracy = 1.0 - new_automl_experiment.best_loss
|
|
# print('Best ML leaner:', new_automl_experiment.best_estimator)
|
|
# print('Best hyperparmeter config:', new_automl_experiment.best_config)
|
|
print(
|
|
"Best accuracy on validation data: {0:.4g}".format(new_automl_val_accuracy)
|
|
)
|
|
# print('Training duration of best run: {0:.4g} s'.format(new_automl_experiment.best_config_train_time))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|