diff --git a/flaml/automl.py b/flaml/automl.py index a0e173409..eb084c1e3 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -495,6 +495,7 @@ class AutoML(BaseEstimator): metric for each model. mem_thres: A float of the memory size constraint in bytes. pred_time_limit: A float of the prediction latency constraint in seconds. + It refers to the average prediction time per row in validation data. train_time_limit: A float of the training time constraint in seconds. verbose: int, default=3 | Controls the verbosity, higher means more messages. @@ -1751,6 +1752,7 @@ class AutoML(BaseEstimator): metric for each model. mem_thres: A float of the memory size constraint in bytes. pred_time_limit: A float of the prediction latency constraint in seconds. + It refers to the average prediction time per row in validation data. train_time_limit: A float of the training time constraint in seconds. X_val: None or a numpy array or a pandas dataframe of validation data. y_val: None or a numpy array or a pandas series of validation labels. diff --git a/flaml/ml.py b/flaml/ml.py index 0441691b4..8293455a1 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -206,7 +206,7 @@ def _eval_estimator( val_loss = sklearn_metric_loss_score( eval_metric, val_pred_y, y_val, labels, weight_val, groups_val ) - metric_for_logging = {} + metric_for_logging = {"pred_time": pred_time} if log_training_metric: train_pred_y = get_y_pred(estimator, X_train, eval_metric, obj) metric_for_logging["train_loss"] = sklearn_metric_loss_score( diff --git a/test/automl/test_notebook_example.py b/test/automl/test_notebook_example.py index 7ef672296..515d569b1 100644 --- a/test/automl/test_notebook_example.py +++ b/test/automl/test_notebook_example.py @@ -1,4 +1,5 @@ from openml.exceptions import OpenMLServerException +from requests.exceptions import ChunkedEncodingError def test_automl(budget=5, dataset_format="dataframe", hpo_method=None): @@ -8,8 +9,8 @@ def test_automl(budget=5, dataset_format="dataframe", hpo_method=None): X_train, X_test, y_train, y_test = load_openml_dataset( dataset_id=1169, data_dir="test/", dataset_format=dataset_format ) - except OpenMLServerException: - print("OpenMLServerException raised") + except (OpenMLServerException, ChunkedEncodingError) as e: + print(e) return """ import AutoML class from flaml package """ from flaml import AutoML @@ -84,8 +85,8 @@ def test_mlflow(): X_train, X_test, y_train, y_test = load_openml_task( task_id=7592, data_dir="test/" ) - except OpenMLServerException: - print("OpenMLServerException raised") + except (OpenMLServerException, ChunkedEncodingError) as e: + print(e) return """ import AutoML class from flaml package """ from flaml import AutoML