diff --git a/README.md b/README.md index b1cfee06c..71884d9df 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ Please find demo and tutorials of FLAML [here](https://www.youtube.com/channel/U For more technical details, please check our papers. -* [FLAML: A Fast and Lightweight AutoML Library](https://www.microsoft.com/en-us/research/publication/flaml-a-fast-and-lightweight-automl-library/). Chi Wang, Qingyun Wu, Markus Weimer, Erkang Zhu. MLSys, 2021. +* [FLAML: A Fast and Lightweight AutoML Library](https://www.microsoft.com/en-us/research/publication/flaml-a-fast-and-lightweight-automl-library/). Chi Wang, Qingyun Wu, Markus Weimer, Erkang Zhu. MLSys 2021. ```bibtex @inproceedings{wang2021flaml, diff --git a/flaml/automl.py b/flaml/automl.py index 1e4fb3aa4..9e3f78169 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -13,8 +13,6 @@ from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold, \ RepeatedKFold, GroupKFold from sklearn.utils import shuffle import pandas as pd -import os -import contextlib from .ml import compute_estimator, train_estimator, get_estimator_class, \ get_classification_objective @@ -56,6 +54,7 @@ class SearchState: self.low_cost_partial_config = {} self.cat_hp_cost = {} self.data_size = data_size + self.ls_ever_converged = False search_space = learner_class.search_space( data_size=data_size, task=task) for name, space in search_space.items(): @@ -215,7 +214,6 @@ class AutoMLState: } if sampled_weight is not None: self.fit_kwargs['sample_weight'] = weight - # with open(os.devnull, "w") as f, contextlib.redirect_stdout(f): # tune.report(**result) return result @@ -820,10 +818,12 @@ class AutoML: Returns: A dict. (a) if there is only one estimator in estimator_list, each key is a - hyperparameter name + hyperparameter name. (b) otherwise, it is a nested dict with 'ml' as the key, and a list of the low_cost_partial_configs as the value, corresponding - to each learner's low_cost_partial_config + to each learner's low_cost_partial_config; the estimator index as + an integer corresponding to the cheapest learner is appeneded to the + list at the end. ''' if len(self.estimator_list) == 1: @@ -835,6 +835,9 @@ class AutoML: for estimator in self.estimator_list: c = self._search_states[estimator].low_cost_partial_config configs.append(c) + configs.append(np.argmin([ + self._state.learner_classes.get(estimator).cost_relative2lgbm() + for estimator in self.estimator_list])) config = {'ml': configs} return config @@ -845,10 +848,11 @@ class AutoML: Returns: A dict. (a) if there is only one estimator in estimator_list, each key is a - hyperparameter name + hyperparameter name. (b) otherwise, it is a nested dict with 'ml' as the key, and a list of the cat_hp_cost's as the value, corresponding - to each learner's cat_hp_cost + to each learner's cat_hp_cost; the cost relative to lgbm for each + learner (as a list itself) is appended to the list at the end. ''' if len(self.estimator_list) == 1: @@ -860,6 +864,9 @@ class AutoML: for estimator in self.estimator_list: c = self._search_states[estimator].cat_hp_cost configs.append(c) + configs.append([ + self._state.learner_classes.get(estimator).cost_relative2lgbm() + for estimator in self.estimator_list]) config = {'ml': configs} return config @@ -930,7 +937,8 @@ class AutoML: config['FLAML_sample_size'] = sample_size estimator = config['learner'] del config['learner'] - states[estimator].training_function(config) + result = states[estimator].training_function(config) + return result return train @@ -943,7 +951,7 @@ class AutoML: ''' def size_func(config: dict) -> float: - config = config.get('ml', config).copy + config = config.get('ml', config) estimator = config['learner'] learner_class = self._state.learner_classes.get(estimator) return learner_class.size(config) @@ -971,7 +979,7 @@ class AutoML: metric='auto', task='classification', n_jobs=-1, - log_file_name='default.log', + log_file_name='flaml.log', estimator_list='auto', time_budget=60, max_iter=1000000, @@ -996,6 +1004,7 @@ class AutoML: learner_selector='sample', hpo_method=None, starting_points={}, + seed=None, **fit_kwargs): '''Find a model for a given task @@ -1063,12 +1072,20 @@ class AutoML: samples used while splitting the dataset into train/valid set verbose: int, default=1 | Controls the verbosity, higher means more messages. + hpo_method: str or None, default=None | The hyperparameter + optimization method. When it is None, CFO is used. + No need to set when using flaml's default search space or using + a simple customized search space. When set to 'bs', BlendSearch + is used. BlendSearch can be tried when the search space is + complex, for example, containing multiple disjoint, discontinuous + subspaces. starting_points: A dictionary to specify the starting hyperparameter config for the estimators. Keys are the name of the estimators, and values are the starting hyperparamter configurations for the corresponding estimators. + seed: int or None, default=None | The random seed for np.random. **fit_kwargs: Other key word arguments to pass to fit() function of - the searched learners, such sample_weight + the searched learners, such as sample_weight. ''' self._start_time_flag = time.time() self._state.task = task @@ -1079,6 +1096,8 @@ class AutoML: self._validate_data(X_train, y_train, dataframe, label, X_val, y_val) self._search_states = {} # key: estimator name; value: SearchState self._random = np.random.RandomState(RANDOM_SEED) + if seed is not None: + np.random.seed(seed) self._learner_selector = learner_selector old_level = logger.getEffectiveLevel() self.verbose = verbose @@ -1144,21 +1163,33 @@ class AutoML: estimator_list)) self.estimator_list = estimator_list self._hpo_method = hpo_method or 'cfo' - with training_log_writer(log_file_name) as save_helper: - self._training_log = save_helper - self._state.time_budget = time_budget - self._active_estimators = estimator_list.copy() - self._ensemble = ensemble - self._max_iter = max_iter - self._mem_thres = mem_thres - self._pred_time_limit = pred_time_limit - self._state.train_time_limit = train_time_limit - self._log_type = log_type - self.split_ratio = split_ratio - self._save_model_history = model_history - self._state.n_jobs = n_jobs + self._state.time_budget = time_budget + self._active_estimators = estimator_list.copy() + self._ensemble = ensemble + self._max_iter = max_iter + self._mem_thres = mem_thres + self._pred_time_limit = pred_time_limit + self._state.train_time_limit = train_time_limit + self._log_type = log_type + self.split_ratio = split_ratio + self._save_model_history = model_history + self._state.n_jobs = n_jobs + if log_file_name: + with training_log_writer(log_file_name) as save_helper: + self._training_log = save_helper + self._search() + else: + self._training_log = None self._search() - logger.info("fit succeeded") + logger.info("fit succeeded") + logger.info(f"Time taken to find the best model: {self._time_taken_best_iter}") + if self._time_taken_best_iter >= time_budget * 0.7 and not \ + all(self._ever_converged_per_learner.values()): + logger.warn("Time taken to find the best model is {0:.0f}% of the " + "provided time budget and not all estimators' hyperparameter " + "search converged. Consider increasing the time budget.".format( + self._time_taken_best_iter / time_budget * 100)) + if verbose == 0: logger.setLevel(old_level) @@ -1169,14 +1200,18 @@ class AutoML: self._state.time_from_start = 0 self._estimator_index = None self._best_iteration = 0 + self._time_taken_best_iter = 0 self._model_history = {} self._config_history = {} self._max_iter_per_learner = 1000000 # TODO self._iter_per_learner = dict([(e, 0) for e in self.estimator_list]) + self._ever_converged_per_learner = dict([(e, False) for e in self.estimator_list]) self._fullsize_reached = False self._trained_estimator = None self._best_estimator = None self._retrained_config = {} + self._warn_threshold = 10 + est_retrain_time = next_trial_time = 0 best_config_sig = None # use ConcurrencyLimiter to limit the amount of concurrency when @@ -1185,20 +1220,27 @@ class AutoML: if self._ensemble: self.best_model = {} try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune.suggest import ConcurrencyLimiter - except ImportError: + except (ImportError, AssertionError): from .searcher.suggestion import ConcurrencyLimiter if self._hpo_method in ('cfo', 'grid'): from flaml import CFO as SearchAlgo elif 'optuna' == self._hpo_method: try: + assert ray_version >= '1.0.0' from ray.tune.suggest.optuna import OptunaSearch as SearchAlgo - except ImportError: + except (ImportError, AssertionError): from .searcher.suggestion import OptunaSearch as SearchAlgo elif 'bs' == self._hpo_method: from flaml import BlendSearch as SearchAlgo + elif 'cfocat' == self._hpo_method: + from flaml import CFOCat as SearchAlgo else: - raise NotImplementedError + raise NotImplementedError( + f"hpo_method={self._hpo_method} is not recognized. " + "'cfo' and 'bs' are supported.") for self._track_iter in range(self._max_iter): if self._estimator_index is None: @@ -1244,7 +1286,7 @@ class AutoML: else: points_to_evaluate = [search_state.init_config] low_cost_partial_config = search_state.low_cost_partial_config - if self._hpo_method in ('bs', 'cfo', 'grid'): + if self._hpo_method in ('bs', 'cfo', 'grid', 'cfocat'): algo = SearchAlgo( metric='val_loss', mode='min', space=search_space, points_to_evaluate=points_to_evaluate, @@ -1267,7 +1309,7 @@ class AutoML: max_concurrent=1) else: search_space = None - if self._hpo_method in ('bs', 'cfo'): + if self._hpo_method in ('bs', 'cfo', 'cfocat'): search_state.search_alg.set_search_properties( metric=None, mode=None, config={ @@ -1320,20 +1362,22 @@ class AutoML: self._trained_estimator = None self._trained_estimator = search_state.trained_estimator self._best_iteration = self._track_iter + self._time_taken_best_iter = self._state.time_from_start better = True next_trial_time = search_state.time2eval_best if better or self._log_type == 'all': - self._training_log.append( - self._iter_per_learner[estimator], - search_state.train_loss, - search_state.trial_time, - self._state.time_from_start, - search_state.val_loss, - search_state.config, - search_state.best_loss, - search_state.best_config, - estimator, - search_state.sample_size) + if self._training_log: + self._training_log.append( + self._iter_per_learner[estimator], + search_state.train_loss, + search_state.trial_time, + self._state.time_from_start, + search_state.val_loss, + search_state.config, + search_state.best_loss, + search_state.best_config, + estimator, + search_state.sample_size) if mlflow is not None and mlflow.active_run(): with mlflow.start_run(nested=True): mlflow.log_metric('iter_counter', @@ -1365,6 +1409,15 @@ class AutoML: search_state.best_loss, self._best_estimator, self._state.best_loss)) + searcher = search_state.search_alg.searcher + if searcher.is_ls_ever_converged and not self._ever_converged_per_learner[estimator]: + self._ever_converged_per_learner[estimator] = searcher.is_ls_ever_converged + if all(self._ever_converged_per_learner.values()) and \ + self._state.time_from_start > self._warn_threshold * self._time_taken_best_iter: + logger.warn("All estimator hyperparameters local search has converged at least once, " + f"and the total search time exceeds {self._warn_threshold} times the time taken " + "to find the best model.") + self._warn_threshold *= 10 else: logger.info(f"no enough budget for learner {estimator}") if self._estimator_index is not None: @@ -1396,7 +1449,8 @@ class AutoML: if time_left < time_ensemble < 2 * time_left: break # Add a checkpoint for the current best config to the log. - self._training_log.checkpoint() + if self._training_log: + self._training_log.checkpoint() if self._best_estimator: self._selected = self._search_states[self._best_estimator] self._trained_estimator = self._selected.trained_estimator diff --git a/flaml/data.py b/flaml/data.py index 06d45ee37..c552ffeeb 100644 --- a/flaml/data.py +++ b/flaml/data.py @@ -11,7 +11,8 @@ from .training_log import training_log_reader from datetime import datetime -def load_openml_dataset(dataset_id, data_dir=None, random_state=0): +def load_openml_dataset(dataset_id, data_dir=None, random_state=0, + dataset_format='dataframe'): '''Load dataset from open ML. If the file is not cached locally, download it from open ML. @@ -20,12 +21,15 @@ def load_openml_dataset(dataset_id, data_dir=None, random_state=0): dataset_id: An integer of the dataset id in openml data_dir: A string of the path to store and load the data random_state: An integer of the random seed for splitting data - + dataset_format: A string specifying the format of returned dataset. Default is 'dataframe'. + Can choose from ['dataframe', 'array']. + If 'dataframe', the returned dataset will be a Pandas DataFrame. + If 'array', the returned dataset will be a NumPy array or a SciPy sparse matrix. Returns: - X_train: A dataframe of training data - X_test: A dataframe of test data - y_train: A series of labels for training data - y_test: A series of labels for test data + X_train: Training data + X_test: Test data + y_train: A series or array of labels for training data + y_test: A series or array of labels for test data ''' import os import openml @@ -48,7 +52,7 @@ def load_openml_dataset(dataset_id, data_dir=None, random_state=0): print('Dataset name:', dataset.name) X, y, * \ __ = dataset.get_data( - target=dataset.default_target_attribute) + target=dataset.default_target_attribute, dataset_format=dataset_format) X_train, X_test, y_train, y_test = train_test_split( X, y, random_state=random_state) print( diff --git a/flaml/ml.py b/flaml/ml.py index 8847f407c..3fc0bbf73 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -23,24 +23,24 @@ logger = logging.getLogger(__name__) def get_estimator_class(task, estimator_name): ''' when adding a new learner, need to add an elif branch ''' - if 'xgboost' in estimator_name: + if 'xgboost' == estimator_name: if 'regression' in task: estimator_class = XGBoostEstimator else: estimator_class = XGBoostSklearnEstimator - elif 'rf' in estimator_name: + elif 'rf' == estimator_name: estimator_class = RandomForestEstimator - elif 'lgbm' in estimator_name: + elif 'lgbm' == estimator_name: estimator_class = LGBMEstimator - elif 'lrl1' in estimator_name: + elif 'lrl1' == estimator_name: estimator_class = LRL1Classifier - elif 'lrl2' in estimator_name: + elif 'lrl2' == estimator_name: estimator_class = LRL2Classifier - elif 'catboost' in estimator_name: + elif 'catboost' == estimator_name: estimator_class = CatBoostEstimator - elif 'extra_tree' in estimator_name: + elif 'extra_tree' == estimator_name: estimator_class = ExtraTreeEstimator - elif 'kneighbor' in estimator_name: + elif 'kneighbor' == estimator_name: estimator_class = KNeighborsEstimator else: raise ValueError( diff --git a/flaml/model.py b/flaml/model.py index fa88c8e81..341714085 100644 --- a/flaml/model.py +++ b/flaml/model.py @@ -183,9 +183,10 @@ class SKLearnEstimator(BaseEstimator): def _preprocess(self, X): if isinstance(X, pd.DataFrame): - X = X.copy() cat_columns = X.select_dtypes(include=['category']).columns - X[cat_columns] = X[cat_columns].apply(lambda x: x.cat.codes) + if not cat_columns.empty: + X = X.copy() + X[cat_columns] = X[cat_columns].apply(lambda x: x.cat.codes) elif isinstance(X, np.ndarray) and X.dtype.kind not in 'buif': # numpy array is not of numeric dtype X = pd.DataFrame(X) @@ -213,7 +214,7 @@ class LGBMEstimator(BaseEstimator): 'low_cost_init_value': 4, }, 'min_child_samples': { - 'domain': tune.lograndint(lower=2, upper=2**7), + 'domain': tune.lograndint(lower=2, upper=2**7 + 1), 'init_value': 20, }, 'learning_rate': { @@ -225,7 +226,7 @@ class LGBMEstimator(BaseEstimator): 'init_value': 1.0, }, 'log_max_bin': { - 'domain': tune.lograndint(lower=3, upper=10), + 'domain': tune.lograndint(lower=3, upper=11), 'init_value': 8, }, 'colsample_bytree': { @@ -270,6 +271,8 @@ class LGBMEstimator(BaseEstimator): self.params["objective"] = objective if "max_bin" not in self.params: self.params['max_bin'] = 1 << int(round(log_max_bin)) - 1 + if "verbose" not in self.params: + self.params['verbose'] = -1 if 'regression' in task: self.estimator_class = LGBMRegressor else: @@ -281,6 +284,13 @@ class LGBMEstimator(BaseEstimator): if not isinstance(X, pd.DataFrame) and issparse(X) and np.issubdtype( X.dtype, np.integer): X = X.astype(float) + elif isinstance(X, np.ndarray) and X.dtype.kind not in 'buif': + # numpy array is not of numeric dtype + X = pd.DataFrame(X) + for col in X.columns: + if isinstance(X[col][0], str): + X[col] = X[col].astype('category').cat.codes + X = X.to_numpy() return X def fit(self, X_train, y_train, budget=None, **kwargs): @@ -455,14 +465,15 @@ class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator): super().__init__(task, **params) del self.params['objective'] del self.params['max_bin'] + del self.params['verbose'] self.params.update({ "n_estimators": int(round(n_estimators)), 'max_leaves': int(round(max_leaves)), 'max_depth': 0, 'grow_policy': params.get("grow_policy", 'lossguide'), 'tree_method': tree_method, - 'verbosity': 0, 'n_jobs': n_jobs, + 'verbosity': 0, 'learning_rate': float(learning_rate), 'subsample': float(subsample), 'reg_alpha': float(reg_alpha), @@ -531,6 +542,7 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator): self.params.update({ "n_estimators": int(round(n_estimators)), "n_jobs": n_jobs, + "verbose": 0, 'max_features': float(max_features), "max_leaf_nodes": params.get('max_leaf_nodes', int(round(max_leaves))), }) @@ -629,7 +641,7 @@ class CatBoostEstimator(BaseEstimator): @classmethod def search_space(cls, data_size, **params): - upper = max(min(round(1500000 / data_size), 150), 11) + upper = max(min(round(1500000 / data_size), 150), 12) return { 'early_stopping_rounds': { 'domain': tune.lograndint(lower=10, upper=upper), @@ -657,6 +669,25 @@ class CatBoostEstimator(BaseEstimator): CatBoostEstimator._time_per_iter = None CatBoostEstimator._train_size = 0 + def _preprocess(self, X): + if isinstance(X, pd.DataFrame): + cat_columns = X.select_dtypes(include=['category']).columns + if not cat_columns.empty: + X = X.copy() + X[cat_columns] = X[cat_columns].apply( + lambda x: + x.cat.rename_categories( + [str(c) if isinstance(c, float) else c + for c in x.cat.categories])) + elif isinstance(X, np.ndarray) and X.dtype.kind not in 'buif': + # numpy array is not of numeric dtype + X = pd.DataFrame(X) + for col in X.columns: + if isinstance(X[col][0], str): + X[col] = X[col].astype('category').cat.codes + X = X.to_numpy() + return X + def __init__( self, task='binary:logistic', n_jobs=1, n_estimators=8192, learning_rate=0.1, early_stopping_rounds=4, **params @@ -685,68 +716,69 @@ class CatBoostEstimator(BaseEstimator): def fit(self, X_train, y_train, budget=None, **kwargs): start_time = time.time() n_iter = self.params["n_estimators"] + X_train = self._preprocess(X_train) if isinstance(X_train, pd.DataFrame): cat_features = list(X_train.select_dtypes( include='category').columns) else: cat_features = [] - from catboost import CatBoostError - try: - if (not CatBoostEstimator._time_per_iter or abs( - CatBoostEstimator._train_size - len(y_train)) > 4) and budget: - # measure the time per iteration - self.params["n_estimators"] = 1 - CatBoostEstimator._smallmodel = self.estimator_class(**self.params) - CatBoostEstimator._smallmodel.fit( - X_train, y_train, cat_features=cat_features, **kwargs) - CatBoostEstimator._t1 = time.time() - start_time - if CatBoostEstimator._t1 >= budget: - self.params["n_estimators"] = n_iter - self._model = CatBoostEstimator._smallmodel - return CatBoostEstimator._t1 - self.params["n_estimators"] = 4 - CatBoostEstimator._smallmodel = self.estimator_class(**self.params) - CatBoostEstimator._smallmodel.fit( - X_train, y_train, cat_features=cat_features, **kwargs) - CatBoostEstimator._time_per_iter = ( - time.time() - start_time - CatBoostEstimator._t1) / ( - self.params["n_estimators"] - 1) - if CatBoostEstimator._time_per_iter <= 0: - CatBoostEstimator._time_per_iter = CatBoostEstimator._t1 - CatBoostEstimator._train_size = len(y_train) - if time.time() - start_time >= budget or n_iter == self.params[ - "n_estimators"]: - self.params["n_estimators"] = n_iter - self._model = CatBoostEstimator._smallmodel - return time.time() - start_time - if budget: - train_times = 1 - self.params["n_estimators"] = min(n_iter, int( - (budget - time.time() + start_time - CatBoostEstimator._t1) - / train_times / CatBoostEstimator._time_per_iter + 1)) + # from catboost import CatBoostError + # try: + if (not CatBoostEstimator._time_per_iter or abs( + CatBoostEstimator._train_size - len(y_train)) > 4) and budget: + # measure the time per iteration + self.params["n_estimators"] = 1 + CatBoostEstimator._smallmodel = self.estimator_class(**self.params) + CatBoostEstimator._smallmodel.fit( + X_train, y_train, cat_features=cat_features, **kwargs) + CatBoostEstimator._t1 = time.time() - start_time + if CatBoostEstimator._t1 >= budget: + self.params["n_estimators"] = n_iter self._model = CatBoostEstimator._smallmodel - if self.params["n_estimators"] > 0: - n = max(int(len(y_train) * 0.9), len(y_train) - 1000) - X_tr, y_tr = X_train[:n], y_train[:n] - if 'sample_weight' in kwargs: - weight = kwargs['sample_weight'] - if weight is not None: - kwargs['sample_weight'] = weight[:n] - else: - weight = None - from catboost import Pool - model = self.estimator_class(**self.params) - model.fit( - X_tr, y_tr, cat_features=cat_features, - eval_set=Pool( - data=X_train[n:], label=y_train[n:], - cat_features=cat_features), - **kwargs) # model.get_best_iteration() + return CatBoostEstimator._t1 + self.params["n_estimators"] = 4 + CatBoostEstimator._smallmodel = self.estimator_class(**self.params) + CatBoostEstimator._smallmodel.fit( + X_train, y_train, cat_features=cat_features, **kwargs) + CatBoostEstimator._time_per_iter = ( + time.time() - start_time - CatBoostEstimator._t1) / ( + self.params["n_estimators"] - 1) + if CatBoostEstimator._time_per_iter <= 0: + CatBoostEstimator._time_per_iter = CatBoostEstimator._t1 + CatBoostEstimator._train_size = len(y_train) + if time.time() - start_time >= budget or n_iter == self.params[ + "n_estimators"]: + self.params["n_estimators"] = n_iter + self._model = CatBoostEstimator._smallmodel + return time.time() - start_time + if budget: + train_times = 1 + self.params["n_estimators"] = min(n_iter, int( + (budget - time.time() + start_time - CatBoostEstimator._t1) + / train_times / CatBoostEstimator._time_per_iter + 1)) + self._model = CatBoostEstimator._smallmodel + if self.params["n_estimators"] > 0: + n = max(int(len(y_train) * 0.9), len(y_train) - 1000) + X_tr, y_tr = X_train[:n], y_train[:n] + if 'sample_weight' in kwargs: + weight = kwargs['sample_weight'] if weight is not None: - kwargs['sample_weight'] = weight - self._model = model - except CatBoostError: - self._model = None + kwargs['sample_weight'] = weight[:n] + else: + weight = None + from catboost import Pool + model = self.estimator_class(**self.params) + model.fit( + X_tr, y_tr, cat_features=cat_features, + eval_set=Pool( + data=X_train[n:], label=y_train[n:], + cat_features=cat_features), + **kwargs) # model.get_best_iteration() + if weight is not None: + kwargs['sample_weight'] = weight + self._model = model + # except CatBoostError: + # self._model = None self.params["n_estimators"] = n_iter train_time = time.time() - start_time return train_time diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py index 88884f982..43f6b3362 100644 --- a/flaml/searcher/blendsearch.py +++ b/flaml/searcher/blendsearch.py @@ -9,16 +9,18 @@ import time import pickle try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune.suggest import Searcher from ray.tune.suggest.optuna import OptunaSearch as GlobalSearch - from ray.tune.suggest.variant_generator import generate_variants from ray.tune.utils.util import flatten_dict -except ImportError: +except (ImportError, AssertionError): from .suggestion import Searcher from .suggestion import OptunaSearch as GlobalSearch - from .variant_generator import generate_variants, flatten_dict + from .variant_generator import flatten_dict from .search_thread import SearchThread from .flow2 import FLOW2 +from ..tune.space import add_cost_to_space, normalize # TODO: , define_by_run_func import logging logger = logging.getLogger(__name__) @@ -125,9 +127,15 @@ class BlendSearch(Searcher): if self._metric_constraints: # metric modified by lagrange metric += self.lagrange + self._cat_hp_cost = cat_hp_cost or {} + if space: + add_cost_to_space(space, init_config, self._cat_hp_cost) if global_search_alg is not None: self._gs = global_search_alg elif getattr(self, '__name__', None) != 'CFO': + gs_space = space + # TODO: when define_by_run is supported + # gs_space = define_by_run_func(space) try: gs_seed = seed - 10 if (seed - 10) >= 0 else seed - 11 + (1 << 32) if experimental: @@ -137,10 +145,11 @@ class BlendSearch(Searcher): else: sampler = None self._gs = GlobalSearch( - space=space, metric=metric, mode=mode, seed=gs_seed, + space=gs_space, metric=metric, mode=mode, seed=gs_seed, sampler=sampler) except TypeError: - self._gs = GlobalSearch(space=space, metric=metric, mode=mode) + self._gs = GlobalSearch(space=gs_space, metric=metric, mode=mode) + self._gs.space = space else: self._gs = None self._experimental = experimental @@ -152,8 +161,10 @@ class BlendSearch(Searcher): else: self._candidate_start_points = None self._ls = self.LocalSearch( - init_config, metric, mode, cat_hp_cost, space, prune_attr, + init_config, metric, mode, space, prune_attr, min_resource, max_resource, reduction_factor, self.cost_attr, seed) + self._is_ls_ever_converged = False + self._subspace = {} # the subspace for each trial id self._init_search() def set_search_properties(self, @@ -174,9 +185,13 @@ class BlendSearch(Searcher): self._mode = mode if not self._ls.space: # the search space can be set only once - self._ls.set_search_properties(metric, mode, config) if self._gs is not None: self._gs.set_search_properties(metric, mode, config) + self._gs.space = config + if config: + add_cost_to_space( + config, self._ls.init_config, self._cat_hp_cost) + self._ls.set_search_properties(metric, mode, config) self._init_search() elif metric_changed or mode_changed: # reset search when metric or mode changed @@ -205,7 +220,9 @@ class BlendSearch(Searcher): self._thread_count = 1 # total # threads created self._init_used = self._ls.init_config is None self._trial_proposed_by = {} # trial_id: str -> thread_id: int - self._ls_bound_min = self._ls.normalize(self._ls.init_config) + self._ls_bound_min = normalize( + self._ls.init_config.copy(), self._ls.space, self._ls.init_config, + {}, recursive=True) self._ls_bound_max = self._ls_bound_min.copy() self._gs_admissible_min = self._ls_bound_min.copy() self._gs_admissible_max = self._ls_bound_max.copy() @@ -231,34 +248,16 @@ class BlendSearch(Searcher): ''' with open(checkpoint_path, "rb") as inputFile: state = pickle.load(inputFile) - self._metric_target = state._metric_target - self._search_thread_pool = state._search_thread_pool - self._thread_count = state._thread_count - self._init_used = state._init_used - self._trial_proposed_by = state._trial_proposed_by - self._ls_bound_min = state._ls_bound_min - self._ls_bound_max = state._ls_bound_max - self._gs_admissible_min = state._gs_admissible_min - self._gs_admissible_max = state._gs_admissible_max - self._result = state._result - self._deadline = state._deadline - self._metric, self._mode = state._metric, state._mode - self._points_to_evaluate = state._points_to_evaluate - self._gs = state._gs - self._ls = state._ls - self._config_constraints = state._config_constraints - self._metric_constraints = state._metric_constraints - self._metric_constraint_satisfied = state._metric_constraint_satisfied - self._metric_constraint_penalty = state._metric_constraint_penalty - self._candidate_start_points = state._candidate_start_points - if self._candidate_start_points: - self._started_from_given = state._started_from_given - self._started_from_low_cost = state._started_from_low_cost + self.__dict__ = state.__dict__ @property def metric_target(self): return self._metric_target + @property + def is_ls_ever_converged(self): + return self._is_ls_ever_converged + def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False): ''' search thread updater and cleaner @@ -296,10 +295,12 @@ class BlendSearch(Searcher): for key, value in result.items(): if key.startswith('config/'): config[key[7:]] = value + signature = self._ls.config_signature( + config, self._subspace.get(trial_id, {})) if error: # remove from result cache - del self._result[self._ls.config_signature(config)] + del self._result[signature] else: # add to result cache - self._result[self._ls.config_signature(config)] = result + self._result[signature] = result # update target metric if improved objective = result[self._ls.metric] if (objective - self._metric_target) * self._ls.metric_op < 0: @@ -307,8 +308,11 @@ class BlendSearch(Searcher): if thread_id: if not self._metric_constraint_satisfied: # no point has been found to satisfy metric constraint - self._expand_admissible_region() + self._expand_admissible_region( + self._ls_bound_min, self._ls_bound_max, + self._subspace.get(trial_id, self._ls.space)) if self._gs is not None and self._experimental: + # TODO: key match for hierarchical space self._gs.add_evaluated_point(flatten_dict(config), objective) elif metric_constraint_satisfied and self._create_condition( result): @@ -320,7 +324,8 @@ class BlendSearch(Searcher): del self._candidate_start_points[trial_id] else: self._started_from_low_cost = True - self._create_thread(config, result) + self._create_thread(config, result, self._subspace.get( + trial_id, self._ls.space)) # reset admissible region to ls bounding box self._gs_admissible_min.update(self._ls_bound_min) self._gs_admissible_max.update(self._ls_bound_max) @@ -328,29 +333,46 @@ class BlendSearch(Searcher): if thread_id and thread_id in self._search_thread_pool: # local search thread self._clean(thread_id) + if trial_id in self._subspace and not (self._candidate_start_points + and trial_id in self._candidate_start_points): + del self._subspace[trial_id] - def _create_thread(self, config, result): + def _create_thread(self, config, result, space): # logger.info(f"create local search thread from {config}") self._search_thread_pool[self._thread_count] = SearchThread( self._ls.mode, self._ls.create( config, result[self._ls.metric], - cost=result.get(self.cost_attr, 1)), + cost=result.get(self.cost_attr, 1), space=space), self.cost_attr ) self._thread_count += 1 self._update_admissible_region( - config, self._ls_bound_min, self._ls_bound_max) + config, self._ls_bound_min, self._ls_bound_max, space) - def _update_admissible_region(self, config, admissible_min, admissible_max): + def _update_admissible_region( + self, config, admissible_min, admissible_max, space: Dict = {} + ): # update admissible region - normalized_config = self._ls.normalize(config) + normalized_config = normalize(config, space, config, {}) for key in admissible_min: value = normalized_config[key] - if value > admissible_max[key]: - admissible_max[key] = value - elif value < admissible_min[key]: - admissible_min[key] = value + if isinstance(admissible_max[key], list): + choice = space[key]['_choice_'] + self._update_admissible_region( + value, + admissible_min[key][choice], admissible_max[key][choice], + space[key] + ) + elif isinstance(value, dict): + self._update_admissible_region( + value, + admissible_min[key], admissible_max[key], space[key]) + else: + if value > admissible_max[key]: + admissible_max[key] = value + elif value < admissible_min[key]: + admissible_min[key] = value def _create_condition(self, result: Dict) -> bool: ''' create thread condition @@ -379,8 +401,11 @@ class BlendSearch(Searcher): break create_new = False if self._search_thread_pool[thread_id].converged: + self._is_ls_ever_converged = True todelete.add(thread_id) - self._expand_admissible_region() + self._expand_admissible_region( + self._ls_bound_min, self._ls_bound_max, + self._search_thread_pool[thread_id].space) if self._candidate_start_points: if not self._started_from_given: # remove start points whose perf is worse than the converged @@ -417,12 +442,21 @@ class BlendSearch(Searcher): config[key[7:]] = value self._started_from_given = True del self._candidate_start_points[best_trial_id] - self._create_thread(config, result) + self._create_thread(config, result, self._subspace.get( + best_trial_id, self._ls.space)) - def _expand_admissible_region(self): - for key in self._ls_bound_max: - self._ls_bound_max[key] += self._ls.STEPSIZE - self._ls_bound_min[key] -= self._ls.STEPSIZE + def _expand_admissible_region(self, lower, upper, space): + for key in upper: + ub = upper[key] + if isinstance(ub, list): + choice = space[key]['_choice_'] + self._expand_admissible_region( + lower[key][choice], upper[key][choice], space[key]) + elif isinstance(ub, dict): + self._expand_admissible_region(lower[key], ub, space[key]) + else: + upper[key] += self._ls.STEPSIZE + lower[key] -= self._ls.STEPSIZE def _inferior(self, id1: int, id2: int) -> bool: ''' whether thread id1 is inferior to id2 @@ -460,34 +494,42 @@ class BlendSearch(Searcher): if choice and config is None: # local search thread finishes if self._search_thread_pool[choice].converged: - self._expand_admissible_region() + self._expand_admissible_region( + self._ls_bound_min, self._ls_bound_max, + self._search_thread_pool[choice].space) del self._search_thread_pool[choice] return None # preliminary check; not checking config validation - skip = self._should_skip(choice, trial_id, config) + space = self._search_thread_pool[choice].space + skip = self._should_skip(choice, trial_id, config, space) + use_rs = 0 if skip: if choice: return None # use rs when BO fails to suggest a config - for _, generated in generate_variants({'config': self._ls.space}): - config = generated['config'] - break # get one random config - skip = self._should_skip(-1, trial_id, config) + config, space = self._ls.complete_config({}) + skip = self._should_skip(-1, trial_id, config, space) if skip: return None - if choice or self._valid(config): + use_rs = 1 + if choice or self._valid( + config, space, self._gs_admissible_min, self._gs_admissible_max): # LS or valid or no backup choice self._trial_proposed_by[trial_id] = choice + self._search_thread_pool[choice].running += use_rs else: # invalid config proposed by GS if choice == backup: # use CFO's init point init_config = self._ls.init_config - config = self._ls.complete_config( + config, space = self._ls.complete_config( init_config, self._ls_bound_min, self._ls_bound_max) self._trial_proposed_by[trial_id] = choice + self._search_thread_pool[choice].running += 1 else: - config = self._search_thread_pool[backup].suggest(trial_id) - skip = self._should_skip(backup, trial_id, config) + thread = self._search_thread_pool[backup] + config = thread.suggest(trial_id) + space = thread.space + skip = self._should_skip(backup, trial_id, config, space) if skip: return None self._trial_proposed_by[trial_id] = backup @@ -498,21 +540,24 @@ class BlendSearch(Searcher): config[self._ls.prune_attr] = self._ls.min_resource # temporarily relax admissible region for parallel proposals self._update_admissible_region( - config, self._gs_admissible_min, self._gs_admissible_max) + config, self._gs_admissible_min, self._gs_admissible_max, + space) else: self._update_admissible_region( - config, self._ls_bound_min, self._ls_bound_max) + config, self._ls_bound_min, self._ls_bound_max, space) self._gs_admissible_min.update(self._ls_bound_min) self._gs_admissible_max.update(self._ls_bound_max) - self._result[self._ls.config_signature(config)] = {} + signature = self._ls.config_signature(config, space) + self._result[signature] = {} + self._subspace[trial_id] = space else: # use init config if self._candidate_start_points is not None and self._points_to_evaluate: self._candidate_start_points[trial_id] = None init_config = self._points_to_evaluate.pop( 0) if self._points_to_evaluate else self._ls.init_config - config = self._ls.complete_config( + config, space = self._ls.complete_config( init_config, self._ls_bound_min, self._ls_bound_max) - config_signature = self._ls.config_signature(config) + config_signature = self._ls.config_signature(config, space) result = self._result.get(config_signature) if result: # tried before return None @@ -523,15 +568,16 @@ class BlendSearch(Searcher): self._init_used = True self._trial_proposed_by[trial_id] = 0 self._search_thread_pool[0].running += 1 + self._subspace[trial_id] = space return config - def _should_skip(self, choice, trial_id, config) -> bool: + def _should_skip(self, choice, trial_id, config, space) -> bool: ''' if config is None or config's result is known or constraints are violated return True; o.w. return False ''' if config is None: return True - config_signature = self._ls.config_signature(config) + config_signature = self._ls.config_signature(config, space) exists = config_signature in self._result # check constraints if not exists and self._config_constraints: @@ -600,23 +646,35 @@ class BlendSearch(Searcher): backup_thread_id = thread_id return top_thread_id, backup_thread_id - def _valid(self, config: Dict) -> bool: + def _valid(self, config: Dict, space: Dict, lower: Dict, upper: Dict) -> bool: ''' config validator ''' - normalized_config = self._ls.normalize(config) - for key in self._gs_admissible_min: + normalized_config = normalize(config, space, config, {}) + for key, lb in lower.items(): if key in config: value = normalized_config[key] - if value + self._ls.STEPSIZE < self._gs_admissible_min[key] \ - or value > self._gs_admissible_max[key] + self._ls.STEPSIZE: + if isinstance(lb, list): + subspace = space[key]['_choice_'] + elif isinstance(lb, dict): + subspace = space[key] + else: + subspace = None + if subspace: + valid = self._valid(value, subspace, lb, upper[key]) + if not valid: + return False + elif (value + self._ls.STEPSIZE < lower[key] + or value > upper[key] + self._ls.STEPSIZE): return False return True try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune import (uniform, quniform, choice, randint, qrandint, randn, qrandn, loguniform, qloguniform) -except ImportError: +except (ImportError, AssertionError): from ..tune.sample import (uniform, quniform, choice, randint, qrandint, randn, qrandn, loguniform, qloguniform) diff --git a/flaml/searcher/flow2.py b/flaml/searcher/flow2.py index b807e6fe2..5ad75d89f 100644 --- a/flaml/searcher/flow2.py +++ b/flaml/searcher/flow2.py @@ -3,17 +3,20 @@ * Licensed under the MIT License. See LICENSE file in the * project root for license information. ''' -from typing import Dict, Optional +from typing import Dict, Optional, Tuple import numpy as np try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune.suggest import Searcher from ray.tune.suggest.variant_generator import generate_variants from ray.tune import sample from ray.tune.utils.util import flatten_dict, unflatten_dict -except ImportError: +except (ImportError, AssertionError): from .suggestion import Searcher from .variant_generator import generate_variants, flatten_dict, unflatten_dict from ..tune import sample +from ..tune.space import complete_config, denormalize, normalize import logging @@ -31,7 +34,6 @@ class FLOW2(Searcher): init_config: dict, metric: Optional[str] = None, mode: Optional[str] = None, - cat_hp_cost: Optional[dict] = None, space: Optional[dict] = None, prune_attr: Optional[str] = None, min_resource: Optional[float] = None, @@ -90,12 +92,11 @@ class FLOW2(Searcher): elif mode == "min": self.metric_op = 1. self.space = space or {} - self.space = flatten_dict(self.space, prevent_delimiter=True) + self._space = flatten_dict(self.space, prevent_delimiter=True) self._random = np.random.RandomState(seed) self._seed = seed self.init_config = init_config self.best_config = flatten_dict(init_config) - self.cat_hp_cost = cat_hp_cost self.prune_attr = prune_attr self.min_resource = min_resource self.resource_multiple_factor = resource_multiple_factor or 4 @@ -109,18 +110,9 @@ class FLOW2(Searcher): def _init_search(self): self._tunable_keys = [] self._bounded_keys = [] - # choices of numeric values. integer encoding. - # value: (ordered list of choices, - # dict from choice to index in the ordered list) - self._ordered_choice_hp = {} - # choices with given cost. integer encoding. - # value: (array of choices ordered by cost, - # dict from choice to index in the ordered array) - self._ordered_cat_hp = {} - # unordered choices. value: cardinality self._unordered_cat_hp = {} - self._cat_hp_cost = {} - for key, domain in self.space.items(): + hier = False + for key, domain in self._space.items(): assert not (isinstance(domain, dict) and 'grid_search' in domain), \ f"{key}'s domain is grid search, not supported in FLOW^2." if callable(getattr(domain, 'get_sampler', None)): @@ -129,41 +121,33 @@ class FLOW2(Searcher): # the step size lower bound for uniform variables doesn't depend # on the current config if isinstance(sampler, sample.Quantized): - sampler_inner = sampler.get_sampler() - if str(sampler_inner) == 'Uniform': + q = sampler.q + sampler = sampler.get_sampler() + if str(sampler) == 'Uniform': self._step_lb = min( - self._step_lb, sampler.q / (domain.upper - domain.lower)) + self._step_lb, q / (domain.upper - domain.lower)) elif isinstance(domain, sample.Integer) and str(sampler) == 'Uniform': self._step_lb = min( - self._step_lb, 1.0 / (domain.upper - domain.lower)) + self._step_lb, 1.0 / (domain.upper - 1 - domain.lower)) if isinstance(domain, sample.Categorical): - cat_hp_cost = self.cat_hp_cost - if cat_hp_cost and key in cat_hp_cost: - cost = np.array(cat_hp_cost[key]) - ind = np.argsort(cost) - ordered = np.array(domain.categories)[ind] - cost = self._cat_hp_cost[key] = cost[ind] - d = {} - for i, choice in enumerate(ordered): - d[choice] = i - self._ordered_cat_hp[key] = (ordered, d) - elif all(isinstance(x, int) or isinstance(x, float) - for x in domain.categories): - ordered = sorted(domain.categories) - d = {} - for i, choice in enumerate(ordered): - d[choice] = i - self._ordered_choice_hp[key] = (ordered, d) - else: + if not domain.ordered: self._unordered_cat_hp[key] = len(domain.categories) + if not hier: + for cat in domain.categories: + if isinstance(cat, dict): + hier = True + break if str(sampler) != 'Normal': self._bounded_keys.append(key) - self._space_keys = list(self.space.keys()) - if (self.prune_attr and self.prune_attr not in self.space + if not hier: + self._space_keys = sorted(self._space.keys()) + self._hierarchical = hier + if (self.prune_attr and self.prune_attr not in self._space and self.max_resource): - self._space_keys.append(self.prune_attr) self.min_resource = self.min_resource or self._min_resource() self._resource = self._round(self.min_resource) + if not hier: + self._space_keys.append(self.prune_attr) else: self._resource = None self.incumbent = {} @@ -203,20 +187,21 @@ class FLOW2(Searcher): for key in self._tunable_keys: if key not in self.best_config: continue - domain = self.space[key] + domain = self._space[key] sampler = domain.get_sampler() # the stepsize lower bound for log uniform variables depends on the # current config if isinstance(sampler, sample.Quantized): + q = sampler.q sampler_inner = sampler.get_sampler() if str(sampler_inner) == 'LogUniform': step_lb = min( - step_lb, np.log(1.0 + sampler.q / self.best_config[key]) + step_lb, np.log(1.0 + q / self.best_config[key]) / np.log(domain.upper / domain.lower)) elif isinstance(domain, sample.Integer) and str(sampler) == 'LogUniform': step_lb = min( step_lb, np.log(1.0 + 1.0 / self.best_config[key]) - / np.log(domain.upper / domain.lower)) + / np.log((domain.upper - 1) / domain.lower)) if np.isinf(step_lb): step_lb = self.STEP_LOWER_BOUND else: @@ -246,56 +231,26 @@ class FLOW2(Searcher): def complete_config( self, partial_config: Dict, lower: Optional[Dict] = None, upper: Optional[Dict] = None - ) -> Dict: + ) -> Tuple[Dict, Dict]: ''' generate a complete config from the partial config input add minimal resource to config if available ''' - if self._reset_times and partial_config == self.init_config: - # not the first time to complete init_config, use random gaussian - normalized = self.normalize(partial_config) - for key in normalized: - # don't change unordered cat choice - if key not in self._unordered_cat_hp: - if upper and lower: - up, low = upper[key], lower[key] - gauss_std = up - low or self.STEPSIZE - # allowed bound - up += self.STEPSIZE - low -= self.STEPSIZE - elif key in self._bounded_keys: - up, low, gauss_std = 1, 0, 1.0 - else: - up, low, gauss_std = np.Inf, -np.Inf, 1.0 - if key in self._bounded_keys: - up = min(up, 1) - low = max(low, 0) - delta = self.rand_vector_gaussian(1, gauss_std)[0] - normalized[key] = max(low, min(up, normalized[key] + delta)) - # use best config for unordered cat choice - config = self.denormalize(normalized) - else: - # first time init_config, or other configs, take as is - config = partial_config.copy() + disturb = self._reset_times and partial_config == self.init_config + # if not the first time to complete init_config, use random gaussian + config, space = complete_config( + partial_config, self.space, self, disturb, lower, upper) if partial_config == self.init_config: self._reset_times += 1 - config = flatten_dict(config) - for key, value in self.space.items(): - if key not in config: - config[key] = value - for _, generated in generate_variants({'config': config}): - config = generated['config'] - break if self._resource: config[self.prune_attr] = self.min_resource - return unflatten_dict(config) + return config, space - def create(self, init_config: Dict, obj: float, cost: float) -> Searcher: - flatten_config = flatten_dict(init_config) - # use the subspace where the init_config is located - space = {k: self.space[k] for k in flatten_config if k in self.space} + def create(self, init_config: Dict, obj: float, cost: float, space: Dict + ) -> Searcher: + # space is the subspace where the init_config is located flow2 = self.__class__( - init_config, self.metric, self.mode, self._cat_hp_cost, - unflatten_dict(space), self.prune_attr, + init_config, self.metric, self.mode, + space, self.prune_attr, self.min_resource, self.max_resource, self.resource_multiple_factor, self.cost_attr, self._seed + 1) flow2.best_obj = obj * self.metric_op # minimize internally @@ -303,115 +258,17 @@ class FLOW2(Searcher): self._seed += 1 return flow2 - def normalize(self, config) -> Dict: + def normalize(self, config, recursive=False) -> Dict: ''' normalize each dimension in config to [0,1] ''' - config_norm = {} - for key, value in flatten_dict(config).items(): - if key in self.space: - # domain: sample.Categorical/Integer/Float/Function - domain = self.space[key] - if not callable(getattr(domain, 'get_sampler', None)): - config_norm[key] = value - else: - if isinstance(domain, sample.Categorical): - # normalize categorical - if key in self._ordered_cat_hp: - l, d = self._ordered_cat_hp[key] - config_norm[key] = (d[value] + 0.5) / len(l) - elif key in self._ordered_choice_hp: - l, d = self._ordered_choice_hp[key] - config_norm[key] = (d[value] + 0.5) / len(l) - elif key in self.incumbent: - config_norm[key] = self.incumbent[ - key] if value == self.best_config[ - key] else ( - self.incumbent[key] - + 1.0 / self._unordered_cat_hp[key]) % 1 - else: - config_norm[key] = 0.5 - continue - # Uniform/LogUniform/Normal/Base - sampler = domain.get_sampler() - if isinstance(sampler, sample.Quantized): - # sampler is sample.Quantized - sampler = sampler.get_sampler() - if str(sampler) == 'LogUniform': - config_norm[key] = np.log(value / domain.lower) / np.log( - domain.upper / domain.lower) - elif str(sampler) == 'Uniform': - config_norm[key] = ( - value - domain.lower) / (domain.upper - domain.lower) - elif str(sampler) == 'Normal': - # N(mean, sd) -> N(0,1) - config_norm[key] = (value - sampler.mean) / sampler.sd - else: - # TODO? elif str(sampler) == 'Base': # sample.Function._CallSampler - # e.g., {test: sample_from(lambda spec: randn(10, 2).sample() * 0.01)} - config_norm[key] = value - else: # prune_attr - config_norm[key] = value - return config_norm + return normalize( + config, self._space, self.best_config, self.incumbent, recursive) def denormalize(self, config): ''' denormalize each dimension in config from [0,1] ''' - config_denorm = {} - for key, value in config.items(): - if key in self.space: - # domain: sample.Categorical/Integer/Float/Function - domain = self.space[key] - if not callable(getattr(domain, 'get_sampler', None)): - config_denorm[key] = value - else: - if isinstance(domain, sample.Categorical): - # denormalize categorical - if key in self._ordered_cat_hp: - l, _ = self._ordered_cat_hp[key] - n = len(l) - config_denorm[key] = l[min(n - 1, int(np.floor(value * n)))] - elif key in self._ordered_choice_hp: - l, _ = self._ordered_choice_hp[key] - n = len(l) - config_denorm[key] = l[min(n - 1, int(np.floor(value * n)))] - else: - assert key in self.incumbent - n = self._unordered_cat_hp[key] - if np.floor(value * n) == np.floor(self.incumbent[key] * n): - config_denorm[key] = self.best_config[key] - else: # ****random value each time!**** - config_denorm[key] = self._random.choice( - [x for x in domain.categories - if x != self.best_config[key]]) - continue - # Uniform/LogUniform/Normal/Base - sampler = domain.get_sampler() - if isinstance(sampler, sample.Quantized): - # sampler is sample.Quantized - sampler = sampler.get_sampler() - # Handle Log/Uniform - if str(sampler) == 'LogUniform': - config_denorm[key] = ( - domain.upper / domain.lower) ** value * domain.lower - elif str(sampler) == 'Uniform': - config_denorm[key] = value * ( - domain.upper - domain.lower) + domain.lower - elif str(sampler) == 'Normal': - # denormalization for 'Normal' - config_denorm[key] = value * sampler.sd + sampler.mean - else: - config_denorm[key] = value - # Handle quantized - sampler = domain.get_sampler() - if isinstance(sampler, sample.Quantized): - config_denorm[key] = np.round( - np.divide(config_denorm[key], sampler.q)) * sampler.q - # Handle int (4.6 -> 5) - if isinstance(domain, sample.Integer): - config_denorm[key] = int(round(config_denorm[key])) - else: # prune_attr - config_denorm[key] = value - return config_denorm + return denormalize( + config, self._space, self.best_config, self.incumbent, self._random) def set_search_properties(self, metric: Optional[str] = None, @@ -428,6 +285,7 @@ class FLOW2(Searcher): self.metric_op = 1. if config: self.space = config + self._space = flatten_dict(self.space) self._init_search() return True @@ -600,7 +458,7 @@ class FLOW2(Searcher): for i, key in enumerate(self._tunable_keys): if self._direction_tried[i] != 0: for _, generated in generate_variants({'config': { - key: self.space[key] + key: self._space[key] }}): if generated['config'][key] != best_config[key]: config[key] = generated['config'][key] @@ -632,26 +490,27 @@ class FLOW2(Searcher): ''' return self._num_allowed4incumbent > 0 - def config_signature(self, config) -> tuple: + def config_signature(self, config, space: Dict = None) -> tuple: ''' return the signature tuple of a config ''' config = flatten_dict(config) + if space: + space = flatten_dict(space) + else: + space = self._space value_list = [] - for key in self._space_keys: - if key in config: - value = config[key] - if key == self.prune_attr: - value_list.append(value) - # else key must be in self.space - # get rid of list type or constant, - # e.g., "eval_metric": ["logloss", "error"] - elif callable(getattr(self.space[key], 'sample', None)): - if isinstance(self.space[key], sample.Integer): - value_list.append(int(round(value))) - else: - value_list.append(value) + keys = sorted(config.keys()) if self._hierarchical else self._space_keys + for key in keys: + value = config[key] + if key == self.prune_attr: + value_list.append(value) + # else key must be in self.space + # get rid of list type or constant, + # e.g., "eval_metric": ["logloss", "error"] + elif isinstance(space[key], sample.Integer): + value_list.append(int(round(value))) else: - value_list.append(None) + value_list.append(value) return tuple(value_list) @property diff --git a/flaml/searcher/search_thread.py b/flaml/searcher/search_thread.py index 8640e6132..9c6a7fb05 100644 --- a/flaml/searcher/search_thread.py +++ b/flaml/searcher/search_thread.py @@ -6,10 +6,13 @@ from typing import Dict, Optional import numpy as np try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune.suggest import Searcher -except ImportError: +except (ImportError, AssertionError): from .suggestion import Searcher from .flow2 import FLOW2 +from ..tune.space import (add_cost_to_space, unflatten_hierarchical) import logging logger = logging.getLogger(__name__) @@ -41,6 +44,12 @@ class SearchThread: self._init_config = True self.running = 0 # the number of running trials from the thread self.cost_attr = cost_attr + if search_alg: + self.space = self._space = search_alg.space # unflattened space + # TODO: remove when define_by_run is supported + if not isinstance(self._search_alg, FLOW2): + # remember const config + self._const = add_cost_to_space(self.space, {}, {}) @classmethod def set_eps(cls, time_budget_s): @@ -54,6 +63,9 @@ class SearchThread: else: try: config = self._search_alg.suggest(trial_id) + # TODO: remove when define_by_run is supported + config.update(self._const) + config, self.space = unflatten_hierarchical(config, self._space) except FloatingPointError: logger.warning( 'The global search method raises FloatingPointError. ' diff --git a/flaml/tune/__init__.py b/flaml/tune/__init__.py index 6a581608b..111b1a532 100644 --- a/flaml/tune/__init__.py +++ b/flaml/tune/__init__.py @@ -1,9 +1,11 @@ try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune import (uniform, quniform, choice, randint, qrandint, randn, - qrandn, loguniform, qloguniform, lograndint) -except ImportError: + qrandn, loguniform, qloguniform, lograndint, qlograndint) +except (ImportError, AssertionError): from .sample import (uniform, quniform, choice, randint, qrandint, randn, - qrandn, loguniform, qloguniform, lograndint) + qrandn, loguniform, qloguniform, lograndint, qlograndint) from .tune import run, report from .sample import polynomial_expansion_set from .sample import PolynomialExpansionSet, Categorical, Float diff --git a/flaml/tune/cgmanifest.json b/flaml/tune/cgmanifest.json index 96626929b..03a3377cc 100644 --- a/flaml/tune/cgmanifest.json +++ b/flaml/tune/cgmanifest.json @@ -6,6 +6,6 @@ "pip": {"Name": "ray[tune]", "Version": "1.5.1" } }, "DevelopmentDependency": false - }, + } ] } \ No newline at end of file diff --git a/flaml/tune/space.py b/flaml/tune/space.py index 0ffbedac1..ff4d5fa98 100644 --- a/flaml/tune/space.py +++ b/flaml/tune/space.py @@ -1,9 +1,14 @@ try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune import sample -except ImportError: + from ray.tune.suggest.variant_generator import generate_variants +except (ImportError, AssertionError): from . import sample -from typing import Dict, Optional, Any + from ..searcher.variant_generator import generate_variants +from typing import Dict, Optional, Any, Tuple +import numpy as np import logging logger = logging.getLogger(__name__) @@ -15,7 +20,7 @@ def define_by_run_func( """Define-by-run function to create the search space. Returns: - None or a dict with constant values. + A dict with constant values. """ config = {} for key, domain in space.items(): @@ -50,12 +55,16 @@ def define_by_run_func( elif isinstance(domain, sample.Integer): if isinstance(sampler, sample.LogUniform): trial.suggest_int( - key, domain.lower, domain.upper, step=quantize or 1, log=True) + key, domain.lower, + domain.upper - int(bool(not quantize)), + step=quantize or 1, log=True) elif isinstance(sampler, sample.Uniform): # Upper bound should be inclusive for quantization and # exclusive otherwise trial.suggest_int( - key, domain.lower, domain.upper, step=quantize or 1) + key, domain.lower, + domain.upper - int(bool(not quantize)), + step=quantize or 1) elif isinstance(domain, sample.Categorical): if isinstance(sampler, sample.Uniform): if not hasattr(domain, 'choices'): @@ -76,3 +85,352 @@ def define_by_run_func( type(domain.sampler).__name__)) # Return all constants in a dictionary. return config + + +def unflatten_hierarchical(config: Dict, space: Dict) -> Tuple[Dict, Dict]: + '''unflatten hierarchical config''' + hier = {} + subspace = {} + for key, value in config.items(): + if '/' in key: + key = key[key.rfind('/') + 1:] + if ':' in key: + pos = key.rfind(':') + true_key = key[:pos] + choice = int(key[pos + 1:]) + hier[true_key], subspace[true_key] = unflatten_hierarchical( + value, space[true_key][choice]) + else: + domain = space.get(key) + if domain is not None: + subspace[key] = domain + if isinstance(domain, sample.Domain): + sampler = domain.sampler + if isinstance(sampler, sample.Quantized): + q = sampler.q + sampler = sampler.sampler + if isinstance(sampler, sample.LogUniform): + value = domain.cast(np.round(value / q) * q) + hier[key] = value + return hier, subspace + + +def add_cost_to_space(space: Dict, low_cost_point: Dict, choice_cost: Dict): + """Update the space in place by adding low_cost_point and choice_cost + + Returns: + A dict with constant values. + """ + config = {} + for key in space: + domain = space[key] + if not isinstance(domain, sample.Domain): + if isinstance(domain, dict): + low_cost = low_cost_point.get(key, {}) + choice_cost_list = choice_cost.get(key, {}) + const = add_cost_to_space( + domain, low_cost, choice_cost_list) + if const: + config[key] = const + else: + config[key] = domain + continue + low_cost = low_cost_point.get(key) + choice_cost_list = choice_cost.get(key) + if callable(getattr(domain, 'get_sampler', None)): + sampler = domain.get_sampler() + if isinstance(sampler, sample.Quantized): + sampler = sampler.get_sampler() + domain.bounded = str(sampler) != 'Normal' + if isinstance(domain, sample.Categorical): + domain.const = [] + for i, cat in enumerate(domain.categories): + if isinstance(cat, dict): + if isinstance(low_cost, list): + low_cost_dict = low_cost[i] + else: + low_cost_dict = {} + if choice_cost_list: + choice_cost_dict = choice_cost_list[i] + else: + choice_cost_dict = {} + domain.const.append(add_cost_to_space( + cat, low_cost_dict, choice_cost_dict)) + else: + domain.const.append(None) + if choice_cost_list: + if len(choice_cost_list) == len(domain.categories): + domain.choice_cost = choice_cost_list + else: + domain.choice_cost = choice_cost_list[-1] + # sort the choices by cost + cost = np.array(domain.choice_cost) + ind = np.argsort(cost) + domain.categories = [domain.categories[i] for i in ind] + domain.choice_cost = cost[ind] + domain.const = [domain.const[i] for i in ind] + domain.ordered = True + elif all(isinstance(x, int) or isinstance(x, float) + for x in domain.categories): + # sort the choices by value + ind = np.argsort(domain.categories) + domain.categories = [domain.categories[i] for i in ind] + domain.ordered = True + else: + domain.ordered = False + if low_cost and low_cost not in domain.categories: + assert isinstance(low_cost, list), \ + f"low cost {low_cost} not in domain {domain.categories}" + if domain.ordered: + sorted_points = [low_cost[i] for i in ind] + for i, point in enumerate(sorted_points): + low_cost[i] = point + if len(low_cost) > len(domain.categories): + if domain.ordered: + low_cost[-1] = int(np.where(ind == low_cost[-1])[0]) + domain.low_cost_point = low_cost[-1] + return + if low_cost: + domain.low_cost_point = low_cost + return config + + +def normalize( + config: Dict, space: Dict, reference_config: Dict, + normalized_reference_config: Dict, recursive: bool = False, +): + '''normalize config in space according to reference_config. + normalize each dimension in config to [0,1]. + ''' + config_norm = {} + for key in config: + value = config[key] + domain = space.get(key) + if domain is None: # e.g., prune_attr + config_norm[key] = value + continue + if not callable(getattr(domain, 'get_sampler', None)): + if recursive and isinstance(domain, dict): + config_norm[key] = normalize( + value, domain, reference_config[key], {}) + else: + config_norm[key] = value + continue + # domain: sample.Categorical/Integer/Float/Function + if isinstance(domain, sample.Categorical): + norm = None + # value is either one category, or the low_cost_point list + if value not in domain.categories: + # nested, low_cost_point list + if recursive: + norm = [] + for i, cat in enumerate(domain.categories): + norm.append(normalize( + value[i], cat, reference_config[key][i], {})) + if isinstance(value, list) and len(value) > len( + domain.categories): + # low_cost_point list + index = value[-1] + config[key] = value[index] + value = domain.categories[index] + else: + continue + # normalize categorical + n = len(domain.categories) + if domain.ordered: + normalized = (domain.categories.index(value) + 0.5) / n + elif key in normalized_reference_config: + normalized = normalized_reference_config[ + key] if value == reference_config[key] else ( + normalized_reference_config[key] + 1 / n) % 1 + else: + normalized = 0.5 + if norm: + norm.append(normalized) + else: + norm = normalized + config_norm[key] = norm + continue + # Uniform/LogUniform/Normal/Base + sampler = domain.get_sampler() + if isinstance(sampler, sample.Quantized): + # sampler is sample.Quantized + quantize = sampler.q + sampler = sampler.get_sampler() + else: + quantize = None + if str(sampler) == 'LogUniform': + upper = domain.upper - ( + isinstance(domain, sample.Integer) & (quantize is None)) + config_norm[key] = np.log(value / domain.lower) / np.log( + upper / domain.lower) + elif str(sampler) == 'Uniform': + upper = domain.upper - ( + isinstance(domain, sample.Integer) & (quantize is None)) + config_norm[key] = (value - domain.lower) / (upper - domain.lower) + elif str(sampler) == 'Normal': + # N(mean, sd) -> N(0,1) + config_norm[key] = (value - sampler.mean) / sampler.sd + else: + # TODO? elif str(sampler) == 'Base': # sample.Function._CallSampler + # e.g., {test: sample_from(lambda spec: randn(10, 2).sample() * 0.01)} + config_norm[key] = value + return config_norm + + +def denormalize( + config: Dict, space: Dict, reference_config: Dict, + normalized_reference_config: Dict, random_state +): + config_denorm = {} + for key, value in config.items(): + if key in space: + # domain: sample.Categorical/Integer/Float/Function + domain = space[key] + if not callable(getattr(domain, 'get_sampler', None)): + config_denorm[key] = value + else: + if isinstance(domain, sample.Categorical): + # denormalize categorical + n = len(domain.categories) + if domain.ordered: + config_denorm[key] = domain.categories[ + min(n - 1, int(np.floor(value * n)))] + else: + assert key in normalized_reference_config + if np.floor(value * n) == np.floor( + normalized_reference_config[key] * n): + config_denorm[key] = reference_config[key] + else: # ****random value each time!**** + config_denorm[key] = random_state.choice( + [x for x in domain.categories + if x != reference_config[key]]) + continue + # Uniform/LogUniform/Normal/Base + sampler = domain.get_sampler() + if isinstance(sampler, sample.Quantized): + # sampler is sample.Quantized + quantize = sampler.q + sampler = sampler.get_sampler() + else: + quantize = None + # Handle Log/Uniform + if str(sampler) == 'LogUniform': + upper = domain.upper - (isinstance(domain, sample.Integer) + & (quantize is None)) + config_denorm[key] = ( + upper / domain.lower) ** value * domain.lower + elif str(sampler) == 'Uniform': + upper = domain.upper - (isinstance(domain, sample.Integer) + & (quantize is None)) + config_denorm[key] = value * ( + upper - domain.lower) + domain.lower + elif str(sampler) == 'Normal': + # denormalization for 'Normal' + config_denorm[key] = value * sampler.sd + sampler.mean + else: + config_denorm[key] = value + # Handle quantized + if quantize is not None: + config_denorm[key] = np.round( + np.divide(config_denorm[key], quantize)) * quantize + # Handle int (4.6 -> 5) + if isinstance(domain, sample.Integer): + config_denorm[key] = int(round(config_denorm[key])) + else: # prune_attr + config_denorm[key] = value + return config_denorm + + +def indexof(domain: Dict, config: Dict) -> int: + '''find the index of config in domain.categories + ''' + index = config.get('_choice_') + if index is not None: + return index + if config in domain.categories: + return domain.categories.index(config) + # print(config) + for i, cat in enumerate(domain.categories): + # print(cat) + if not isinstance(cat, dict): + continue + # print(len(cat), len(config)) + if len(cat) != len(config): + continue + # print(cat.keys()) + if not set(cat.keys()).issubset(set(config.keys())): + continue + # print(domain.const[i]) + if all(config[key] == value for key, value in domain.const[i].items()): + return i + return None + + +def complete_config( + partial_config: Dict, space: Dict, flow2, disturb: bool = False, + lower: Optional[Dict] = None, upper: Optional[Dict] = None +) -> Tuple[Dict, Dict]: + '''Complete partial config in space + + Returns: + config, space + ''' + config = partial_config.copy() + normalized = normalize(config, space, config, {}) + if disturb: + for key in normalized: + domain = space.get(key) + if getattr(domain, 'ordered', True) is False: + # don't change unordered cat choice + continue + if not callable(getattr(domain, 'get_sampler', None)): + continue + if upper and lower: + up, low = upper[key], lower[key] + gauss_std = up - low or flow2.STEPSIZE + # allowed bound + up += flow2.STEPSIZE + low -= flow2.STEPSIZE + elif domain.bounded: + up, low, gauss_std = 1, 0, 1.0 + else: + up, low, gauss_std = np.Inf, -np.Inf, 1.0 + if domain.bounded: + up = min(up, 1) + low = max(low, 0) + delta = flow2.rand_vector_gaussian(1, gauss_std)[0] + normalized[key] = max(low, min(up, normalized[key] + delta)) + config = denormalize(normalized, space, config, normalized, flow2._random) + for key, value in space.items(): + if key not in config: + config[key] = value + for _, generated in generate_variants({'config': config}): + config = generated['config'] + break + subspace = {} + for key, domain in space.items(): + value = config[key] + if isinstance(value, dict): + if isinstance(domain, sample.Categorical): + # nested space + index = indexof(domain, value) + # point = partial_config.get(key) + # if isinstance(point, list): # low cost point list + # point = point[index] + # else: + # point = {} + config[key], subspace[key] = complete_config( + value, domain.categories[index], flow2, disturb, + lower and lower[key][index], upper and upper[key][index] + ) + assert '_choice_' not in subspace[key], \ + "_choice_ is a reserved key for hierarchical search space" + subspace[key]['_choice_'] = index + else: + config[key], subspace[key] = complete_config( + value, space[key], flow2, disturb, + lower and lower[key], upper and upper[key]) + continue + subspace[key] = domain + return config, subspace diff --git a/flaml/tune/trial.py b/flaml/tune/trial.py index ad15edb02..394c1ffaa 100644 --- a/flaml/tune/trial.py +++ b/flaml/tune/trial.py @@ -21,11 +21,10 @@ import uuid import time from numbers import Number from collections import deque -import copy def flatten_dict(dt, delimiter="/", prevent_delimiter=False): - dt = copy.deepcopy(dt) + dt = dt.copy() if prevent_delimiter and any(delimiter in key for key in dt): # Raise if delimiter is any of the keys raise ValueError( diff --git a/flaml/tune/trial_runner.py b/flaml/tune/trial_runner.py index 9f0e8c191..690e57df3 100644 --- a/flaml/tune/trial_runner.py +++ b/flaml/tune/trial_runner.py @@ -4,10 +4,12 @@ * project root for license information. ''' from typing import Optional -try: - from ray.tune.trial import Trial -except ImportError: - from .trial import Trial +# try: +# from ray import __version__ as ray_version +# assert ray_version >= '1.0.0' +# from ray.tune.trial import Trial +# except (ImportError, AssertionError): +from .trial import Trial import logging logger = logging.getLogger(__name__) diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py index fe4ac1cd5..a46054869 100644 --- a/flaml/tune/tune.py +++ b/flaml/tune/tune.py @@ -8,8 +8,10 @@ import numpy as np import datetime import time try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune.analysis import ExperimentAnalysis as EA -except ImportError: +except (ImportError, AssertionError): from .analysis import ExperimentAnalysis as EA import logging logger = logging.getLogger(__name__) @@ -288,9 +290,11 @@ def run(training_function, if reduction_factor: params['reduction_factor'] = reduction_factor try: + from ray import __version__ as ray_version + assert ray_version >= '1.0.0' from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(**params) - except ImportError: + except (ImportError, AssertionError): pass if use_ray: try: diff --git a/flaml/version.py b/flaml/version.py index 57f9f92e6..e1e093c04 100644 --- a/flaml/version.py +++ b/flaml/version.py @@ -1 +1 @@ -__version__ = "0.5.11" +__version__ = "0.5.12" diff --git a/test/hf/test_deberta.py b/test/hf/test_deberta.py index 92afe51f8..18c392785 100644 --- a/test/hf/test_deberta.py +++ b/test/hf/test_deberta.py @@ -52,6 +52,11 @@ try: except ImportError: print("pip install torch transformers datasets flaml[blendsearch,ray]") +except ValueError as e: + if 'Connection error' in str(e): + print(e) + else: + raise(e) import logging logger = logging.getLogger(__name__) diff --git a/test/hf/test_distillbert.py b/test/hf/test_distillbert.py index 1f1945517..a81e5fece 100644 --- a/test/hf/test_distillbert.py +++ b/test/hf/test_distillbert.py @@ -35,6 +35,11 @@ try: except ImportError: print("pip install torch transformers datasets flaml[blendsearch,ray]") +except ValueError as e: + if 'Connection error' in str(e): + print(e) + else: + raise(e) logger = logging.getLogger(__name__) os.makedirs('logs', exist_ok=True) diff --git a/test/hf/test_electra.py b/test/hf/test_electra.py index 2255452b4..b85e18c09 100644 --- a/test/hf/test_electra.py +++ b/test/hf/test_electra.py @@ -52,6 +52,11 @@ try: except ImportError: print("pip install torch transformers datasets flaml[blendsearch,ray]") +except ValueError as e: + if 'Connection error' in str(e): + print(e) + else: + raise(e) import logging logger = logging.getLogger(__name__) diff --git a/test/test_automl.py b/test/test_automl.py index e7ef9c6b3..7d5b2df16 100644 --- a/test/test_automl.py +++ b/test/test_automl.py @@ -156,11 +156,42 @@ class TestAutoML(unittest.TestCase): X = pd.DataFrame({ 'f1': [1, -2, 3, -4, 5, -6, -7, 8, -9, -10, -11, -12, -13, -14], 'f2': [3., 16., 10., 12., 3., 14., 11., 12., 5., 14., 20., 16., 15., 11.], - 'f3': ['a', 'b', 'a', 'c', 'c', 'b', 'b', 'b', 'b', 'a', 'b', 'e', 'e', 'a'], + 'f3': ['a', 'b', 'a', 'c', 'c', 'b', 'b', 'b', 'b', 'a', 'b', 1.0, 1.0, 'a'], 'f4': [True, True, False, True, True, False, False, False, True, True, False, False, True, True], }) y = pd.Series([0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]) + automl = AutoML() + automl_settings = { + "time_budget": 6, + "task": 'classification', + "n_jobs": 1, + "estimator_list": ['catboost', 'lrl2'], + "eval_method": "cv", + "n_splits": 3, + "metric": "accuracy", + "log_training_metric": True, + "verbose": 1, + "ensemble": True, + } + automl.fit(X, y, **automl_settings) + + automl = AutoML() + automl_settings = { + "time_budget": 2, + "task": 'classification', + "n_jobs": 1, + "estimator_list": ['lrl2', 'kneighbor'], + "eval_method": "cv", + "n_splits": 3, + "metric": "accuracy", + "log_training_metric": True, + "verbose": 1, + "ensemble": True, + } + automl.fit(X, y, **automl_settings) + + automl = AutoML() automl_settings = { "time_budget": 3, "task": 'classification', @@ -175,6 +206,21 @@ class TestAutoML(unittest.TestCase): } automl.fit(X, y, **automl_settings) + automl = AutoML() + automl_settings = { + "time_budget": 3, + "task": 'classification', + "n_jobs": 1, + "estimator_list": ['lgbm', 'catboost', 'kneighbor'], + "eval_method": "cv", + "n_splits": 3, + "metric": "accuracy", + "log_training_metric": True, + "verbose": 1, + "ensemble": True, + } + automl.fit(X, y, **automl_settings) + def test_dataframe(self): self.test_classification(True) diff --git a/test/test_notebook_example.py b/test/test_notebook_example.py index a669a65cb..12056ba26 100644 --- a/test/test_notebook_example.py +++ b/test/test_notebook_example.py @@ -1,6 +1,14 @@ -def test_automl(budget=5): +from openml.exceptions import OpenMLServerException + + +def test_automl(budget=5, dataset_format='dataframe'): from flaml.data import load_openml_dataset - X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir='test/') + try: + X_train, X_test, y_train, y_test = load_openml_dataset( + dataset_id=1169, data_dir='test/', dataset_format=dataset_format) + except OpenMLServerException: + print("OpenMLServerException raised") + return ''' import AutoML class from flaml package ''' from flaml import AutoML automl = AutoML() @@ -42,13 +50,22 @@ def test_automl(budget=5): print(automl.min_resource) +def test_automl_array(): + test_automl(5, 'array') + + def test_mlflow(): import subprocess import sys subprocess.check_call([sys.executable, "-m", "pip", "install", "mlflow"]) import mlflow from flaml.data import load_openml_task - X_train, X_test, y_train, y_test = load_openml_task(task_id=7592, data_dir='test/') + try: + X_train, X_test, y_train, y_test = load_openml_task( + task_id=7592, data_dir='test/') + except OpenMLServerException: + print("OpenMLServerException raised") + return ''' import AutoML class from flaml package ''' from flaml import AutoML automl = AutoML() diff --git a/test/test_python_log.py b/test/test_python_log.py index 732c881d1..d5027d596 100644 --- a/test/test_python_log.py +++ b/test/test_python_log.py @@ -1,3 +1,4 @@ +from flaml.tune.space import unflatten_hierarchical from flaml import AutoML from sklearn.datasets import load_boston import os @@ -45,13 +46,38 @@ class TestLogging(unittest.TestCase): logger.info(automl.search_space) logger.info(automl.low_cost_partial_config) logger.info(automl.points_to_evalaute) + logger.info(automl.cat_hp_cost) import optuna as ot study = ot.create_study() - from flaml.tune.space import define_by_run_func - logger.info(define_by_run_func(study.ask(), automl.search_space)) + from flaml.tune.space import define_by_run_func, add_cost_to_space + sample = define_by_run_func(study.ask(), automl.search_space) + logger.info(sample) + logger.info(unflatten_hierarchical(sample, automl.search_space)) + add_cost_to_space( + automl.search_space, automl.low_cost_partial_config, + automl.cat_hp_cost + ) + logger.info(automl.search_space["ml"].categories) config = automl.best_config.copy() config['learner'] = automl.best_estimator automl.trainable({"ml": config}) + from flaml import tune, CFO + search_alg = CFO( + metric='val_loss', + space=automl.search_space, + low_cost_partial_config=automl.low_cost_partial_config, + points_to_evaluate=automl.points_to_evalaute, + cat_hp_cost=automl.cat_hp_cost, + prune_attr=automl.prune_attr, + min_resource=automl.min_resource, + max_resource=automl.max_resource, + config_constraints=[(automl.size, '<=', automl._mem_thres)], + metric_constraints=automl.metric_constraints) + analysis = tune.run( + automl.trainable, search_alg=search_alg, # verbose=2, + time_budget_s=1, num_samples=-1) + print(min((trial.last_result["val_loss"], trial.last_result) + for trial in analysis.trials)) # Check if the log buffer is populated. self.assertTrue(len(buf.getvalue()) > 0) diff --git a/test/test_training_log.py b/test/test_training_log.py index 8895d5cfd..1b5952430 100644 --- a/test/test_training_log.py +++ b/test/test_training_log.py @@ -10,10 +10,10 @@ from flaml.training_log import training_log_reader class TestTrainingLog(unittest.TestCase): - def test_training_log(self): + def test_training_log(self, path='test_training_log.log'): with TemporaryDirectory() as d: - filename = os.path.join(d, 'test_training_log.log') + filename = os.path.join(d, path) # Run a simple job. automl_experiment = AutoML() @@ -42,3 +42,15 @@ class TestTrainingLog(unittest.TestCase): print(record) count += 1 self.assertGreater(count, 0) + + automl_settings["log_file_name"] = None + automl_experiment.fit(X_train=X_train, y_train=y_train, + **automl_settings) + + def test_illfilename(self): + try: + self.test_training_log('/') + except IsADirectoryError: + print("IsADirectoryError happens as expected in linux.") + except PermissionError: + print("PermissionError happens as expected in windows.") diff --git a/test/test_xgboost2d.py b/test/test_xgboost2d.py index c9f380c67..a98ec00ba 100644 --- a/test/test_xgboost2d.py +++ b/test/test_xgboost2d.py @@ -59,7 +59,7 @@ def test_simple(method=None): automl.trainable(config) from flaml import tune analysis = tune.run( - automl.trainable, automl.search_space, metric='val_loss', + automl.trainable, automl.search_space, metric='val_loss', mode="min", low_cost_partial_config=automl.low_cost_partial_config, points_to_evaluate=automl.points_to_evalaute, cat_hp_cost=automl.cat_hp_cost, @@ -68,7 +68,7 @@ def test_simple(method=None): max_resource=automl.max_resource, time_budget_s=automl._state.time_budget, config_constraints=[(automl.size, '<=', automl._mem_thres)], - metric_constraints=automl.metric_constraints) + metric_constraints=automl.metric_constraints, num_samples=5) print(analysis.trials[-1]) diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py index 1d0f96032..ded2b294f 100644 --- a/test/tune/test_tune.py +++ b/test/tune/test_tune.py @@ -52,7 +52,7 @@ def _test_xgboost(method='BlendSearch'): else: from ray import tune search_space = { - "max_depth": tune.randint(1, 8) if method in [ + "max_depth": tune.randint(1, 9) if method in [ "BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), @@ -61,7 +61,7 @@ def _test_xgboost(method='BlendSearch'): max_iter = 10 for num_samples in [128]: time_budget_s = 60 - for n_cpu in [8]: + for n_cpu in [4]: start_time = time.time() ray.init(num_cpus=n_cpu, num_gpus=0) # ray.init(address='auto') @@ -168,7 +168,7 @@ def test_nested(): search_space = { # test nested search space "cost_related": { - "a": tune.randint(1, 8), + "a": tune.randint(1, 9), }, "b": tune.uniform(0.5, 1.0), } @@ -194,7 +194,7 @@ def test_nested(): metric_constraints=[("ab", "<=", 4)]), local_dir='logs/', num_samples=-1, - time_budget_s=.1) + time_budget_s=1) best_trial = analysis.get_best_trial() logger.info(f"CFO best config: {best_trial.config}") @@ -216,7 +216,7 @@ def test_nested(): metric_constraints=[("ab", "<=", 4)]), local_dir='logs/', num_samples=-1, - time_budget_s=.1) + time_budget_s=1) best_trial = analysis.get_best_trial() logger.info(f"BlendSearch exp best config: {best_trial.config}") @@ -233,7 +233,7 @@ def test_nested(): metric_constraints=[("ab", "<=", 4)], local_dir='logs/', num_samples=-1, - time_budget_s=.1) + time_budget_s=1) best_trial = analysis.get_best_trial() logger.info(f"BlendSearch best config: {best_trial.config}") @@ -251,10 +251,11 @@ def test_run_training_function_return_value(): tune.run( evaluate_config_dict, config={ - 'x': tune.lograndint(lower=1, upper=100000), - 'y': tune.randint(lower=1, upper=100000) + 'x': tune.qloguniform(lower=1, upper=100000, q=1), + 'y': tune.qrandint(lower=2, upper=100000, q=2) }, - metric='metric', + metric='metric', mode='max', + num_samples=100, ) # Test scalar return value @@ -265,9 +266,10 @@ def test_run_training_function_return_value(): tune.run( evaluate_config_scalar, config={ - 'x': tune.lograndint(lower=1, upper=100000), - 'y': tune.randint(lower=1, upper=100000) + 'x': tune.qloguniform(lower=1, upper=100000, q=1), + 'y': tune.qlograndint(lower=2, upper=100000, q=2) }, + num_samples=100, mode='max', )