yujunjun/autogen
1
0
Fork 0
mirror of https://github.com/microsoft/autogen.git synced 2025-09-26 08:34:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

include default value in rf search space (#317)

Browse Source 
* include default value in rf search space

* init _mem_per_iter with -1

* bump version to 0.8.2

* docstr for search space's arguments
This commit is contained in:
Chi Wang 2021-12-03 09:15:21 -08:00 committed by GitHub
parent 1545d5a6d2
commit c57954fbbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 57 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
flaml/automl.py
View File

@ -119,7 +119,7 @@ class SearchState:
if config and "FLAML_sample_size" in config: if config and "FLAML_sample_size" in config:
self.sample_size = config["FLAML_sample_size"] self.sample_size = config["FLAML_sample_size"]
else: else:
self.sample_size = self.data_size self.sample_size = self.data_size[0]
obj = result["val_loss"] obj = result["val_loss"]
metric_for_logging = result["metric_for_logging"] metric_for_logging = result["metric_for_logging"]
time2eval = result["time_total_s"] time2eval = result["time_total_s"]
@ -181,7 +181,7 @@ class SearchState:
class AutoMLState: class AutoMLState:
def _prepare_sample_train_data(self, sample_size): def _prepare_sample_train_data(self, sample_size):
sampled_weight = groups = None sampled_weight = groups = None
if sample_size <= self.data_size: if sample_size <= self.data_size[0]:
if isinstance(self.X_train, pd.DataFrame): if isinstance(self.X_train, pd.DataFrame):
sampled_X_train = self.X_train.iloc[:sample_size] sampled_X_train = self.X_train.iloc[:sample_size]
else: else:
@ -205,7 +205,7 @@ class AutoMLState:
if "FLAML_sample_size" in config_w_resource: if "FLAML_sample_size" in config_w_resource:
sample_size = int(config_w_resource["FLAML_sample_size"]) sample_size = int(config_w_resource["FLAML_sample_size"])
else: else:
sample_size = self.data_size sample_size = self.data_size[0]
( (
sampled_X_train, sampled_X_train,
sampled_y_train, sampled_y_train,
@ -226,11 +226,11 @@ class AutoMLState:
None None
if self.time_budget is None if self.time_budget is None
else self.time_budget - self.time_from_start else self.time_budget - self.time_from_start
if sample_size == self.data_size if sample_size == self.data_size[0]
else (self.time_budget - self.time_from_start) else (self.time_budget - self.time_from_start)
/ 2 / 2
* sample_size * sample_size
/ self.data_size / self.data_size[0]
) )
if _is_nlp_task(self.task): if _is_nlp_task(self.task):
@ -1122,7 +1122,7 @@ class AutoML(BaseEstimator):
test_size=split_ratio, test_size=split_ratio,
random_state=RANDOM_SEED, random_state=RANDOM_SEED,
) )
self._state.data_size = X_train.shape[0] self._state.data_size = X_train.shape
self.data_size_full = len(y_train_all) self.data_size_full = len(y_train_all)
self._state.X_train, self._state.y_train = X_train, y_train self._state.X_train, self._state.y_train = X_train, y_train
self._state.X_val, self._state.y_val = X_val, y_val self._state.X_val, self._state.y_val = X_val, y_val
@ -1555,7 +1555,7 @@ class AutoML(BaseEstimator):
Returns: Returns:
A float for the maximal sample size or None. A float for the maximal sample size or None.
""" """
return self._state.data_size if self._sample else None return self._state.data_size[0] if self._sample else None
@property @property
def trainable(self) -> Callable[[dict], Optional[float]]: def trainable(self) -> Callable[[dict], Optional[float]]:
@ -1965,7 +1965,10 @@ class AutoML(BaseEstimator):
sample sample
and task != "rank" and task != "rank"
and eval_method != "cv" and eval_method != "cv"
and (self._min_sample_size * SAMPLE_MULTIPLY_FACTOR < self._state.data_size) and (
self._min_sample_size * SAMPLE_MULTIPLY_FACTOR
< self._state.data_size[0]
)
) )
if "auto" == metric: if "auto" == metric:
if "binary" in self._state.task: if "binary" in self._state.task:
@ -2205,7 +2208,7 @@ class AutoML(BaseEstimator):
search_state.update(result, 0) search_state.update(result, 0)
if result["wall_clock_time"] is not None: if result["wall_clock_time"] is not None:
self._state.time_from_start = result["wall_clock_time"] self._state.time_from_start = result["wall_clock_time"]
if search_state.sample_size == self._state.data_size: if search_state.sample_size == self._state.data_size[0]:
self._iter_per_learner[estimator] += 1 self._iter_per_learner[estimator] += 1
if not self._fullsize_reached: if not self._fullsize_reached:
self._fullsize_reached = True self._fullsize_reached = True
@ -2275,7 +2278,7 @@ class AutoML(BaseEstimator):
self._max_iter = 0 self._max_iter = 0
self._best_estimator = estimator = self.estimator_list[0] self._best_estimator = estimator = self.estimator_list[0]
self._selected = state = self._search_states[estimator] self._selected = state = self._search_states[estimator]
state.best_config_sample_size = self._state.data_size state.best_config_sample_size = self._state.data_size[0]
state.best_config = ( state.best_config = (
state.init_config state.init_config
if isinstance(state.init_config, dict) if isinstance(state.init_config, dict)
@ -2298,7 +2301,7 @@ class AutoML(BaseEstimator):
or better or better
or (not self.best_estimator) or (not self.best_estimator)
or self._search_states[self.best_estimator].sample_size or self._search_states[self.best_estimator].sample_size
< self._state.data_size < self._state.data_size[0]
else time_left - est_retrain_time else time_left - est_retrain_time
) )
if not search_state.search_alg: if not search_state.search_alg:
@ -2309,7 +2312,7 @@ class AutoML(BaseEstimator):
if self._sample: if self._sample:
prune_attr = "FLAML_sample_size" prune_attr = "FLAML_sample_size"
min_resource = self._min_sample_size min_resource = self._min_sample_size
max_resource = self._state.data_size max_resource = self._state.data_size[0]
else: else:
prune_attr = min_resource = max_resource = None prune_attr = min_resource = max_resource = None
learner_class = self._state.learner_classes.get(estimator) learner_class = self._state.learner_classes.get(estimator)
@ -2398,7 +2401,7 @@ class AutoML(BaseEstimator):
min_budget = max(10 * self._eci[0], sum(self._eci)) min_budget = max(10 * self._eci[0], sum(self._eci))
max_budget = 10000 * self._eci[0] max_budget = 10000 * self._eci[0]
if search_state.sample_size: if search_state.sample_size:
ratio = search_state.data_size / search_state.sample_size ratio = search_state.data_size[0] / search_state.sample_size
min_budget *= ratio min_budget *= ratio
max_budget *= ratio max_budget *= ratio
logger.info( logger.info(
@ -2408,7 +2411,7 @@ class AutoML(BaseEstimator):
if result["wall_clock_time"] is not None: if result["wall_clock_time"] is not None:
self._state.time_from_start = result["wall_clock_time"] self._state.time_from_start = result["wall_clock_time"]
# logger.info(f"{self._search_states[estimator].sample_size}, {data_size}") # logger.info(f"{self._search_states[estimator].sample_size}, {data_size}")
if search_state.sample_size == self._state.data_size: if search_state.sample_size == self._state.data_size[0]:
self._iter_per_learner[estimator] += 1 self._iter_per_learner[estimator] += 1
self._fullsize_reached = True self._fullsize_reached = True
if search_state.best_loss < self._state.best_loss: if search_state.best_loss < self._state.best_loss:
@ -2519,7 +2522,7 @@ class AutoML(BaseEstimator):
and est_retrain_time and est_retrain_time
and not better and not better
and self._search_states[self._best_estimator].sample_size and self._search_states[self._best_estimator].sample_size
== self._state.data_size == self._state.data_size[0]
and ( and (
est_retrain_time est_retrain_time
<= self._state.time_budget - self._state.time_from_start <= self._state.time_budget - self._state.time_from_start
@ -2560,7 +2563,7 @@ class AutoML(BaseEstimator):
self._best_iteration = 0 self._best_iteration = 0
self._time_taken_best_iter = 0 self._time_taken_best_iter = 0
self._config_history = {} self._config_history = {}
self._max_iter_per_learner = 1000000 # TODO self._max_iter_per_learner = 10000
self._iter_per_learner = dict([(e, 0) for e in self.estimator_list]) self._iter_per_learner = dict([(e, 0) for e in self.estimator_list])
self._fullsize_reached = False self._fullsize_reached = False
self._trained_estimator = None self._trained_estimator = None
@ -2680,7 +2683,7 @@ class AutoML(BaseEstimator):
self._state.time_budget - self._state.time_from_start self._state.time_budget - self._state.time_from_start
> self._selected.est_retrain_time(self.data_size_full) > self._selected.est_retrain_time(self.data_size_full)
and self._selected.best_config_sample_size and self._selected.best_config_sample_size
== self._state.data_size == self._state.data_size[0]
) )
): ):
state = self._search_states[self._best_estimator] state = self._search_states[self._best_estimator]
@ -2736,13 +2739,13 @@ class AutoML(BaseEstimator):
inv.append(0) inv.append(0)
continue continue
estimated_cost = search_state.estimated_cost4improvement estimated_cost = search_state.estimated_cost4improvement
if search_state.sample_size < self._state.data_size: if search_state.sample_size < self._state.data_size[0]:
estimated_cost = min( estimated_cost = min(
estimated_cost, estimated_cost,
search_state.time2eval_best search_state.time2eval_best
* min( * min(
SAMPLE_MULTIPLY_FACTOR, SAMPLE_MULTIPLY_FACTOR,
self._state.data_size / search_state.sample_size, self._state.data_size[0] / search_state.sample_size,
), ),
) )
gap = search_state.best_loss - self._state.best_loss gap = search_state.best_loss - self._state.best_loss

47
flaml/model.py
View File

@ -233,9 +233,13 @@ class BaseEstimator:
self._model = None self._model = None
@classmethod @classmethod
def search_space(cls, **params): def search_space(cls, data_size, task, **params):
"""[required method] search space. """[required method] search space.
Args:
data_size: A tuple of two integers, number of rows and columns.
task: A str of the task type, e.g., "binary", "multi", "regression".
Returns: Returns:
A dictionary of the search space. A dictionary of the search space.
Each key is the name of a hyperparameter, and value is a dict with Each key is the name of a hyperparameter, and value is a dict with
@ -674,7 +678,7 @@ class LGBMEstimator(BaseEstimator):
@classmethod @classmethod
def search_space(cls, data_size, **params): def search_space(cls, data_size, **params):
upper = min(32768, int(data_size)) upper = min(32768, int(data_size[0]))
return { return {
"n_estimators": { "n_estimators": {
"domain": tune.lograndint(lower=4, upper=upper), "domain": tune.lograndint(lower=4, upper=upper),
@ -728,7 +732,7 @@ class LGBMEstimator(BaseEstimator):
round( round(
config.get("num_leaves") config.get("num_leaves")
or config.get("max_leaves") or config.get("max_leaves")
or 1 << config["max_depth"] or 1 << config.get("max_depth", 16)
) )
) )
n_estimators = int(round(config["n_estimators"])) n_estimators = int(round(config["n_estimators"]))
@ -752,7 +756,7 @@ class LGBMEstimator(BaseEstimator):
self.estimator_class = LGBMClassifier self.estimator_class = LGBMClassifier
self._time_per_iter = None self._time_per_iter = None
self._train_size = 0 self._train_size = 0
self._mem_per_iter = 1 self._mem_per_iter = -1
self.HAS_CALLBACK = self.HAS_CALLBACK and self._callbacks(0, 0) is not None self.HAS_CALLBACK = self.HAS_CALLBACK and self._callbacks(0, 0) is not None
def _preprocess(self, X): def _preprocess(self, X):
@ -784,7 +788,7 @@ class LGBMEstimator(BaseEstimator):
or abs(self._train_size - X_train.shape[0]) > 4 or abs(self._train_size - X_train.shape[0]) > 4
) )
and budget is not None and budget is not None
or self._mem_per_iter <= 1 or self._mem_per_iter < 0
and psutil is not None and psutil is not None
) and n_iter > 1: ) and n_iter > 1:
self.params[self.ITER_HP] = 1 self.params[self.ITER_HP] = 1
@ -806,8 +810,8 @@ class LGBMEstimator(BaseEstimator):
self._mem_per_iter = min( self._mem_per_iter = min(
self._mem1, self._mem2 / self.params[self.ITER_HP] self._mem1, self._mem2 / self.params[self.ITER_HP]
) )
if self._mem_per_iter <= 1 and psutil is not None: # if self._mem_per_iter <= 1 and psutil is not None:
n_iter = self.params[self.ITER_HP] # n_iter = self.params[self.ITER_HP]
self._time_per_iter = ( self._time_per_iter = (
(self._t2 - self._t1) / (self.params[self.ITER_HP] - 1) (self._t2 - self._t1) / (self.params[self.ITER_HP] - 1)
if self._t2 > self._t1 if self._t2 > self._t1
@ -837,7 +841,7 @@ class LGBMEstimator(BaseEstimator):
if budget is not None if budget is not None
else n_iter, else n_iter,
int((1 - FREE_MEM_RATIO) * mem0 / self._mem_per_iter) int((1 - FREE_MEM_RATIO) * mem0 / self._mem_per_iter)
if psutil is not None if psutil is not None and self._mem_per_iter > 0
else n_iter, else n_iter,
) )
if trained and max_iter <= self.params[self.ITER_HP]: if trained and max_iter <= self.params[self.ITER_HP]:
@ -887,7 +891,7 @@ class XGBoostEstimator(SKLearnEstimator):
@classmethod @classmethod
def search_space(cls, data_size, **params): def search_space(cls, data_size, **params):
upper = min(32768, int(data_size)) upper = min(32768, int(data_size[0]))
return { return {
"n_estimators": { "n_estimators": {
"domain": tune.lograndint(lower=4, upper=upper), "domain": tune.lograndint(lower=4, upper=upper),
@ -1086,7 +1090,7 @@ class XGBoostLimitDepthEstimator(XGBoostSklearnEstimator):
def search_space(cls, data_size, **params): def search_space(cls, data_size, **params):
space = XGBoostEstimator.search_space(data_size) space = XGBoostEstimator.search_space(data_size)
space.pop("max_leaves") space.pop("max_leaves")
upper = max(6, int(np.log2(data_size))) upper = max(6, int(np.log2(data_size[0])))
space["max_depth"] = { space["max_depth"] = {
"domain": tune.randint(lower=1, upper=min(upper, 16)), "domain": tune.randint(lower=1, upper=min(upper, 16)),
"init_value": 6, "init_value": 6,
@ -1105,11 +1109,14 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
"""The class for tuning Random Forest.""" """The class for tuning Random Forest."""
HAS_CALLBACK = False HAS_CALLBACK = False
nrows = 101
@classmethod @classmethod
def search_space(cls, data_size, task, **params): def search_space(cls, data_size, task, **params):
data_size = int(data_size) RandomForestEstimator.nrows = int(data_size[0])
upper = min(2048, data_size) upper = min(2048, RandomForestEstimator.nrows)
init = 1 / np.sqrt(data_size[1]) if task in CLASSIFICATION else 1
lower = min(0.1, init)
space = { space = {
"n_estimators": { "n_estimators": {
"domain": tune.lograndint(lower=4, upper=upper), "domain": tune.lograndint(lower=4, upper=upper),
@ -1117,11 +1124,13 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
"low_cost_init_value": 4, "low_cost_init_value": 4,
}, },
"max_features": { "max_features": {
"domain": tune.loguniform(lower=0.1, upper=1.0), "domain": tune.loguniform(lower=lower, upper=1.0),
"init_value": 1.0, "init_value": init,
}, },
"max_leaves": { "max_leaves": {
"domain": tune.lograndint(lower=4, upper=min(32768, data_size)), "domain": tune.lograndint(
lower=4, upper=min(32768, RandomForestEstimator.nrows >> 1)
),
"init_value": 4, "init_value": 4,
"low_cost_init_value": 4, "low_cost_init_value": 4,
}, },
@ -1129,13 +1138,13 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
if task in CLASSIFICATION: if task in CLASSIFICATION:
space["criterion"] = { space["criterion"] = {
"domain": tune.choice(["gini", "entropy"]), "domain": tune.choice(["gini", "entropy"]),
# 'init_value': 'gini', # "init_value": "gini",
} }
return space return space
@classmethod @classmethod
def cost_relative2lgbm(cls): def cost_relative2lgbm(cls):
return 2.0 return 2
def config2params(cls, config: dict) -> dict: def config2params(cls, config: dict) -> dict:
params = config.copy() params = config.copy()
@ -1234,7 +1243,7 @@ class CatBoostEstimator(BaseEstimator):
@classmethod @classmethod
def search_space(cls, data_size, **params): def search_space(cls, data_size, **params):
upper = max(min(round(1500000 / data_size), 150), 12) upper = max(min(round(1500000 / data_size[0]), 150), 12)
return { return {
"early_stopping_rounds": { "early_stopping_rounds": {
"domain": tune.lograndint(lower=10, upper=upper), "domain": tune.lograndint(lower=10, upper=upper),
@ -1380,7 +1389,7 @@ class CatBoostEstimator(BaseEstimator):
class KNeighborsEstimator(BaseEstimator): class KNeighborsEstimator(BaseEstimator):
@classmethod @classmethod
def search_space(cls, data_size, **params): def search_space(cls, data_size, **params):
upper = min(512, int(data_size / 2)) upper = min(512, int(data_size[0] / 2))
return { return {
"n_neighbors": { "n_neighbors": {
"domain": tune.lograndint(lower=1, upper=upper), "domain": tune.lograndint(lower=1, upper=upper),

2
flaml/version.py
View File

@ -1 +1 @@
__version__ = "0.8.1" __version__ = "0.8.2"

6
notebook/flaml_automl.ipynb
View File

@ -893,8 +893,8 @@
" {'domain': tune.randint(lower=1, upper=10), 'init_value': 1}.\n", " {'domain': tune.randint(lower=1, upper=10), 'init_value': 1}.\n",
" '''\n", " '''\n",
" space = { \n", " space = { \n",
" 'max_leaf': {'domain': tune.lograndint(lower=4, upper=data_size), 'init_value': 4, 'low_cost_init_value': 4},\n", " 'max_leaf': {'domain': tune.lograndint(lower=4, upper=data_size[0]), 'init_value': 4, 'low_cost_init_value': 4},\n",
" 'n_iter': {'domain': tune.lograndint(lower=1, upper=data_size), 'init_value': 1, 'low_cost_init_value': 1},\n", " 'n_iter': {'domain': tune.lograndint(lower=1, upper=data_size[0]), 'init_value': 1, 'low_cost_init_value': 1},\n",
" 'n_tree_search': {'domain': tune.lograndint(lower=1, upper=32768), 'init_value': 1, 'low_cost_init_value': 1},\n", " 'n_tree_search': {'domain': tune.lograndint(lower=1, upper=32768), 'init_value': 1, 'low_cost_init_value': 1},\n",
" 'opt_interval': {'domain': tune.lograndint(lower=1, upper=10000), 'init_value': 100},\n", " 'opt_interval': {'domain': tune.lograndint(lower=1, upper=10000), 'init_value': 100},\n",
" 'learning_rate': {'domain': tune.loguniform(lower=0.01, upper=20.0)},\n", " 'learning_rate': {'domain': tune.loguniform(lower=0.01, upper=20.0)},\n",
@ -1278,7 +1278,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.7.12" "version": "3.9.7"
} }
}, },
"nbformat": 4, "nbformat": 4,

4
test/automl/test_multiclass.py
View File

@ -29,11 +29,11 @@ class MyRegularizedGreedyForest(SKLearnEstimator):
def search_space(cls, data_size, task): def search_space(cls, data_size, task):
space = { space = {
"max_leaf": { "max_leaf": {
"domain": tune.lograndint(lower=4, upper=data_size), "domain": tune.lograndint(lower=4, upper=data_size[0]),
"init_value": 4, "init_value": 4,
}, },
"n_iter": { "n_iter": {
"domain": tune.lograndint(lower=1, upper=data_size), "domain": tune.lograndint(lower=1, upper=data_size[0]),
"init_value": 1, "init_value": 1,
}, },
"n_tree_search": { "n_tree_search": {

2
test/automl/test_xgboost2d.py
View File

@ -13,7 +13,7 @@ dataset = "credit-g"
class XGBoost2D(XGBoostSklearnEstimator): class XGBoost2D(XGBoostSklearnEstimator):
@classmethod @classmethod
def search_space(cls, data_size, task): def search_space(cls, data_size, task):
upper = min(32768, int(data_size)) upper = min(32768, int(data_size[0]))
return { return {
"n_estimators": { "n_estimators": {
"domain": tune.lograndint(lower=4, upper=upper), "domain": tune.lograndint(lower=4, upper=upper),