yujunjun/autogen
1
0
Fork 0
mirror of https://github.com/microsoft/autogen.git synced 2025-11-07 21:34:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
skzhang1 2022-08-15 14:41:30 +00:00
parent 3f33a9700b
commit 34085b8c25
2 changed files with 44 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
flaml/automl.py
View File

@ -1706,7 +1706,11 @@ class AutoML(BaseEstimator):
self._state.fit_kwargs = fit_kwargs self._state.fit_kwargs = fit_kwargs
self._state.custom_hp = custom_hp or self._settings.get("custom_hp") self._state.custom_hp = custom_hp or self._settings.get("custom_hp")
self._skip_transform = self._settings.get("skip_transform") if skip_transform is None else skip_transform self._skip_transform = (
self._settings.get("skip_transform")
if skip_transform is None
else skip_transform
)
self._state.fit_kwargs_by_estimator = ( self._state.fit_kwargs_by_estimator = (
fit_kwargs_by_estimator or self._settings.get("fit_kwargs_by_estimator") fit_kwargs_by_estimator or self._settings.get("fit_kwargs_by_estimator")
) )
@ -2357,28 +2361,27 @@ class AutoML(BaseEstimator):
have the following signature: have the following signature:
```python ```python
def cv_score_agg_func(metrics_across_folds): def cv_score_agg_func(val_loss_folds, log_metrics_folds):
return metric_to_minimize, metrics_to_log return metric_to_minimize, metrics_to_log
``` ```
The input "metrics_across_folds" is a list of 2-tuples. Each tuple records the loss and metrics information of the corresponding fold. “val_loss_folds” - list of float, it records the loss scores of each ford; “log_metrics_folds” - list of dict/float, it records the metrics of each fords to log.
On each tuple, the first element is a float number that represents the loss score to minimize, and the second is a dict of all the metrics to log or None. This function should return the final aggregate result of all folds. A float number of the minimization objective, and a dictionary as the metrics to log or None.
It returns the final aggregate result of all folds. A float number of the minimization objective, and a dictionary as the metrics to log or None.
E.g., E.g.,
```python ```python
def cv_score_agg_func(metrics_across_folds): def cv_score_agg_func(val_loss_folds, log_metrics_folds):
metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds) metric_to_minimize = sum(val_loss_folds)/len(val_loss_folds)
metrics_to_log = None metrics_to_log = None
for single_fold in metrics_across_folds: for single_fold in log_metrics_folds:
if single_fold[1] is None: if metrics_to_log is None:
break metrics_to_log = single_fold
elif metrics_to_log is None: elif isinstance(metrics_to_log, dict):
metrics_to_log = single_fold[1] metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}
else: else:
metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()} metrics_to_log += single_fold
if metrics_to_log: if metrics_to_log:
n = len(metrics_across_folds) n = len(val_loss_folds)
metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log/n
return metric_to_minimize, metrics_to_log return metric_to_minimize, metrics_to_log
``` ```
@ -2549,7 +2552,11 @@ class AutoML(BaseEstimator):
self._state.fit_kwargs = fit_kwargs self._state.fit_kwargs = fit_kwargs
custom_hp = custom_hp or self._settings.get("custom_hp") custom_hp = custom_hp or self._settings.get("custom_hp")
self._skip_transform = self._settings.get("skip_transform") if skip_transform is None else skip_transform self._skip_transform = (
self._settings.get("skip_transform")
if skip_transform is None
else skip_transform
)
fit_kwargs_by_estimator = fit_kwargs_by_estimator or self._settings.get( fit_kwargs_by_estimator = fit_kwargs_by_estimator or self._settings.get(
"fit_kwargs_by_estimator" "fit_kwargs_by_estimator"
) )
@ -2579,7 +2586,9 @@ class AutoML(BaseEstimator):
eval_method = self._decide_eval_method(eval_method, time_budget) eval_method = self._decide_eval_method(eval_method, time_budget)
self._state.eval_method = eval_method self._state.eval_method = eval_method
logger.info("Evaluation method: {}".format(eval_method)) logger.info("Evaluation method: {}".format(eval_method))
self._state.cv_score_agg_func = cv_score_agg_func or self._settings.get("cv_score_agg_func") self._state.cv_score_agg_func = cv_score_agg_func or self._settings.get(
"cv_score_agg_func"
)
self._retrain_in_budget = retrain_full == "budget" and ( self._retrain_in_budget = retrain_full == "budget" and (
eval_method == "holdout" and self._state.X_val is None eval_method == "holdout" and self._state.X_val is None

31
flaml/ml.py
View File

@ -430,21 +430,27 @@ def get_val_loss(
train_time = time.time() - start train_time = time.time() - start
return val_loss, metric_for_logging, train_time, pred_time return val_loss, metric_for_logging, train_time, pred_time
def default_cv_score_agg_func(metrics_across_folds):
metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds) def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):
metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)
metrics_to_log = None metrics_to_log = None
for single_fold in metrics_across_folds: for single_fold in log_metrics_folds:
if single_fold[1] is None: if metrics_to_log is None:
break metrics_to_log = single_fold
elif metrics_to_log is None: elif isinstance(metrics_to_log, dict):
metrics_to_log = single_fold[1] metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}
else: else:
metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()} metrics_to_log += single_fold
if metrics_to_log: if metrics_to_log:
n = len(metrics_across_folds) n = len(val_loss_folds)
metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} metrics_to_log = (
{k: v / n for k, v in metrics_to_log.items()}
if isinstance(metrics_to_log, dict)
else metrics_to_log / n
)
return metric_to_minimize, metrics_to_log return metric_to_minimize, metrics_to_log
def evaluate_model_CV( def evaluate_model_CV(
config, config,
estimator, estimator,
@ -541,10 +547,7 @@ def evaluate_model_CV(
pred_time += pred_time_i pred_time += pred_time_i
if time.time() - start_time >= budget: if time.time() - start_time >= budget:
break break
if log_training_metric or not isinstance(eval_metric, str): val_loss, metric = cv_score_agg_func(val_loss_folds, log_metric_folds)
val_loss, metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds)))
else:
val_loss, metric = cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))
n = total_fold_num n = total_fold_num
pred_time /= n pred_time /= n
return val_loss, metric, train_time, pred_time return val_loss, metric, train_time, pred_time