From e5a422c41e621a94055295ddb12a5202a696fd11 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sun, 7 Aug 2022 18:11:04 +0000 Subject: [PATCH 01/20] update --- flaml/automl.py | 16 +++++++++++++++- flaml/ml.py | 16 +++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/flaml/automl.py b/flaml/automl.py index c14772c06..d6c335dd2 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -364,6 +364,7 @@ class AutoMLState: state.best_loss, state.n_jobs, state.learner_classes.get(estimator), + state.cv_strategy, state.log_training_metric, this_estimator_kwargs, ) @@ -2070,6 +2071,7 @@ class AutoML(BaseEstimator): use_ray=None, metric_constraints=None, custom_hp=None, + cv_strategy=None, fit_kwargs_by_estimator=None, **fit_kwargs, ): @@ -2287,6 +2289,17 @@ class AutoML(BaseEstimator): } ``` + cv_strategy: function, the strategy of conducting cross-validation. Default to average the optimization metric across folds. + We give an example here: + + ```python + def cv_strategy(val_loss_folds): + return sum(val_loss_folds)/len(val_loss_folds) + ``` + + where val_loss_folds is the list that stores the metrics values of all folds. In this example, we return the average of the optimization + metric across all folds (default strategy). + fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name. For TransformersEstimator, available fit_kwargs can be found from [TrainingArgumentsForAuto](nlp/huggingface/training_args). @@ -2307,7 +2320,7 @@ class AutoML(BaseEstimator): gpu_per_trial: float, default = 0 | A float of the number of gpus per trial, only used by TransformersEstimator and XGBoostSklearnEstimator. """ - + self._state._start_time_flag = self._start_time_flag = time.time() task = task or self._settings.get("task") self._estimator_type = "classifier" if task in CLASSIFICATION else "regressor" @@ -2447,6 +2460,7 @@ class AutoML(BaseEstimator): eval_method = self._decide_eval_method(eval_method, time_budget) self._state.eval_method = eval_method logger.info("Evaluation method: {}".format(eval_method)) + self._state.cv_strategy = cv_strategy self._retrain_in_budget = retrain_full == "budget" and ( eval_method == "holdout" and self._state.X_val is None diff --git a/flaml/ml.py b/flaml/ml.py index cc7a0e4b8..02ae93954 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -438,11 +438,14 @@ def evaluate_model_CV( task, eval_metric, best_val_loss, + cv_strategy, log_training_metric=False, fit_kwargs={}, ): + if cv_strategy is None: + cv_strategy = lambda val_loss_folds: sum(val_loss_folds)/len(val_loss_folds) start_time = time.time() - total_val_loss = 0 + val_loss_folds = [] total_metric = None metric = None train_time = pred_time = 0 @@ -515,7 +518,7 @@ def evaluate_model_CV( fit_kwargs["sample_weight"] = weight valid_fold_num += 1 total_fold_num += 1 - total_val_loss += val_loss_i + val_loss_folds.append(val_loss_i) if log_training_metric or not isinstance(eval_metric, str): if isinstance(total_metric, dict): total_metric = {k: total_metric[k] + v for k, v in metric_i.items()} @@ -526,10 +529,11 @@ def evaluate_model_CV( train_time += train_time_i pred_time += pred_time_i if valid_fold_num == n: - val_loss_list.append(total_val_loss / valid_fold_num) - total_val_loss = valid_fold_num = 0 + val_loss_list.append(cv_strategy(val_loss_folds)) + val_loss_folds = [] + valid_fold_num = 0 elif time.time() - start_time >= budget: - val_loss_list.append(total_val_loss / valid_fold_num) + val_loss_list.append(cv_strategy(val_loss_folds)) break val_loss = np.max(val_loss_list) n = total_fold_num @@ -559,6 +563,7 @@ def compute_estimator( best_val_loss=np.Inf, n_jobs=1, estimator_class=None, + cv_strategy=None, log_training_metric=False, fit_kwargs={}, ): @@ -605,6 +610,7 @@ def compute_estimator( task, eval_metric, best_val_loss, + cv_strategy, log_training_metric=log_training_metric, fit_kwargs=fit_kwargs, ) From 7851a463aaf1edebf58e943abba8cfe5903a1546 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sun, 7 Aug 2022 18:16:34 +0000 Subject: [PATCH 02/20] clean up --- flaml/automl.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flaml/automl.py b/flaml/automl.py index d6c335dd2..f19757bc6 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -2289,17 +2289,17 @@ class AutoML(BaseEstimator): } ``` - cv_strategy: function, the strategy of conducting cross-validation. Default to average the optimization metric across folds. + cv_strategy: customized function, the strategy of conducting cross-validation. Default to average the optimization metric across folds. We give an example here: ```python def cv_strategy(val_loss_folds): return sum(val_loss_folds)/len(val_loss_folds) ``` - - where val_loss_folds is the list that stores the metrics values of all folds. In this example, we return the average of the optimization + + where val_loss_folds is the list that stores the metrics values of all folds. In this example, we return the average of the optimization metric across all folds (default strategy). - + fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name. For TransformersEstimator, available fit_kwargs can be found from [TrainingArgumentsForAuto](nlp/huggingface/training_args). @@ -2320,7 +2320,7 @@ class AutoML(BaseEstimator): gpu_per_trial: float, default = 0 | A float of the number of gpus per trial, only used by TransformersEstimator and XGBoostSklearnEstimator. """ - + self._state._start_time_flag = self._start_time_flag = time.time() task = task or self._settings.get("task") self._estimator_type = "classifier" if task in CLASSIFICATION else "regressor" @@ -2460,7 +2460,7 @@ class AutoML(BaseEstimator): eval_method = self._decide_eval_method(eval_method, time_budget) self._state.eval_method = eval_method logger.info("Evaluation method: {}".format(eval_method)) - self._state.cv_strategy = cv_strategy + self._state.cv_strategy = cv_strategy self._retrain_in_budget = retrain_full == "budget" and ( eval_method == "holdout" and self._state.X_val is None From e3c9da50dabb46d3861ab4c6d4d4f394f60e23cd Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Wed, 10 Aug 2022 00:42:47 +0000 Subject: [PATCH 03/20] update --- flaml/automl.py | 47 +++++++++++++++++++++++++++++++++-------------- flaml/ml.py | 38 ++++++++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/flaml/automl.py b/flaml/automl.py index f19757bc6..ddaf478e7 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -364,7 +364,7 @@ class AutoMLState: state.best_loss, state.n_jobs, state.learner_classes.get(estimator), - state.cv_strategy, + state.cv_score_agg_func, state.log_training_metric, this_estimator_kwargs, ) @@ -729,6 +729,7 @@ class AutoML(BaseEstimator): settings["min_sample_size"] = settings.get("min_sample_size", MIN_SAMPLE_TRAIN) settings["use_ray"] = settings.get("use_ray", False) settings["metric_constraints"] = settings.get("metric_constraints", []) + settings["cv_score_agg_func"] = settings.get("cv_score_agg_func", None) settings["fit_kwargs_by_estimator"] = settings.get( "fit_kwargs_by_estimator", {} ) @@ -2071,7 +2072,7 @@ class AutoML(BaseEstimator): use_ray=None, metric_constraints=None, custom_hp=None, - cv_strategy=None, + cv_score_agg_func=None, fit_kwargs_by_estimator=None, **fit_kwargs, ): @@ -2289,21 +2290,39 @@ class AutoML(BaseEstimator): } ``` - cv_strategy: customized function, the strategy of conducting cross-validation. Default to average the optimization metric across folds. - We give an example here: + cv_score_agg_func: customized cross-validation scores aggregate function. Default to average metrics across folds. If specificed, this function needs to + have the following signature: ```python - def cv_strategy(val_loss_folds): - return sum(val_loss_folds)/len(val_loss_folds) + def cv_score_agg_func(metrics_across_folds): + return metric_to_minimize, metrics_to_log + ``` + The input "metrics_across_folds" is a list of 2-tuples. Each tuple records the loss and metrics information of the corresponding fold. + On each tuple, the first element is a float number that represents the loss score to minimize, and the second is a dict of all the metrics to log or None. + It returns the final aggregate result of all folds. A float number of the minimization objective, and a dictionary as the metrics to log or None. + E.g., + + ```python + def cv_score_agg_func(metrics_across_folds): + metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds) + metrics_to_log = None + for single_fold in metrics_across_folds: + if single_fold[1] is None: + break + elif metrics_to_log is None: + metrics_to_log = single_fold[1] + else: + metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()} + if metrics_to_log: + n = len(metrics_across_folds) + metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} + return metric_to_minimize, metrics_to_log ``` - where val_loss_folds is the list that stores the metrics values of all folds. In this example, we return the average of the optimization - metric across all folds (default strategy). - - fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name. - For TransformersEstimator, available fit_kwargs can be found from - [TrainingArgumentsForAuto](nlp/huggingface/training_args). - e.g., + fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name. + For TransformersEstimator, available fit_kwargs can be found from + [TrainingArgumentsForAuto](nlp/huggingface/training_args). + e.g., ```python fit_kwargs_by_estimator = { @@ -2460,7 +2479,7 @@ class AutoML(BaseEstimator): eval_method = self._decide_eval_method(eval_method, time_budget) self._state.eval_method = eval_method logger.info("Evaluation method: {}".format(eval_method)) - self._state.cv_strategy = cv_strategy + self._state.cv_score_agg_func = cv_score_agg_func or self._settings.get("cv_score_agg_func") self._retrain_in_budget = retrain_full == "budget" and ( eval_method == "holdout" and self._state.X_val is None diff --git a/flaml/ml.py b/flaml/ml.py index 02ae93954..0c442d1c2 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -438,14 +438,28 @@ def evaluate_model_CV( task, eval_metric, best_val_loss, - cv_strategy, + cv_score_agg_func, log_training_metric=False, fit_kwargs={}, ): - if cv_strategy is None: - cv_strategy = lambda val_loss_folds: sum(val_loss_folds)/len(val_loss_folds) + if cv_score_agg_func is None: + def cv_score_agg_func(metrics_across_folds): + metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds) + metrics_to_log = None + for single_fold in metrics_across_folds: + if single_fold[1] is None: + break + elif metrics_to_log is None: + metrics_to_log = single_fold[1] + else: + metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()} + if metrics_to_log: + n = len(metrics_across_folds) + metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} + return metric_to_minimize, metrics_to_log start_time = time.time() val_loss_folds = [] + log_metric_folds = [] total_metric = None metric = None train_time = pred_time = 0 @@ -520,8 +534,8 @@ def evaluate_model_CV( total_fold_num += 1 val_loss_folds.append(val_loss_i) if log_training_metric or not isinstance(eval_metric, str): - if isinstance(total_metric, dict): - total_metric = {k: total_metric[k] + v for k, v in metric_i.items()} + if isinstance(metric_i, dict): + log_metric_folds.append(metric_i) elif total_metric is not None: total_metric += metric_i else: @@ -529,17 +543,17 @@ def evaluate_model_CV( train_time += train_time_i pred_time += pred_time_i if valid_fold_num == n: - val_loss_list.append(cv_strategy(val_loss_folds)) - val_loss_folds = [] + val_loss_list.append(cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))[0]) valid_fold_num = 0 + val_loss_folds = [] elif time.time() - start_time >= budget: - val_loss_list.append(cv_strategy(val_loss_folds)) + val_loss_list.append(cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))[0]) break val_loss = np.max(val_loss_list) n = total_fold_num if log_training_metric or not isinstance(eval_metric, str): - if isinstance(total_metric, dict): - metric = {k: v / n for k, v in total_metric.items()} + if len(log_metric_folds): + metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds)))[1] else: metric = total_metric / n pred_time /= n @@ -563,7 +577,7 @@ def compute_estimator( best_val_loss=np.Inf, n_jobs=1, estimator_class=None, - cv_strategy=None, + cv_score_agg_func=None, log_training_metric=False, fit_kwargs={}, ): @@ -610,7 +624,7 @@ def compute_estimator( task, eval_metric, best_val_loss, - cv_strategy, + cv_score_agg_func, log_training_metric=log_training_metric, fit_kwargs=fit_kwargs, ) From fc633ef15e86fec76b959c5c86f7d65a2c753dcc Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sat, 13 Aug 2022 18:51:33 +0000 Subject: [PATCH 04/20] update --- flaml/ml.py | 57 +++++++++++++++++++++-------------------------------- 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/flaml/ml.py b/flaml/ml.py index 0c442d1c2..ec9a31063 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -427,6 +427,20 @@ def get_val_loss( train_time = time.time() - start return val_loss, metric_for_logging, train_time, pred_time +def default_cv_score_agg_func(metrics_across_folds): + metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds) + metrics_to_log = None + for single_fold in metrics_across_folds: + if single_fold[1] is None: + break + elif metrics_to_log is None: + metrics_to_log = single_fold[1] + else: + metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()} + if metrics_to_log: + n = len(metrics_across_folds) + metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} + return metric_to_minimize, metrics_to_log def evaluate_model_CV( config, @@ -438,32 +452,18 @@ def evaluate_model_CV( task, eval_metric, best_val_loss, - cv_score_agg_func, + cv_score_agg_func = None, log_training_metric=False, fit_kwargs={}, ): if cv_score_agg_func is None: - def cv_score_agg_func(metrics_across_folds): - metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds) - metrics_to_log = None - for single_fold in metrics_across_folds: - if single_fold[1] is None: - break - elif metrics_to_log is None: - metrics_to_log = single_fold[1] - else: - metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()} - if metrics_to_log: - n = len(metrics_across_folds) - metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} - return metric_to_minimize, metrics_to_log + cv_score_agg_func = default_cv_score_agg_func start_time = time.time() val_loss_folds = [] log_metric_folds = [] - total_metric = None metric = None train_time = pred_time = 0 - valid_fold_num = total_fold_num = 0 + total_fold_num = 0 n = kf.get_n_splits() X_train_split, y_train_split = X_train_all, y_train_all if task in CLASSIFICATION: @@ -485,7 +485,6 @@ def evaluate_model_CV( else: kf = kf.split(X_train_split) rng = np.random.RandomState(2020) - val_loss_list = [] budget_per_train = budget / n if "sample_weight" in fit_kwargs: weight = fit_kwargs["sample_weight"] @@ -530,32 +529,20 @@ def evaluate_model_CV( ) if weight is not None: fit_kwargs["sample_weight"] = weight - valid_fold_num += 1 total_fold_num += 1 val_loss_folds.append(val_loss_i) if log_training_metric or not isinstance(eval_metric, str): if isinstance(metric_i, dict): log_metric_folds.append(metric_i) - elif total_metric is not None: - total_metric += metric_i - else: - total_metric = metric_i train_time += train_time_i pred_time += pred_time_i - if valid_fold_num == n: - val_loss_list.append(cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))[0]) - valid_fold_num = 0 - val_loss_folds = [] - elif time.time() - start_time >= budget: - val_loss_list.append(cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))[0]) + if time.time() - start_time >= budget: break - val_loss = np.max(val_loss_list) - n = total_fold_num if log_training_metric or not isinstance(eval_metric, str): - if len(log_metric_folds): - metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds)))[1] - else: - metric = total_metric / n + val_loss, metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds))) + else: + val_loss, metric = cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds)))) + n = total_fold_num pred_time /= n return val_loss, metric, train_time, pred_time From 6485c28a08c8d202102bc8188b5fb24165efa695 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sat, 13 Aug 2022 18:57:28 +0000 Subject: [PATCH 05/20] fix --- flaml/automl.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/flaml/automl.py b/flaml/automl.py index 7da169abc..8bc558548 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -2132,11 +2132,8 @@ class AutoML(BaseEstimator): use_ray=None, metric_constraints=None, custom_hp=None, -<<<<<<< HEAD cv_score_agg_func=None, -======= skip_transform=None, ->>>>>>> main fit_kwargs_by_estimator=None, **fit_kwargs, ): @@ -2356,7 +2353,6 @@ class AutoML(BaseEstimator): } ``` -<<<<<<< HEAD cv_score_agg_func: customized cross-validation scores aggregate function. Default to average metrics across folds. If specificed, this function needs to have the following signature: @@ -2390,13 +2386,11 @@ class AutoML(BaseEstimator): For TransformersEstimator, available fit_kwargs can be found from [TrainingArgumentsForAuto](nlp/huggingface/training_args). e.g., -======= skip_transform: boolean, default=False | Whether to pre-process data prior to modeling. fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name. For TransformersEstimator, available fit_kwargs can be found from [TrainingArgumentsForAuto](nlp/huggingface/training_args). e.g., ->>>>>>> main ```python fit_kwargs_by_estimator = { From 34085b8c251b68f883654fc8394cb59954317e2c Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Mon, 15 Aug 2022 14:41:30 +0000 Subject: [PATCH 06/20] update --- flaml/automl.py | 43 ++++++++++++++++++++++++++----------------- flaml/ml.py | 33 ++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 32 deletions(-) diff --git a/flaml/automl.py b/flaml/automl.py index 8bc558548..ba71cfd09 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -1706,7 +1706,11 @@ class AutoML(BaseEstimator): self._state.fit_kwargs = fit_kwargs self._state.custom_hp = custom_hp or self._settings.get("custom_hp") - self._skip_transform = self._settings.get("skip_transform") if skip_transform is None else skip_transform + self._skip_transform = ( + self._settings.get("skip_transform") + if skip_transform is None + else skip_transform + ) self._state.fit_kwargs_by_estimator = ( fit_kwargs_by_estimator or self._settings.get("fit_kwargs_by_estimator") ) @@ -2357,28 +2361,27 @@ class AutoML(BaseEstimator): have the following signature: ```python - def cv_score_agg_func(metrics_across_folds): + def cv_score_agg_func(val_loss_folds, log_metrics_folds): return metric_to_minimize, metrics_to_log ``` - The input "metrics_across_folds" is a list of 2-tuples. Each tuple records the loss and metrics information of the corresponding fold. - On each tuple, the first element is a float number that represents the loss score to minimize, and the second is a dict of all the metrics to log or None. - It returns the final aggregate result of all folds. A float number of the minimization objective, and a dictionary as the metrics to log or None. + “val_loss_folds” - list of float, it records the loss scores of each ford; “log_metrics_folds” - list of dict/float, it records the metrics of each fords to log. + This function should return the final aggregate result of all folds. A float number of the minimization objective, and a dictionary as the metrics to log or None. E.g., ```python - def cv_score_agg_func(metrics_across_folds): - metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds) + def cv_score_agg_func(val_loss_folds, log_metrics_folds): + metric_to_minimize = sum(val_loss_folds)/len(val_loss_folds) metrics_to_log = None - for single_fold in metrics_across_folds: - if single_fold[1] is None: - break - elif metrics_to_log is None: - metrics_to_log = single_fold[1] + for single_fold in log_metrics_folds: + if metrics_to_log is None: + metrics_to_log = single_fold + elif isinstance(metrics_to_log, dict): + metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()} else: - metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()} + metrics_to_log += single_fold if metrics_to_log: - n = len(metrics_across_folds) - metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} + n = len(val_loss_folds) + metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log/n return metric_to_minimize, metrics_to_log ``` @@ -2549,7 +2552,11 @@ class AutoML(BaseEstimator): self._state.fit_kwargs = fit_kwargs custom_hp = custom_hp or self._settings.get("custom_hp") - self._skip_transform = self._settings.get("skip_transform") if skip_transform is None else skip_transform + self._skip_transform = ( + self._settings.get("skip_transform") + if skip_transform is None + else skip_transform + ) fit_kwargs_by_estimator = fit_kwargs_by_estimator or self._settings.get( "fit_kwargs_by_estimator" ) @@ -2579,7 +2586,9 @@ class AutoML(BaseEstimator): eval_method = self._decide_eval_method(eval_method, time_budget) self._state.eval_method = eval_method logger.info("Evaluation method: {}".format(eval_method)) - self._state.cv_score_agg_func = cv_score_agg_func or self._settings.get("cv_score_agg_func") + self._state.cv_score_agg_func = cv_score_agg_func or self._settings.get( + "cv_score_agg_func" + ) self._retrain_in_budget = retrain_full == "budget" and ( eval_method == "holdout" and self._state.X_val is None diff --git a/flaml/ml.py b/flaml/ml.py index 02226a10e..c5054ca5a 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -430,21 +430,27 @@ def get_val_loss( train_time = time.time() - start return val_loss, metric_for_logging, train_time, pred_time -def default_cv_score_agg_func(metrics_across_folds): - metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds) + +def default_cv_score_agg_func(val_loss_folds, log_metrics_folds): + metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds) metrics_to_log = None - for single_fold in metrics_across_folds: - if single_fold[1] is None: - break - elif metrics_to_log is None: - metrics_to_log = single_fold[1] + for single_fold in log_metrics_folds: + if metrics_to_log is None: + metrics_to_log = single_fold + elif isinstance(metrics_to_log, dict): + metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()} else: - metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()} + metrics_to_log += single_fold if metrics_to_log: - n = len(metrics_across_folds) - metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} + n = len(val_loss_folds) + metrics_to_log = ( + {k: v / n for k, v in metrics_to_log.items()} + if isinstance(metrics_to_log, dict) + else metrics_to_log / n + ) return metric_to_minimize, metrics_to_log + def evaluate_model_CV( config, estimator, @@ -455,7 +461,7 @@ def evaluate_model_CV( task, eval_metric, best_val_loss, - cv_score_agg_func = None, + cv_score_agg_func=None, log_training_metric=False, fit_kwargs={}, ): @@ -541,10 +547,7 @@ def evaluate_model_CV( pred_time += pred_time_i if time.time() - start_time >= budget: break - if log_training_metric or not isinstance(eval_metric, str): - val_loss, metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds))) - else: - val_loss, metric = cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds)))) + val_loss, metric = cv_score_agg_func(val_loss_folds, log_metric_folds) n = total_fold_num pred_time /= n return val_loss, metric, train_time, pred_time From 39d8c0e3e8140b50298883aa1dec7c34f0db27d0 Mon Sep 17 00:00:00 2001 From: zsk Date: Sun, 21 Aug 2022 08:35:44 -0400 Subject: [PATCH 07/20] Update flaml/automl.py Co-authored-by: Chi Wang --- flaml/automl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flaml/automl.py b/flaml/automl.py index 70f8ed4bf..0006b2be0 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -2376,7 +2376,7 @@ class AutoML(BaseEstimator): def cv_score_agg_func(val_loss_folds, log_metrics_folds): return metric_to_minimize, metrics_to_log ``` - “val_loss_folds” - list of float, it records the loss scores of each ford; “log_metrics_folds” - list of dict/float, it records the metrics of each fords to log. + “val_loss_folds” - list of floats, the loss scores of each fold; “log_metrics_folds” - list of dicts/floats, the metrics of each fold to log. This function should return the final aggregate result of all folds. A float number of the minimization objective, and a dictionary as the metrics to log or None. E.g., From 92dc41dc6939402d25bada30d5e4c15ab3e88fbc Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sun, 21 Aug 2022 12:40:54 +0000 Subject: [PATCH 08/20] update doc --- flaml/automl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flaml/automl.py b/flaml/automl.py index 70f8ed4bf..0006b2be0 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -2376,7 +2376,7 @@ class AutoML(BaseEstimator): def cv_score_agg_func(val_loss_folds, log_metrics_folds): return metric_to_minimize, metrics_to_log ``` - “val_loss_folds” - list of float, it records the loss scores of each ford; “log_metrics_folds” - list of dict/float, it records the metrics of each fords to log. + “val_loss_folds” - list of floats, the loss scores of each fold; “log_metrics_folds” - list of dicts/floats, the metrics of each fold to log. This function should return the final aggregate result of all folds. A float number of the minimization objective, and a dictionary as the metrics to log or None. E.g., From f9c764c097e1420b9f789953e0c605a95f2c1caf Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sun, 21 Aug 2022 12:43:43 +0000 Subject: [PATCH 09/20] update --- flaml/automl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flaml/automl.py b/flaml/automl.py index 0006b2be0..2a0c20059 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -2393,7 +2393,7 @@ class AutoML(BaseEstimator): metrics_to_log += single_fold if metrics_to_log: n = len(val_loss_folds) - metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log/n + metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log / n return metric_to_minimize, metrics_to_log ``` From 5d2929cb83d5057cd912c1c7c1f562cd46ae9679 Mon Sep 17 00:00:00 2001 From: zsk Date: Sun, 21 Aug 2022 08:47:20 -0400 Subject: [PATCH 10/20] Update flaml/automl.py Co-authored-by: Chi Wang --- flaml/automl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flaml/automl.py b/flaml/automl.py index 0006b2be0..2a0c20059 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -2393,7 +2393,7 @@ class AutoML(BaseEstimator): metrics_to_log += single_fold if metrics_to_log: n = len(val_loss_folds) - metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log/n + metrics_to_log = {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log / n return metric_to_minimize, metrics_to_log ``` From e3aa7ea9d13082a615169bf434043c576d193bbf Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sun, 21 Aug 2022 12:52:27 +0000 Subject: [PATCH 11/20] update --- flaml/ml.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flaml/ml.py b/flaml/ml.py index c5054ca5a..0dc597e17 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -541,7 +541,6 @@ def evaluate_model_CV( total_fold_num += 1 val_loss_folds.append(val_loss_i) if log_training_metric or not isinstance(eval_metric, str): - if isinstance(metric_i, dict): log_metric_folds.append(metric_i) train_time += train_time_i pred_time += pred_time_i From 462c27f8aec40e1ca3034879886d8a9a7314cc80 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sun, 21 Aug 2022 12:54:58 +0000 Subject: [PATCH 12/20] fix --- flaml/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flaml/ml.py b/flaml/ml.py index 0dc597e17..044424801 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -541,7 +541,7 @@ def evaluate_model_CV( total_fold_num += 1 val_loss_folds.append(val_loss_i) if log_training_metric or not isinstance(eval_metric, str): - log_metric_folds.append(metric_i) + log_metric_folds.append(metric_i) train_time += train_time_i pred_time += pred_time_i if time.time() - start_time >= budget: From 50fb20ebbcc2726d5868f5441e86f01f5bd4bbb6 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sun, 21 Aug 2022 17:44:26 +0000 Subject: [PATCH 13/20] update --- flaml/ml.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flaml/ml.py b/flaml/ml.py index 044424801..1d50512fb 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -540,8 +540,7 @@ def evaluate_model_CV( fit_kwargs["sample_weight"] = weight total_fold_num += 1 val_loss_folds.append(val_loss_i) - if log_training_metric or not isinstance(eval_metric, str): - log_metric_folds.append(metric_i) + log_metric_folds.append(metric_i) train_time += train_time_i pred_time += pred_time_i if time.time() - start_time >= budget: From 066842cfcb667c830f810d1f74ab028bfe68f36c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Aug 2022 22:45:46 +0000 Subject: [PATCH 14/20] Bump actions/cache from 1 to 3 Bumps [actions/cache](https://github.com/actions/cache) from 1 to 3. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v1...v3) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/CD.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index f68d2989c..3f48366f9 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -20,7 +20,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Cache conda - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/conda_pkgs_dir key: conda-${{ matrix.os }}-python-${{ matrix.python-version }}-${{ hashFiles('environment.yml') }} From 07656e4af0ed763cc703981518fe5fa03f0f8fca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Aug 2022 22:45:54 +0000 Subject: [PATCH 15/20] Bump actions/setup-node from 2 to 3 Bumps [actions/setup-node](https://github.com/actions/setup-node) from 2 to 3. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy-website.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml index 4130b16e4..62eb2591d 100644 --- a/.github/workflows/deploy-website.yml +++ b/.github/workflows/deploy-website.yml @@ -16,7 +16,7 @@ jobs: working-directory: website steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: node-version: 14.x # cache: yarn @@ -52,7 +52,7 @@ jobs: working-directory: website steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: node-version: 14.x # cache: yarn From f3bfc68412f1fa69bb7dd5db1f0fe8a7f7498880 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Mon, 22 Aug 2022 00:02:10 -0400 Subject: [PATCH 16/20] talks and tutorials (#694) --- README.md | 2 +- website/docs/Getting-Started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 421ef87fc..ca68ad073 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ You can find a detailed documentation about FLAML [here](https://microsoft.githu In addition, you can find: -- Demo and tutorials of FLAML [here](https://www.youtube.com/channel/UCfU0zfFXHXdAd5x-WvFBk5A). +- [Talks](https://www.youtube.com/channel/UCfU0zfFXHXdAd5x-WvFBk5A) and [tutorials](https://github.com/microsoft/FLAML/tree/tutorial/tutorial) about FLAML. - Research around FLAML [here](https://microsoft.github.io/FLAML/docs/Research). diff --git a/website/docs/Getting-Started.md b/website/docs/Getting-Started.md index 26e866bfc..f7373931e 100644 --- a/website/docs/Getting-Started.md +++ b/website/docs/Getting-Started.md @@ -90,7 +90,7 @@ Then, you can use it just like you use the original `LGMBClassifier`. Your other * Understand the use cases for [Task-oriented AutoML](Use-Cases/task-oriented-automl), [Tune user-defined function](Use-Cases/Tune-User-Defined-Function) and [Zero-shot AutoML](Use-Cases/Zero-Shot-AutoML). * Find code examples under "Examples": from [AutoML - Classification](Examples/AutoML-Classification) to [Tune - PyTorch](Examples/Tune-PyTorch). -* Watch [video tutorials](https://www.youtube.com/channel/UCfU0zfFXHXdAd5x-WvFBk5A). +* Find [talks](https://www.youtube.com/channel/UCfU0zfFXHXdAd5x-WvFBk5A) and [tutorials](https://github.com/microsoft/FLAML/tree/tutorial/tutorial) about FLAML. * Learn about [research](Research) around FLAML. * Refer to [SDK](reference/automl) and [FAQ](FAQ). From 2ead2966765b8aa17ae262b13eb41b81d09fd502 Mon Sep 17 00:00:00 2001 From: Xueqing Liu Date: Mon, 22 Aug 2022 07:20:48 -0400 Subject: [PATCH 17/20] updating nlp notebook (#693) * updating nlp notebook --- .github/dependabot.yml | 6 - notebook/automl_nlp.ipynb | 9449 +++++++++++++++++++++++++++++-------- 2 files changed, 7510 insertions(+), 1945 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 123014908..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" diff --git a/notebook/automl_nlp.ipynb b/notebook/automl_nlp.ipynb index 4f59a0a3e..b6c83cb19 100644 --- a/notebook/automl_nlp.ipynb +++ b/notebook/automl_nlp.ipynb @@ -1,1984 +1,7555 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "Copyright (c) Microsoft Corporation. All rights reserved. \n", - "\n", - "Licensed under the MIT License.\n", - "\n", - "# FineTuning NLP Models with FLAML Library\n", - "\n", - "\n", - "## 1. Introduction\n", - "\n", - "FLAML is a Python library (https://github.com/microsoft/FLAML) designed to automatically produce accurate machine learning models \n", - "with low computational cost. It is fast and economical. The simple and lightweight design makes it easy to use and extend, such as adding new learners. FLAML can \n", - "- serve as an economical AutoML engine,\n", - "- be used as a fast hyperparameter tuning tool, or \n", - "- be embedded in self-tuning software that requires low latency & resource in repetitive\n", - " tuning tasks.\n", - "\n", - "In this notebook, we demonstrate how to use the FLAML library to fine tune an NLP language model with hyperparameter search. We have tested this notebook on a server with 4 NVidia V100 GPU (32GB) and 400GB CPU Ram.\n", - "\n", - "FLAML requires `Python>=3.7`. To run this notebook example, please install flaml with the `nlp,ray,notebook` and `blendsearch` option:\n", - "```bash\n", - "pip install flaml[nlp,ray,notebook,blendsearch];\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ + "cells": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33mWARNING: Ignoring invalid distribution -andas (/home/xliu127/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -andas (/home/xliu127/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", - "\u001b[0mRequirement already satisfied: flaml[blendsearch,nlp,notebook,ray] in /data/xliu127/projects/hyperopt/FLAML (1.0.7)\n", - "Requirement already satisfied: NumPy>=1.17.0rc1 in /home/xliu127/.local/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.21.2)\n", - "Requirement already satisfied: lightgbm>=2.3.1 in /home/xliu127/.local/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.2.1)\n", - "Requirement already satisfied: xgboost<=1.3.3,>=0.90 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.3.3)\n", - "Requirement already satisfied: scipy>=1.4.1 in /home/xliu127/.local/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.6.2)\n", - "Requirement already satisfied: pandas>=1.1.4 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.4.3)\n", - "Requirement already satisfied: scikit-learn>=0.24 in /home/xliu127/.local/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (0.24.1)\n", - "Requirement already satisfied: openml==0.10.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (0.10.2)\n", - "Requirement already satisfied: jupyter in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.0.0)\n", - "Requirement already satisfied: matplotlib in /home/xliu127/.local/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.4.3)\n", - "Requirement already satisfied: rgf-python in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.12.0)\n", - "Requirement already satisfied: catboost>=0.26 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.0.6)\n", - "Requirement already satisfied: ray[tune]~=1.10 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.13.0)\n", - "Requirement already satisfied: protobuf<4 in /home/xliu127/.local/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.15.8)\n", - "Requirement already satisfied: transformers>=4.14 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (4.18.0)\n", - "Requirement already satisfied: datasets in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (2.4.0)\n", - "Requirement already satisfied: torch in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.12.0)\n", - "Requirement already satisfied: seqeval in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.2.2)\n", - "Requirement already satisfied: nltk in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.7)\n", - "Requirement already satisfied: rouge_score in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (0.1.2)\n", - "Requirement already satisfied: optuna==2.8.0 in /home/xliu127/.local/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (2.8.0)\n", - "Requirement already satisfied: liac-arff>=2.4.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2.5.0)\n", - "Requirement already satisfied: xmltodict in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (0.13.0)\n", - "Requirement already satisfied: python-dateutil in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2.8.2)\n", - "Requirement already satisfied: requests in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2.28.1)\n", - "Requirement already satisfied: packaging>=20.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (21.3)\n", - "Requirement already satisfied: sqlalchemy>=1.1.0 in /home/xliu127/.local/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.4.11)\n", - "Requirement already satisfied: tqdm in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (4.64.0)\n", - "Requirement already satisfied: colorlog in /home/xliu127/.local/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (5.0.1)\n", - "Requirement already satisfied: cmaes>=0.8.2 in /home/xliu127/.local/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (0.8.2)\n", - "Requirement already satisfied: alembic in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.8.1)\n", - "Requirement already satisfied: cliff in /home/xliu127/.local/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (3.7.0)\n", - "Requirement already satisfied: plotly in /home/xliu127/.local/lib/python3.8/site-packages (from catboost>=0.26->flaml[blendsearch,nlp,notebook,ray]) (4.14.3)\n", - "Requirement already satisfied: six in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from catboost>=0.26->flaml[blendsearch,nlp,notebook,ray]) (1.16.0)\n", - "Requirement already satisfied: graphviz in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from catboost>=0.26->flaml[blendsearch,nlp,notebook,ray]) (0.20.1)\n", - "Requirement already satisfied: wheel in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from lightgbm>=2.3.1->flaml[blendsearch,nlp,notebook,ray]) (0.37.1)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/xliu127/.local/lib/python3.8/site-packages (from pandas>=1.1.4->flaml[blendsearch,nlp,notebook,ray]) (2021.1)\n", - "Requirement already satisfied: attrs in /home/xliu127/.local/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (20.3.0)\n", - "Requirement already satisfied: frozenlist in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (1.3.0)\n", - "Requirement already satisfied: virtualenv in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (20.16.2)\n", - "Requirement already satisfied: click<=8.0.4,>=7.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (8.0.4)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /home/xliu127/.local/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (1.0.2)\n", - "Requirement already satisfied: filelock in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (3.7.1)\n", - "Requirement already satisfied: grpcio<=1.43.0,>=1.28.1 in /home/xliu127/.local/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (1.40.0)\n", - "Requirement already satisfied: pyyaml in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (6.0)\n", - "Requirement already satisfied: aiosignal in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (1.2.0)\n", - "Requirement already satisfied: jsonschema in /home/xliu127/.local/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (3.2.0)\n", - "Requirement already satisfied: tensorboardX>=1.9 in /home/xliu127/.local/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (2.2)\n", - "Requirement already satisfied: tabulate in /home/xliu127/.local/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (0.8.9)\n" - ] + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/xliu127/.local/lib/python3.8/site-packages (from scikit-learn>=0.24->flaml[blendsearch,nlp,notebook,ray]) (2.1.0)\n", - "Requirement already satisfied: joblib>=0.11 in /home/xliu127/.local/lib/python3.8/site-packages (from scikit-learn>=0.24->flaml[blendsearch,nlp,notebook,ray]) (1.0.1)\n", - "Requirement already satisfied: sacremoses in /home/xliu127/.local/lib/python3.8/site-packages (from transformers>=4.14->flaml[blendsearch,nlp,notebook,ray]) (0.0.45)\n", - "Requirement already satisfied: regex!=2019.12.17 in /home/xliu127/.local/lib/python3.8/site-packages (from transformers>=4.14->flaml[blendsearch,nlp,notebook,ray]) (2021.8.28)\n", - "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from transformers>=4.14->flaml[blendsearch,nlp,notebook,ray]) (0.12.1)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from transformers>=4.14->flaml[blendsearch,nlp,notebook,ray]) (0.8.1)\n", - "Requirement already satisfied: xxhash in /home/xliu127/.local/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (2.0.2)\n", - "Requirement already satisfied: fsspec[http]>=2021.11.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (2022.7.1)\n", - "Requirement already satisfied: multiprocess in /home/xliu127/.local/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (0.70.12.2)\n", - "Requirement already satisfied: dill<0.3.6 in /home/xliu127/.local/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (0.3.4)\n", - "Requirement already satisfied: aiohttp in /home/xliu127/.local/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (3.7.4.post0)\n", - "Requirement already satisfied: responses<0.19 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (0.18.0)\n", - "Requirement already satisfied: pyarrow>=6.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (8.0.0)\n", - "Requirement already satisfied: ipywidgets in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (7.7.1)\n", - "Requirement already satisfied: notebook in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.4.12)\n", - "Requirement already satisfied: qtconsole in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.3.1)\n", - "Requirement already satisfied: nbconvert in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.5.1)\n", - "Requirement already satisfied: jupyter-console in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.4.4)\n", - "Requirement already satisfied: ipykernel in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.15.1)\n", - "Requirement already satisfied: pyparsing>=2.2.1 in /home/xliu127/.local/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (2.4.7)\n", - "Requirement already satisfied: kiwisolver>=1.0.1 in /home/xliu127/.local/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (1.3.1)\n", - "Requirement already satisfied: cycler>=0.10 in /home/xliu127/.local/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (0.10.0)\n", - "Requirement already satisfied: pillow>=6.2.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (9.2.0)\n", - "Requirement already satisfied: absl-py in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from rouge_score->flaml[blendsearch,nlp,notebook,ray]) (1.2.0)\n", - "Requirement already satisfied: typing-extensions in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from torch->flaml[blendsearch,nlp,notebook,ray]) (4.3.0)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from requests->openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2.1.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from requests->openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2022.6.15)\n", - "Requirement already satisfied: idna<4,>=2.5 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from requests->openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (3.3)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from requests->openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (1.26.11)\n", - "Requirement already satisfied: greenlet!=0.4.17 in /home/xliu127/.local/lib/python3.8/site-packages (from sqlalchemy>=1.1.0->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.0.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /home/xliu127/.local/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (5.1.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /home/xliu127/.local/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (1.6.3)\n", - "Requirement already satisfied: chardet<5.0,>=2.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (4.0.0)\n", - "Requirement already satisfied: async-timeout<4.0,>=3.0 in /home/xliu127/.local/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (3.0.1)\n", - "Requirement already satisfied: importlib-metadata in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from alembic->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (4.12.0)\n", - "Requirement already satisfied: importlib-resources in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from alembic->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (5.9.0)\n", - "Requirement already satisfied: Mako in /home/xliu127/.local/lib/python3.8/site-packages (from alembic->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.1.4)\n", - "Requirement already satisfied: cmd2>=1.0.0 in /home/xliu127/.local/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.5.0)\n", - "Requirement already satisfied: PrettyTable>=0.7.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (3.3.0)\n", - "Requirement already satisfied: stevedore>=2.0.1 in /home/xliu127/.local/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (3.3.0)\n", - "Requirement already satisfied: pbr!=2.1.0,>=2.0.0 in /home/xliu127/.local/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (5.5.1)\n", - "Requirement already satisfied: debugpy>=1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.6.2)\n", - "Requirement already satisfied: traitlets>=5.1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.3.0)\n", - "Requirement already satisfied: matplotlib-inline>=0.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.1.3)\n", - "Requirement already satisfied: psutil in /home/xliu127/.local/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.8.0)\n", - "Requirement already satisfied: ipython>=7.23.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (8.4.0)\n", - "Requirement already satisfied: jupyter-client>=6.1.12 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (7.3.4)\n", - "Requirement already satisfied: tornado>=6.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.2)\n", - "Requirement already satisfied: nest-asyncio in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.5.5)\n", - "Requirement already satisfied: pyzmq>=17 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (23.2.0)\n", - "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipywidgets->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.1.1)\n", - "Requirement already satisfied: ipython-genutils~=0.2.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipywidgets->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.2.0)\n", - "Requirement already satisfied: widgetsnbextension~=3.6.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipywidgets->jupyter->flaml[blendsearch,nlp,notebook,ray]) (3.6.1)\n", - "Requirement already satisfied: pyrsistent>=0.14.0 in /home/xliu127/.local/lib/python3.8/site-packages (from jsonschema->ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (0.17.3)\n", - "Requirement already satisfied: setuptools in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jsonschema->ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (61.2.0)\n", - "Requirement already satisfied: pygments in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter-console->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.12.0)\n", - "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter-console->jupyter->flaml[blendsearch,nlp,notebook,ray]) (3.0.30)\n" - ] + "cell_type": "markdown", + "metadata": { + "id": "43f7-wG-Tjg_" + }, + "source": [ + "# FineTuning NLP Models with FLAML Library\n", + "\n", + "\n", + "## 1. Introduction\n", + "\n", + "FLAML is a Python library (https://github.com/microsoft/FLAML) designed to automatically produce accurate machine learning models \n", + "with low computational cost. It is fast and economical. The simple and lightweight design makes it easy to use and extend, such as adding new learners. FLAML can \n", + "- serve as an economical AutoML engine,\n", + "- be used as a fast hyperparameter tuning tool, or \n", + "- be embedded in self-tuning software that requires low latency & resource in repetitive\n", + " tuning tasks.\n", + "\n", + "In this notebook, we demonstrate how to use the FLAML library to fine tune an NLP language model with hyperparameter search. We will use [flaml.tune](https://microsoft.github.io/FLAML/docs/Use-Cases/Tune-User-Defined-Function) with the built in GPU in colab for the tuning. However, if you have a machine with more than 1 GPU, you can also use FLAML's [parallel tuning](https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#parallel-tuning) with the ray tune option. \n", + "\n", + "FLAML requires `Python>=3.7`. To run this notebook example, please install flaml with the `nlp,notebook` and `blendsearch` option:\n", + "```bash\n", + "pip install flaml[nlp,notebook,blendsearch]==1.0.11; \n", + "```" + ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: defusedxml in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.7.1)\n", - "Requirement already satisfied: pandocfilters>=1.4.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.5.0)\n", - "Requirement already satisfied: beautifulsoup4 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (4.11.1)\n", - "Requirement already satisfied: jupyter-core>=4.7 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (4.11.1)\n", - "Requirement already satisfied: lxml in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (4.9.1)\n", - "Requirement already satisfied: jupyterlab-pygments in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.2.2)\n", - "Requirement already satisfied: bleach in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.0.1)\n", - "Requirement already satisfied: tinycss2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.1.1)\n", - "Requirement already satisfied: mistune<2,>=0.8.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.8.4)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /home/xliu127/.local/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.0.1)\n", - "Requirement already satisfied: jinja2>=3.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (3.1.2)\n", - "Requirement already satisfied: nbclient>=0.5.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.6.6)\n", - "Requirement already satisfied: nbformat>=5.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.4.0)\n", - "Requirement already satisfied: entrypoints>=0.2.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.4)\n", - "Requirement already satisfied: prometheus-client in /home/xliu127/.local/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.10.1)\n", - "Requirement already satisfied: argon2-cffi in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (21.3.0)\n", - "Requirement already satisfied: Send2Trash>=1.8.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.8.0)\n", - "Requirement already satisfied: terminado>=0.8.3 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.15.0)\n", - "Requirement already satisfied: retrying>=1.3.3 in /home/xliu127/.local/lib/python3.8/site-packages (from plotly->catboost>=0.26->flaml[blendsearch,nlp,notebook,ray]) (1.3.3)\n", - "Requirement already satisfied: qtpy>=2.0.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from qtconsole->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.1.0)\n", - "Requirement already satisfied: platformdirs<3,>=2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from virtualenv->ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (2.5.2)\n", - "Requirement already satisfied: distlib<1,>=0.3.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from virtualenv->ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (0.3.5)\n", - "Requirement already satisfied: wcwidth>=0.1.7 in /home/xliu127/.local/lib/python3.8/site-packages (from cmd2>=1.0.0->cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (0.2.5)\n", - "Requirement already satisfied: colorama>=0.3.7 in /home/xliu127/.local/lib/python3.8/site-packages (from cmd2>=1.0.0->cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (0.4.4)\n", - "Requirement already satisfied: pyperclip>=1.6 in /home/xliu127/.local/lib/python3.8/site-packages (from cmd2>=1.0.0->cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.8.2)\n", - "Requirement already satisfied: stack-data in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.3.0)\n", - "Requirement already satisfied: pexpect>4.3 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (4.8.0)\n", - "Requirement already satisfied: backcall in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.2.0)\n", - "Requirement already satisfied: pickleshare in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.7.5)\n", - "Requirement already satisfied: decorator in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.18.1)\n", - "Requirement already satisfied: fastjsonschema in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbformat>=5.1->nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.16.1)\n", - "Requirement already satisfied: ptyprocess in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from terminado>=0.8.3->notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.7.0)\n", - "Requirement already satisfied: argon2-cffi-bindings in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from argon2-cffi->notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (21.2.0)\n", - "Requirement already satisfied: soupsieve>1.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from beautifulsoup4->nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.3.2.post1)\n", - "Requirement already satisfied: webencodings in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from bleach->nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.5.1)\n", - "Requirement already satisfied: zipp>=0.5 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from importlib-metadata->alembic->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (3.8.1)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.8.3)\n", - "Requirement already satisfied: cffi>=1.0.1 in /home/xliu127/.local/lib/python3.8/site-packages (from argon2-cffi-bindings->argon2-cffi->notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.14.6)\n", - "Requirement already satisfied: executing in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.9.1)\n", - "Requirement already satisfied: asttokens in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.0.7)\n", - "Requirement already satisfied: pure-eval in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.2.2)\n", - "Requirement already satisfied: pycparser in /home/xliu127/.local/lib/python3.8/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.20)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33mWARNING: Ignoring invalid distribution -andas (/home/xliu127/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -andas (/home/xliu127/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -andas (/home/xliu127/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -andas (/home/xliu127/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", - "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "%pip install flaml[nlp,ray,notebook,blendsearch]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's run some examples. \n", - "\n", - "Note: throughout this notebook, you may see a few ModuleNotFoundErrors. As long as the cell successfully executes, you can ignore that error." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Sentiment Classification Example\n", - "### Load data and preprocess\n", - "\n", - "The Stanford Sentiment treebank (SST-2) dataset is a dataset for sentiment classification. First, let's load this dataset into pandas dataframes:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n", - "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n", - "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n" - ] - } - ], - "source": [ - "from datasets import load_dataset\n", - "\n", - "train_dataset = load_dataset(\"glue\", \"sst2\", split=\"train\").to_pandas()\n", - "dev_dataset = load_dataset(\"glue\", \"sst2\", split=\"validation\").to_pandas()\n", - "test_dataset = load_dataset(\"glue\", \"sst2\", split=\"test\").to_pandas()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Take a look at the first 5 examples of this dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
sentencelabelidx
0hide new secretions from the parental units00
1contains no wit , only labored gags01
2that loves its characters and communicates som...12
3remains utterly satisfied to remain the same t...03
4on the worst revenge-of-the-nerds clichés the ...04
\n", - "
" + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Q8c3VMy6TjhC", + "outputId": "0eaa0dd7-e163-46c6-a637-a982ca62fff2" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", + "Requirement already satisfied: flaml[blendsearch,nlp,notebook] in /usr/local/lib/python3.7/dist-packages (1.0.11)\n", + "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from plotly->catboost>=0.26->flaml[blendsearch,nlp,notebook]) (8.0.1)\n", + "Requirement already satisfied: qtpy>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from qtconsole->jupyter->flaml[blendsearch,nlp,notebook]) (2.2.0)\n", + "Requirement already satisfied: absl-py in /usr/local/lib/python3.7/dist-packages (from rouge-score->flaml[blendsearch,nlp,notebook]) (1.2.0)\n" + ] + } ], - "text/plain": [ - " sentence label idx\n", - "0 hide new secretions from the parental units 0 0\n", - "1 contains no wit , only labored gags 0 1\n", - "2 that loves its characters and communicates som... 1 2\n", - "3 remains utterly satisfied to remain the same t... 0 3\n", - "4 on the worst revenge-of-the-nerds clichés the ... 0 4" + "source": [ + "%pip install flaml[nlp,notebook,blendsearch]==1.0.11;" ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "train_dataset.head(5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Separate the data into X and y:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "custom_sent_keys = [\"sentence\"] # specify the column names of the input sentences\n", - "label_key = \"label\" # specify the column name of the label\n", - "\n", - "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n", - "X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]\n", - "X_test = test_dataset[custom_sent_keys]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Run FLAML" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ + }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/xgboost/compat.py:31: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n", - " from pandas import MultiIndex, Int64Index\n", - "2022-08-13 16:52:12,092\tINFO services.py:1470 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n" - ] - } - ], - "source": [ - "''' import AutoML class from flaml package '''\n", - "from flaml import AutoML\n", - "automl = AutoML()\n", - "\n", - "import ray\n", - "if not ray.is_initialized():\n", - " ray.init() " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "TIME_BUDGET=1800\n", - "automl_settings = {\n", - " \"time_budget\": TIME_BUDGET, # setting the time budget\n", - " \"task\": \"seq-classification\", # setting the task as seq-classification\n", - " \"fit_kwargs_by_estimator\": {\n", - " \"transformer\": {\n", - " \"output_dir\": \"data/output/\", # setting the output directory\n", - " \"model_path\": \"google/electra-small-discriminator\", # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base\n", - " }\n", - " },\n", - " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n", - " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n", - " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n", - " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n", - " \"n_concurrent_trials\": 4,\n", - " \"keep_search_state\": True, # keeping the search state\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "id": "efPlAWTdTjhD" + }, + "source": [ + "Let's run some examples. To use CoLab's built in GPU, you need to select Runtime -> Change runtime type and select GPU. Then you can print the device information using:" + ] + }, { - "data": { - "text/html": [ - "== Status ==
Current time: 2022-08-13 17:22:28 (running for 00:30:10.04)
Memory usage on this node: 25.3/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/249.66 GiB heap, 0.0/110.99 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: ac41a40a with val_loss=0.07798165137614677 and parameters={'learning_rate': 3.864623804361677e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 24, 'global_max_steps': 9223372036854775807, 'learner': 'transformer', 'FLAML_sample_size': 67349}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17
Number of trials: 49/1000000 (49 TERMINATED)

" + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2kx9QbI7uaU8", + "outputId": "7b6f5fc2-1406-4460-e3f3-22a23751e136" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[]\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "import torch\n", + "print([torch.cuda.device(i) for i in range(torch.cuda.device_count())])" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'loss': 0.1493, 'learning_rate': 9.269681649947959e-06, 'epoch': 2.26}\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m {'eval_loss': 0.3011920750141144, 'eval_automl_metric': 0.09633027522935778, 'eval_runtime': 6.9816, 'eval_samples_per_second': 124.899, 'eval_steps_per_second': 124.899, 'epoch': 2.0}\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m {'loss': 0.1551, 'learning_rate': 3.2693767566133266e-05, 'epoch': 1.9}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.2014, 'learning_rate': 1.2879756568772649e-05, 'epoch': 1.43}\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m {'loss': 0.1524, 'learning_rate': 7.4163905463114075e-06, 'epoch': 2.14}\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m {'eval_loss': 0.3561520278453827, 'eval_automl_metric': 0.11238532110091748, 'eval_runtime': 7.1398, 'eval_samples_per_second': 122.133, 'eval_steps_per_second': 122.133, 'epoch': 2.0}\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'loss': 0.1492, 'learning_rate': 7.788901833662341e-06, 'epoch': 2.38}\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m {'loss': 0.1292, 'learning_rate': 2.5632478674959775e-05, 'epoch': 2.14}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.1936, 'learning_rate': 1.169158714360912e-05, 'epoch': 1.66}\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m {'loss': 0.155, 'learning_rate': 5.373307751184297e-06, 'epoch': 2.38}\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'loss': 0.1405, 'learning_rate': 6.308122017376726e-06, 'epoch': 2.49}\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m {'loss': 0.1078, 'learning_rate': 1.8571189783786284e-05, 'epoch': 2.38}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.1975, 'learning_rate': 1.0503417718445592e-05, 'epoch': 1.9}\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'loss': 0.1215, 'learning_rate': 4.827342201091109e-06, 'epoch': 2.61}\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m {'loss': 0.1437, 'learning_rate': 3.330224956057188e-06, 'epoch': 2.61}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.1698, 'learning_rate': 9.315248293282063e-06, 'epoch': 2.14}\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m {'loss': 0.1083, 'learning_rate': 1.1509900892612791e-05, 'epoch': 2.61}\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'loss': 0.1355, 'learning_rate': 3.346562384805493e-06, 'epoch': 2.73}\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m {'loss': 0.1428, 'learning_rate': 1.287142160930079e-06, 'epoch': 2.85}\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m {'loss': 0.106, 'learning_rate': 4.4486120014393e-06, 'epoch': 2.85}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.1661, 'learning_rate': 8.127078868118535e-06, 'epoch': 2.38}\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m {'eval_loss': 0.2953557074069977, 'eval_automl_metric': 0.0905963302752294, 'eval_runtime': 7.0221, 'eval_samples_per_second': 124.18, 'eval_steps_per_second': 124.18, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'loss': 0.1445, 'learning_rate': 1.8657825685198765e-06, 'epoch': 2.85}\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m {'train_runtime': 416.8987, 'train_samples_per_second': 484.643, 'train_steps_per_second': 15.148, 'train_loss': 0.2114269960332464, 'epoch': 3.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m Num examples = 872\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m Batch size = 1\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_80bc1972_47_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-17-39/checkpoint-6315/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_80bc1972_47_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-17-39/checkpoint-6315/vocab.txt\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_80bc1972_47_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-17-39/checkpoint-6315/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_80bc1972_47_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-17-39/checkpoint-6315/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=50106)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_80bc1972_47_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-17-39/checkpoint-6315/tokenizer_config.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m {'eval_loss': 0.3420582413673401, 'eval_automl_metric': 0.09633027522935778, 'eval_runtime': 8.4313, 'eval_samples_per_second': 103.424, 'eval_steps_per_second': 103.424, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m {'train_runtime': 404.0512, 'train_samples_per_second': 500.053, 'train_steps_per_second': 15.629, 'train_loss': 0.18374049291663574, 'epoch': 3.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m Num examples = 872\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m Batch size = 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.1614, 'learning_rate': 6.938909442955006e-06, 'epoch': 2.61}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_918542ec_48_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train__2022-08-13_17-18-07/checkpoint-6315/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_918542ec_48_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train__2022-08-13_17-18-07/checkpoint-6315/vocab.txt\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_918542ec_48_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train__2022-08-13_17-18-07/checkpoint-6315/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_918542ec_48_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train__2022-08-13_17-18-07/checkpoint-6315/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=50265)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_918542ec_48_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train__2022-08-13_17-18-07/checkpoint-6315/tokenizer_config.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'loss': 0.136, 'learning_rate': 3.8500275223426025e-07, 'epoch': 2.97}\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'eval_loss': 0.3623480498790741, 'eval_automl_metric': 0.08600917431192656, 'eval_runtime': 6.9268, 'eval_samples_per_second': 125.888, 'eval_steps_per_second': 125.888, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m {'train_runtime': 794.5838, 'train_samples_per_second': 254.28, 'train_steps_per_second': 15.895, 'train_loss': 0.20292378529045002, 'epoch': 3.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m Num examples = 872\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m Batch size = 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.1528, 'learning_rate': 5.750740017791478e-06, 'epoch': 2.85}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_bbde8ec8_45_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-12-09/checkpoint-12630/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_bbde8ec8_45_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-12-09/checkpoint-12630/vocab.txt\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_bbde8ec8_45_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-12-09/checkpoint-12630/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_bbde8ec8_45_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-12-09/checkpoint-12630/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=49559)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_bbde8ec8_45_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-12-09/checkpoint-12630/tokenizer_config.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'eval_loss': 0.29546257853507996, 'eval_automl_metric': 0.08830275229357798, 'eval_runtime': 7.0421, 'eval_samples_per_second': 123.827, 'eval_steps_per_second': 123.827, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.1462, 'learning_rate': 4.56257059262795e-06, 'epoch': 3.09}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.1474, 'learning_rate': 3.374401167464421e-06, 'epoch': 3.33}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.134, 'learning_rate': 2.1862317423008924e-06, 'epoch': 3.56}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'loss': 0.138, 'learning_rate': 9.98062317137364e-07, 'epoch': 3.8}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'eval_loss': 0.32222986221313477, 'eval_automl_metric': 0.09518348623853212, 'eval_runtime': 7.0713, 'eval_samples_per_second': 123.315, 'eval_steps_per_second': 123.315, 'epoch': 4.0}\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m {'train_runtime': 539.1601, 'train_samples_per_second': 499.659, 'train_steps_per_second': 15.617, 'train_loss': 0.2059766846428008, 'epoch': 4.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m Num examples = 872\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m Batch size = 1\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_b7e08512_49_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-19-11/checkpoint-6315/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_b7e08512_49_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-19-11/checkpoint-6315/vocab.txt\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_b7e08512_49_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-19-11/checkpoint-6315/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_b7e08512_49_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-19-11/checkpoint-6315/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=50443)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-13_16-52-17/train_b7e08512_49_FLAML_sample_size=67349,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-13_17-19-11/checkpoint-6315/tokenizer_config.json\n", - "2022-08-13 17:28:47,144\tINFO tune.py:747 -- Total run time: 2189.40 seconds (1803.73 seconds for the tuning loop).\n", - "[flaml.automl: 08-13 17:28:52] {3322} INFO - selected model: None\n", - "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 0.417, 'learning_rate': 3.55863617139559e-05, 'epoch': 0.24}\n", - "{'loss': 0.2872, 'learning_rate': 3.252648538429503e-05, 'epoch': 0.48}\n", - "{'loss': 0.2452, 'learning_rate': 2.9466609054634162e-05, 'epoch': 0.71}\n", - "{'loss': 0.2204, 'learning_rate': 2.6406732724973297e-05, 'epoch': 0.95}\n", - "{'loss': 0.1823, 'learning_rate': 2.334685639531243e-05, 'epoch': 1.19}\n", - "{'loss': 0.1743, 'learning_rate': 2.0286980065651558e-05, 'epoch': 1.43}\n", - "{'loss': 0.1662, 'learning_rate': 1.722710373599069e-05, 'epoch': 1.66}\n", - "{'loss': 0.1674, 'learning_rate': 1.416722740632982e-05, 'epoch': 1.9}\n", - "{'loss': 0.1422, 'learning_rate': 1.1107351076668952e-05, 'epoch': 2.14}\n", - "{'loss': 0.1271, 'learning_rate': 8.047474747008085e-06, 'epoch': 2.38}\n", - "{'loss': 0.123, 'learning_rate': 4.987598417347216e-06, 'epoch': 2.61}\n", - "{'loss': 0.1209, 'learning_rate': 1.927722087686347e-06, 'epoch': 2.85}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[flaml.automl: 08-13 17:36:12] {3465} INFO - retrain transformer for 440.2s\n", - "[flaml.automl: 08-13 17:36:12] {3472} INFO - retrained model: None\n", - "[flaml.automl: 08-13 17:36:12] {2749} INFO - fit succeeded\n", - "[flaml.automl: 08-13 17:36:12] {2750} INFO - Time taken to find the best model: 1610.794395685196\n", - "[flaml.automl: 08-13 17:36:12] {2761} WARNING - Time taken to find the best model is 89% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'train_runtime': 415.3339, 'train_samples_per_second': 486.469, 'train_steps_per_second': 15.205, 'train_loss': 0.194211708848097, 'epoch': 3.0}\n" - ] - } - ], - "source": [ - "'''The main flaml automl API'''\n", - "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The best loss by FLAML: 0.07798165137614677\n" - ] - } - ], - "source": [ - "print(\"The best loss by FLAML: {}\".format(automl.best_loss))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Best model and metric" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best hyperparmeter config: {'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 1878, 'learner': 'transformer'}\n", - "Best accuracy on validation data: 0.9404\n", - "Training duration of best run: 157.7 s\n" - ] - } - ], - "source": [ - "'''retrieve best config and best learner'''\n", - "print('Best hyperparmeter config:', automl.best_config)\n", - "print('Best accuracy on validation data: {0:.4g}'.format(1-automl.best_loss))\n", - "print('Training duration of best run: {0:.4g} s'.format(automl.best_config_train_time))" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 0.528, 'learning_rate': 8.898933352349567e-06, 'epoch': 1.0}\n", - "{'eval_loss': 0.2549280524253845, 'eval_automl_metric': 0.08600917431192656, 'eval_runtime': 1.0003, 'eval_samples_per_second': 871.751, 'eval_steps_per_second': 54.984, 'epoch': 1.0}\n", - "{'loss': 0.2278, 'learning_rate': 1.3880017803076292e-05, 'epoch': 2.0}\n", - "{'eval_loss': 0.24966619908809662, 'eval_automl_metric': 0.06766055045871555, 'eval_runtime': 1.0201, 'eval_samples_per_second': 854.778, 'eval_steps_per_second': 53.914, 'epoch': 2.0}\n", - "{'loss': 0.1455, 'learning_rate': 1.1410179501562432e-05, 'epoch': 3.0}\n", - "{'eval_loss': 0.23046882450580597, 'eval_automl_metric': 0.059633027522935755, 'eval_runtime': 1.0097, 'eval_samples_per_second': 863.6, 'eval_steps_per_second': 54.47, 'epoch': 3.0}\n", - "{'eval_loss': 0.23046882450580597, 'eval_automl_metric': 0.059633027522935755, 'eval_runtime': 0.9726, 'eval_samples_per_second': 896.568, 'eval_steps_per_second': 56.55, 'epoch': 3.0}\n", - "{'train_runtime': 146.7879, 'train_samples_per_second': 519.346, 'train_steps_per_second': 32.462, 'train_loss': 0.30043953600021217, 'epoch': 3.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using amp half precision backend\n", - "***** Running Prediction *****\n", - " Num examples = 872\n", - " Batch size = 64\n" - ] - } - ], - "source": [ - "import pickle\n", - "automl.pickle(\"automl.pkl\")\n", - "\n", - "with open(\"automl.pkl\", \"rb\") as f:\n", - " automl = pickle.load(f)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using amp half precision backend\n", - "***** Running Prediction *****\n", - " Num examples = 1821\n", - " Batch size = 4\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Predicted labels [0 0 1 ... 1 1 1]\n" - ] - } - ], - "source": [ - "'''compute predictions of testing dataset''' \n", - "y_pred = automl.predict(X_test, **{\"per_device_eval_batch_size\": 1})\n", - "print('Predicted labels', y_pred)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Log history" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.4211966209891772e-05, 'num_train_epochs': 0.22667066737595704, 'per_device_train_batch_size': 4, 'warmup_ratio': 0.0638407085008166, 'weight_decay': 0.24365576482793252, 'adam_epsilon': 1.2017005181798623e-08, 'seed': 44, 'global_max_steps': 567, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 6.105850155006292e-06, 'num_train_epochs': 0.5203392488419986, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.02982185620209511, 'weight_decay': 0.06967105125024514, 'adam_epsilon': 2.8808753274240916e-07, 'seed': 42, 'global_max_steps': 163, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.029571768192494e-05, 'num_train_epochs': 0.23081935882872334, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.05927331754619176, 'weight_decay': 0.06990187648448716, 'adam_epsilon': 2.6275569358808137e-08, 'seed': 41, 'global_max_steps': 145, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.00010208514406839324, 'num_train_epochs': 0.24916036398153807, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.28313267937361647, 'adam_epsilon': 1.5042842082223077e-08, 'seed': 44, 'global_max_steps': 78, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.00011786584823407102, 'num_train_epochs': 1.1449809097488275, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.015398649752204965, 'weight_decay': 0.3, 'adam_epsilon': 4.275263179285732e-08, 'seed': 43, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00011786584823407102, 'num_train_epochs': 1.1449809097488275, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.015398649752204965, 'weight_decay': 0.3, 'adam_epsilon': 4.275263179285732e-08, 'seed': 43, 'global_max_steps': 313, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 3.59933640287326e-05, 'num_train_epochs': 1.7937390219164033, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.016745580745240057, 'weight_decay': 0.2892279950527897, 'adam_epsilon': 6.514301011066509e-08, 'seed': 44, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 3.59933640287326e-05, 'num_train_epochs': 1.7937390219164033, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.016745580745240057, 'weight_decay': 0.2892279950527897, 'adam_epsilon': 6.514301011066509e-08, 'seed': 44, 'global_max_steps': 313, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 1878, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 1878, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.0001178658482340711, 'num_train_epochs': 1.1449809097488262, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.015398649752204969, 'weight_decay': 0.3, 'adam_epsilon': 4.275263179285729e-08, 'seed': 43, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 1878, 'learner': 'transformer'}}\n" - ] - } - ], - "source": [ - "from flaml.data import get_output_from_log\n", - "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", - " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n", - "for config in config_history:\n", - " print(config)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAiUElEQVR4nO3de7xVdZ3/8ddbQqHxAgg1CnKxkNRqxHZaWXkphfxVojkN9qtxbCamKf1VTs7gdHP050TZ5WePcXKwn2M2maFjRGWRhVrjJTmEglAYoilHSxApUwKBz/yxvhsW23X2WQfOvp3zfj4e+8Fe3/Vde332Pof9Oet7W4oIzMzMau3V6gDMzKw9OUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFXKCMNsNkt4gaVWr4zBrJCcI6ziSHpb05lbGEBE/jYgpjXp9SdMk/UTS05LWSbpd0tsbdT6zIk4QZgUkDWnhuc8EbgCuBcYBLwY+CbxtN15Lkvz/3HaLf3FswJC0l6TZkh6U9KSkeZJG5fbfIOk3kn6X/jo/MrfvGklflnSzpGeAE9OVykclLUvHfFPSsFT/BElrc8f3WDft/wdJj0t6TNLfSApJLy14DwK+AFwSEV+JiN9FxPaIuD0i3pfqXCTpP3PHTEyv94K0fZukSyXdATwLXCCpq+Y8H5G0ID3fR9LnJD0i6beSrpQ0fA9/HDYAOEHYQHIeMAM4HjgYeAq4Irf/+8Bk4EXAz4Gv1xz/LuBSYD/gv1PZO4HpwCTglcBf1Tl/YV1J04HzgTcDLwVOqPMaU4BDgBvr1CnjPcAssvdyJTBF0uTc/ncB16Xnc4DDgKNSfGPJrlhskHOCsIHk/cDHImJtRGwGLgLOrP5lHRFXR8TTuX1/JumA3PHfjog70l/sf0xlX4qIxyJiA/Adsi/RnvRU953Af0TEioh4Np27Jwemfx8v95Z7dE0639aI+B3wbeAsgJQoXgYsSFcss4CPRMSGiHga+Bdg5h6e3wYAJwgbSCYA35K0UdJG4BfANuDFkoZImpOan34PPJyOGZ07/tGC1/xN7vmzwL51zt9T3YNrXrvoPFVPpn8PqlOnjNpzXEdKEGRXD/NTshoDvBBYkvvcfpDKbZBzgrCB5FHgLRExIvcYFhHdZF+Kp5E18xwATEzHKHd8o5Y2fpyss7nqkDp1V5G9j3fUqfMM2Zd61Z8W1Kl9L7cAYyQdRZYoqs1L64FNwJG5z+yAiKiXCG2QcIKwTjVU0rDc4wVkbe2XSpoAIGmMpNNS/f2AzWR/ob+QrBmlWeYB50g6XNILgU/0VDGy9ffPBz4h6RxJ+6fO99dLmpuq3Qu8UdL41ER2YW8BRMRzZCOjLgNGkSUMImI7cBXwRUkvApA0VtK03X2zNnA4QVinupnsL9/q4yLgcmAB8ENJTwN3A8em+tcCvwa6gZVpX1NExPeBLwG3Aqtz597cQ/0bgb8A3gs8BvwW+L9k/QhExC3AN4FlwBLguyVDuY7sCuqGiNiaK//Halyp+e1HZJ3lNsjJNwwyay5JhwP3A/vUfFGbtRVfQZg1gaTT03yDkcBngO84OVi7c4Iwa46/BZ4AHiQbWfV3rQ3HrHduYjIzs0K+gjAzs0IvaHUA/WX06NExceLEVodhZtZRlixZsj4iCidGNjRBpDVoLgeGAF+JiDk1+ycAV5PN2twAvDsi1qZ924DlqeojEVF3qeOJEyfS1dVVr4qZmdWQ9Oue9jUsQaTlkq8ATgbWAoslLYiIlblqnwOujYivSjoJ+DTZImMAmyLiqEbFZ2Zm9TWyD+IYYHVErImILcD1ZEsd5B0BLErPby3Yb2ZmLdLIBDGWXRcMW5vK8u4DzkjPTwf2k1RdzXKYpC5Jd0uaUXQCSbNSna5169b1Y+hmZtbqUUwfBY6XtJRsDf9usjHiABMiokK2yNr/k/SS2oMjYm5EVCKiMmaMF580M+tPjeyk7mbXVSvHpbIdIuIx0hWEpH2Bd0TExrSvO/27RtJtwFSySUZmZtYEjUwQi4HJkiaRJYaZZFcDO0gaDWxIK0peSDaiibQcwbMRsTnVOQ74bANjNTPrGPOXdnPZwlU8tnETB48YzgXTpjBjam0L/p5rWBNTWmfmXGAh2Y1b5kXECkkXS6oOWT0BWCXpAbIbs1+ayg8HuiTdR9Z5Padm9JOZ2aA0f2k3F960nO6Nmwige+MmLrxpOfOXdvd6bF8NmKU2KpVKeB6EmQ10x81ZRPfGTc8rHztiOHfMPqnPrydpServfZ4BM5PazAauZjWpdILHCpJDvfI90epRTGZmdTWzSaUTHDxieJ/K94SvIMysrV22cBWbntu2S9mm57bxDzcu4xv3PNKiqFpn2NC92EuwPdc7MHzoEC6Y1v83AXSCMLO21lPTyZZt25scSXsYve8+ADy6YRNbtm1nbAOb3JwgzKytHTxieI+dst/829e2IKLBw30QZtbWLpg2heFDh+xS1qgmFduVryDMGsQjb/pH9TP7hxuXNbxJpdM0+nfMCcKsAaojb6qdq9WRN4C/2HbDjKljd3RIu1kp04zfMScIswbwyJv+t/Lx33PEQfu3Ooy20dPv2GULV/VbgnAfhFkDeORN/zvioP057ShffVU1Y8KcryDMGsAjb6zRevod688Jc76CMGsAj7yxRmvG75ivIMwawCNvrNGqv0sexWTWgTzyxhptxtSxDf2jw01MZmZWyFcQbc6TrcysVZwg2pgnW5lZKzlBtLFOmmy1/g+bd6wuufeQvThk1PAdq04OZp7cZZ3MCaKNdcpkq/V/2MxD65/ZsT79lm3beWj9MwCDPkl4cpd1MieINtYpk62Om7Nol5uXQHYzkz8+t72t4jSzvvEopjbWKZOtmnmPXDNrHieINjZj6lg+fcYr2HtI9mMaO2I4nz7jFW3XQd3Me+SaWfM0NEFImi5plaTVkmYX7J8g6ceSlkm6TdK43L6zJf0qPc5uZJztbMbUsUwdP4JjJ43ijtkntV1ygM650jGzvmlYH4SkIcAVwMnAWmCxpAURsTJX7XPAtRHxVUknAZ8G3iNpFPApoAIEsCQd+1R/x+l5BnuuGVP+zaz5GtlJfQywOiLWAEi6HjgNyCeII4Dz0/Nbgfnp+TTglojYkI69BZgOfKM/A/Q8g/7T6Cn/ZtZ8jUwQY4FHc9trgWNr6twHnAFcDpwO7CfpwB6Ofd63j6RZwCyA8ePH9znATpln4LH0ZtYKre6k/ihwvKSlwPFAN7Ct/iE7RcTciKhERGXMmDF9PnmnzDPwWHoza4VGXkF0A4fktselsh0i4jGyKwgk7Qu8IyI2SuoGTqg59rb+DrBT5hmYmbVCI68gFgOTJU2StDcwE1iQryBptKRqDBcCV6fnC4FTJI2UNBI4JZX1K4++MTPrWcMSRERsBc4l+2L/BTAvIlZIuljS21O1E4BVkh4AXgxcmo7dAFxClmQWAxdXO6z7U3WewdgRwxHtO8/AzKwVFBG91+oAlUolurq6Wh2GmVlHkbQkIipF+1rdSW1mZm3Ki/XV8MQ5M7OME0SOJ86Zme3kJqacnibOXbZwVYsiMjNrHSeIHC9bbWa2kxNEjpetNjPbyQkixxPn+s/8pd0cN2cRk2Z/j+PmLGL+0u7eDzKztuJO6hwvW90/3NlvNjA4QdTwstV7rl5nvz9bs87hJibrd+7sNxsYnCCs37mz32xgcIKwfufOfrOBwX0Q1u/c2W82MDhBWEO4s9+s87mJyczMCjlBmJlZIScIMzMr5ARhZmaFnCDMzKyQE4SZmRXqNUFIOrAZgZiZWXspcwVxt6QbJJ0qSQ2PyMzM2kKZBHEYMBd4D/ArSf8i6bAyLy5puqRVklZLml2wf7ykWyUtlbRM0qmpfKKkTZLuTY8r+/KmzMxsz/U6kzoiArgFuEXSicB/Ah+QdB8wOyLuKjpO0hDgCuBkYC2wWNKCiFiZq/ZxYF5EfFnSEcDNwMS078GIOGr33paZme2pXhNE6oN4N9kVxG+B84AFwFHADcCkHg49BlgdEWvS61wPnAbkE0QA+6fnBwCP9fkdmJlZQ5RpYrqL7Et8RkT8r4i4KSK2RkQXUK/pZyzwaG57bSrLuwh4t6S1ZFcP5+X2TUpNT7dLekPRCSTNktQlqWvdunUl3oqZmZVVZrG+KamZ6Xki4jN7eP6zgGsi4vOSXgt8TdLLgceB8RHxpKRXAfMlHRkRv685/1yy/hEqlUphjGZmtnvKXEH8UNKI6oakkZIWljiuGzgktz0uleX9NTAPIPVlDANGR8TmiHgylS8BHiTrLDczsyYpkyDGRMTG6kZEPAW8qMRxi4HJkiZJ2huYSdZ3kfcI8CYASYeTJYh1ksakTm4kHQpMBtaUOKeZmfWTMk1M2ySNj4hHACRNIOtcrisitko6F1gIDAGujogVki4GuiJiAfD3wFWSPpJe868iIiS9EbhY0nPAduD9EbFht96hmZntFvXQvbCzgjSdrJ3/dkDAG4BZEVGmmalpKpVKdHV1tToMM7OOImlJRFSK9pWZB/EDSUcDr0lFH46I9f0ZoJmZtZ+ytxzdBjxB1kdwhCQi4ieNC8vMzFqtzES5vwE+RDYK6V6yK4m7gJMaGpmZmbVUmVFMHwJeDfw6Ik4EpgIbGxmUmZm1XpkE8ceI+COApH0i4pfAlMaGZWZmrVamD2Jtmig3n2zBvqeAXzcyKDMza70yo5hOT08vknQr2aJ6P2hoVGZm1nJ1E0SazbwiIl4GEBG3NyUqMzNrubp9EBGxDVglaXyT4jEzszZRpg9iJLBC0j3AM9XCiHh7w6IyM7OWK5MgPtHwKMzMrO2U6aR2v4OZ2SBUZib10+xcvXVvYCjwTETs3/NRZmbW6cpcQexXfS5JZPeVfk3PR5iZ2UBQZib1DpGZD0xrTDhmZtYuyjQxnZHb3AuoAH9sWERmZtYWyoxielvu+VbgYbJmJjMzG8DK9EGc04xAzMysvfTaByHpq2mxvur2SElXNzQqMzNruTKd1K+MiI3VjYh4iuyeEGZmNoCVSRB7SRpZ3ZA0ivK3KjUzsw5VJkF8HrhL0iWSLgHuBD5b5sUlTZe0StJqSbML9o+XdKukpZKWSTo1t+/CdNwqSR5Wa2bWZGU6qa+V1MXOe1CfERErezsuLRV+BXAysBZYLGlBzbEfB+ZFxJclHQHcDExMz2cCRwIHAz+SdFhaXdbMzJqgTCf1a4BHI+JfI+Jfye4wd2yJ1z4GWB0RayJiC3A9zx8eG0B1yY4DgMfS89OA6yNic0Q8BKxOr2dmZk1Sponpy8Afctt/SGW9GQs8mttem8ryLgLeLWkt2dXDeX04FkmzJHVJ6lq3bl2JkMzMrKwyCUIRUV2sj4jYTv91Up8FXBMR44BTga9JKr38R0TMjYhKRFTGjBnTTyGZmRmUSxBrJP0fSUPT40PAmhLHdQOH5LbHpbK8vwbmAUTEXcAwYHTJY83MrIHKJIj3A68j+4JeCxwLvK/EcYuByZImSdqbrNN5QU2dR4A3AUg6nCxBrEv1ZkraR9IkYDJwT4lzmplZPykziukJsi93ACQNB94K3NDLcVslnQssBIYAV0fECkkXA10RsQD4e+AqSR8h67D+q9SctULSPGAl2fpPH/QIJjOz5lKue6HnStmQ1WlkfQYnA/8dEWc2OLY+qVQq0dXV1eowzMw6iqQlEVEp2lf3CkLS8cC7yDqQ7wGOAw6NiGf7PUozM2srPSaINPT0EbIhrR+NiKclPeTkYGY2ONTrpL6RbBbzXwBvk/Qn7Lw3tZmZDXA9JoiI+DAwiWwtphOAVcAYSe+UtG9TojMzs5apO8w13YP61oiYRZYsziJbBuPhJsRmZmYtVHpGdEQ8B3wX+G4a6mpmZgNY6WUt8iJiU38HYmZm7WW3EoSZmQ18ThBmZlao1z4ISYcBFwAT8vUj4qQeDzIzs45XppP6BuBK4CrA6yGZmQ0SZRLE1ogoc4MgMzMbQMr0QXxH0gckHSRpVPXR8MjMzKylylxBnJ3+vSBXFsCh/R+OmZm1izL3g5jUjEDMzKy9lBnFNBT4O+CNqeg24N/TzGozMxugyjQxfRkYCvxb2n5PKvubRgVlZmatVyZBvDoi/iy3vUjSfY0KyMzM2kOZUUzbJL2kuiHpUDwfwsxswCtzBXEBcKukNYDIZlSf09CozMys5cqMYvqxpMnAlFS0KiI2NzYsMzNrtXr3pD4pIhZJOqNm10slERE3NTg2MzNroXpXEMcDi4C3FewLoNcEIWk6cDkwBPhKRMyp2f9F4MS0+ULgRRExIu3bBixP+x6JiLf3dj4zM+s/PSaIiPhUenpxRDyU3yep18lzkoYAVwAnA2uBxZIWRMTK3Dk+kqt/HjA19xKbIuKoMm/CzMz6X5lRTP9VUHZjieOOAVZHxJqI2AJcT3Y/656cBXyjxOuamVkT1OuDeBlwJHBATT/E/sCwEq89Fng0t70WOLaHc00AJpE1aVUNk9QFbAXmRMT8guNmAbMAxo8fXyIkMzMrq14fxBTgrcAIdu2HeBp4Xz/HMRO4MSLy8ysmRER3mnexSNLyiHgwf1BEzAXmAlQqlejnmMzMBrV6fRDfBr4t6bURcdduvHY3cEhue1wqKzIT+GDN+bvTv2sk3UbWP/Hg8w81M7NGKDNRbqmkD5I1N+1oWoqI9/Zy3GJgcurQ7iZLAu+qrZSaskYCd+XKRgLPRsRmSaOB44DPlojVzMz6SZlO6q8BfwpMA24nuxJ4ureDImIrcC6wEPgFMC8iVki6WFJ+yOpM4PqIyDcRHQ50pTWfbiXrg1iJmZk1jXb9Xi6oIC2NiKmSlkXEK9Py3z+NiNc0J8RyKpVKdHV1tToMM7OOImlJRFSK9pW5gqje92GjpJcDBwAv6q/gzMysPZXpg5ib+gQ+ASwA9gU+2dCozMys5cos1veV9PR2fB9qM7NBo95EufPrHRgRX+j/cMzMrF3Uu4LYL/07BXg1WfMSZJPm7mlkUGZm1nr1Jsr9M4CknwBHR8TTafsi4HtNic7MzFqmzCimFwNbcttbUpmZmQ1gZUYxXQvcI+lbaXsGcE2jAjIzs/ZQZhTTpZK+D7whFZ0TEUsbG5aZmbVavVFM+0fE7yWNAh5Oj+q+URGxofHhmZlZq9S7griObLnvJWS3GK1S2vacCDOzAazeKKa3pn97vb2omZkNPPWamI6ud2BE/Lz/wzEzs3ZRr4np83X2BXBSP8diZmZtpF4T04nNDMTMzNpLmXkQpGW+j2DXO8pd26igzMys9XpNEJI+BZxAliBuBt4C/DfZBDozMxugyiy1cSbwJuA3EXEO8GdkNw0yM7MBrEyC2BQR24GtkvYHngAOaWxYZmbWamX6ILokjQCuIps09wfgrkYGZWZmrVdvHsQVwHUR8YFUdKWkHwD7R8SypkRnZmYtU+8K4gHgc5IOAuYB3/AifWZmg0ePfRARcXlEvBY4HngSuFrSLyV9StJhZV5c0nRJqyStljS7YP8XJd2bHg9I2pjbd7akX6XH2X1/a2ZmticUEb3XqlaWpgJXA6+MiCG91B1CdhVyMrAWWAycFREre6h/HjA1It6bVpDtAipks7aXAK+KiKd6Ol+lUomurq7S78XMzEDSkoioFO3rdRSTpBdIepukrwPfB1YBZ5Q47zHA6ohYExFbgOuB0+rUPwv4Rno+DbglIjakpHALML3EOc3MrJ/U66Q+mexL+1TgHrIv+FkR8UzJ1x4LPJrbXgsc28O5JgCTgEV1jh1bcNwsYBbA+PHjS4ZlZmZl1LuCuBC4Ezg8It4eEdf1ITn01UzgxojY1peDImJuRFQiojJmzJgGhWZmNjjVW6xvT1dr7WbXCXXjUlmRmcAHa449oebY2/YwHjMz64MyM6l312JgsqRJkvYmSwILaitJehkwkl0n3y0ETpE0UtJI4JRUZmZmTVJqNdfdERFbJZ1L9sU+BLg6IlZIuhjoiohqspgJXB+54VQRsUHSJWRJBuBi3wPbzKy5+jTMtZ15mKuZWd/t0TBXMzMbnJwgzMyskBOEmZkVcoIwM7NCThBmZlbICcLMzAo5QZiZWSEnCDMzK+QEYWZmhZwgzMyskBOEmZkVcoIwM7NCThBmZlbICcLMzAo5QZiZWSEnCDMzK+QEYWZmhZwgzMyskBOEmZkVcoIwM7NCThBmZlaooQlC0nRJqyStljS7hzrvlLRS0gpJ1+XKt0m6Nz0WNDJOMzN7vhc06oUlDQGuAE4G1gKLJS2IiJW5OpOBC4HjIuIpSS/KvcSmiDiqUfGZmVl9jbyCOAZYHRFrImILcD1wWk2d9wFXRMRTABHxRAPjMTOzPmhkghgLPJrbXpvK8g4DDpN0h6S7JU3P7RsmqSuVzyg6gaRZqU7XunXr+jV4M7PBrmFNTH04/2TgBGAc8BNJr4iIjcCEiOiWdCiwSNLyiHgwf3BEzAXmAlQqlWhq5GZmA1wjryC6gUNy2+NSWd5aYEFEPBcRDwEPkCUMIqI7/bsGuA2Y2sBYzcysRiMTxGJgsqRJkvYGZgK1o5Hmk109IGk0WZPTGkkjJe2TKz8OWImZmTVNw5qYImKrpHOBhcAQ4OqIWCHpYqArIhakfadIWglsAy6IiCclvQ74d0nbyZLYnPzoJzMzazxFDIym+0qlEl1dXa0Ow8yso0haEhGVon2eSW1mZoWcIMzMrJAThJmZFXKCMDOzQk4QZmZWyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFWr1/SBsD8xf2s1lC1fx2MZNHDxiOBdMm8KMqbX3ZDIz2z1OEB1q/tJuLrxpOZue2wZA98ZNXHjTcgAnCTPrF25i6lCXLVy1IzlUbXpuG5ctXNWiiMxsoHGC6FCPbdzUp3Izs75yguhQB48Y3qdyM7O+coLoUBdMm8LwoUN2KRs+dAgXTJvSoojMbKBxJ3WHqnZEexSTmTWKE0QHmzF1rBOCmTWMm5jMzKyQE4SZmRVygjAzs0JOEGZmVsgJwszMCikiWh1Dv5C0Dvh1iaqjgfUNDqeRHH9rOf7Wcvz9b0JEjCnaMWASRFmSuiKi0uo4dpfjby3H31qOv7ncxGRmZoWcIMzMrNBgTBBzWx3AHnL8reX4W8vxN9Gg64MwM7NyBuMVhJmZleAEYWZmhQZ8gpD0sKTlku6V1JXKRkm6RdKv0r8jWx1nlaSrJT0h6f5cWWG8ynxJ0mpJyyQd3brId8RaFP9FkrrTz+BeSafm9l2Y4l8laVprot4RyyGSbpW0UtIKSR9K5R3x+deJvyM+/xTPMEn3SLovvYd/TuWTJP0sxfpNSXun8n3S9uq0f2Kbxn+NpIdyP4OjUnlb/Q49T0QM6AfwMDC6puyzwOz0fDbwmVbHmYvtjcDRwP29xQucCnwfEPAa4GdtGv9FwEcL6h4B3AfsA0wCHgSGtDD2g4Cj0/P9gAdSjB3x+deJvyM+/xSTgH3T86HAz9JnOw+YmcqvBP4uPf8AcGV6PhP4ZpvGfw1wZkH9tvodqn0M+CuIHpwGfDU9/yowo3Wh7CoifgJsqCnuKd7TgGsjczcwQtJBTQm0Bz3E35PTgOsjYnNEPASsBo5pWHC9iIjHI+Ln6fnTwC+AsXTI518n/p601ecPkD7LP6TNoekRwEnAjam89mdQ/dncCLxJkpoT7fPVib8nbfU7VGswJIgAfihpiaRZqezFEfF4ev4b4MWtCa20nuIdCzyaq7eW+l8IrXRuuoS+Otek17bxp6aKqWR/AXbc518TP3TQ5y9piKR7gSeAW8iubDZGxNZUJR/njveQ9v8OOLCpAdeojT8iqj+DS9PP4IuS9kllbfkzqBoMCeL1EXE08Bbgg5LemN8Z2XVex4z17bR4ky8DLwGOAh4HPt/SaHohaV/gv4APR8Tv8/s64fMviL+jPv+I2BYRRwHjyK5oXtbaiPqmNn5JLwcuJHsfrwZGAf/YugjLG/AJIiK6079PAN8i+4X7bfUyLv37ROsiLKWneLuBQ3L1xqWythIRv03/abYDV7GzGaPt4pc0lOzL9esRcVMq7pjPvyj+Tvr88yJiI3Ar8FqyppfqLZLzce54D2n/AcCTzY20WC7+6an5LyJiM/AfdMjPYEAnCEl/Imm/6nPgFOB+YAFwdqp2NvDt1kRYWk/xLgD+Mo2EeA3wu1xTSNuoaVM9nexnAFn8M9NIlEnAZOCeZsdXldqu/z/wi4j4Qm5XR3z+PcXfKZ8/gKQxkkak58OBk8n6Um4FzkzVan8G1Z/NmcCidJXXEj3E/8vcHxgi6z/J/wza5nfoeVrdS97IB3Ao2SiN+4AVwMdS+YHAj4FfAT8CRrU61lzM3yBrBniOrD3yr3uKl2zkwxVkbbTLgUqbxv+1FN8ysv8QB+XqfyzFvwp4S4tjfz1Z89Ey4N70OLVTPv868XfE55/ieSWwNMV6P/DJVH4oWfJaDdwA7JPKh6Xt1Wn/oW0a/6L0M7gf+E92jnRqq9+h2oeX2jAzs0IDuonJzMx2nxOEmZkVcoIwM7NCThBmZlbICcLMzAo5QVhHSMsTfDi3vVDSV3Lbn5d0fp3jr5F0Znp+m6Tn3The0lBJc5St2vpzSXdJekva97Ck0bsR947z9rD/irS650pJm3KrfZ4p6ebqmPr+JOkgSd+ts39vST/JTUyzQcoJwjrFHcDrACTtBYwGjsztfx1w5x6e4xKyFVFfHtnyLDPIVkVtmIj4YGTLMpwKPBgRR6XHjRFxamSzcfvb+WQzqnuKaQvZvI+/aMC5rYM4QVinuJNsyQXIEsP9wNOSRqaFzw4Hfi7pk5IWS7pf0tyyK3tKeiHwPuC8yJZDILIlKuYV1D0/vf79NVc1f5kWY7tP0tcKjrskXVEMKRnTw5JGS5oo6Zfp2AckfV3SmyXdka52jkn1/0TZYnz3SFoq6bQeXvodwA/SMUem+vem2CenOvOB/10mThu4fAlpHSEiHpO0VdJ4squFu8hWvXwt2QqeyyNii6R/jYiLAdKX9FuB75Q4xUuBR6Jmcb5akl4FnAMcSzYL9meSbge2AB8HXhcR6yWNqjnuMrKrkXNi92anvhT4c+C9wGLgXWQzp98O/BPZ1c7HyJaaeG9qmrpH0o8i4plcHJOAp6pJEHg/cHlEfF3ZTXiqyet+soXlbBDzFYR1kjvJkkM1QdyV274j1TlR2Z3FlpPdQ+DIohfaA68HvhURz0S27v9NwBvSuW6IiPUAEZG/J8YngAMi4v27mRwAHoqI5ZEtuLcC+HF6reXAxFTnFGC2sqWmbyNbhmJ8zescBKzLbd8F/JOkfwQmRMSmFP82YIvSWmY2ODlBWCep9kO8guwv3LvJriBeB9wpaRjwb2R37noFWTv7sJKvvRoYL2n/fo86+4v/VbVXFX20Ofd8e257OztbAgS8I9ePMT4iflHzOpvIfSYRcR3ZVcgm4GZJJ+Xq7gP8cQ9itg7nBGGd5E6yJqMNkS1fvQEYQZYk7mTnF996ZfdE6HH0UK2IeJZsJdTLtfN+x2Mk/XlN1Z8CMyS9UNkKwaenskXAn0s6MB2bTwY/AOYA32vwX+QLgfOq/S6SphbUeYCdVxxIOhRYExFfIlsh9ZWp/EBgfUQ818B4rc05QVgnWU42eunumrLfRcT6NOLnKrKri4Vkf7n3xcfJml9WSrof+C5Qe8Ogn5PdX/gesru1fSUilkbECuBS4HZJ9wFfqDnuhhTbgrQMdCNcQnaLy2WSVqTtXaT+iAclvTQVvRO4PzVLvRy4NpWfCHyvQXFah/BqrmaDjKTTgVdFxMfr1LkJmB0RDzQvMms3HsVkNshExLeqTWFFUhPbfCcH8xWEmZkVch+EmZkVcoIwM7NCThBmZlbICcLMzAo5QZiZWaH/AVC/n89BGYiaAAAAAElFTkSuQmCC", - "text/plain": [ - "
" + "cell_type": "markdown", + "metadata": { + "id": "-yEuLXoHua-f" + }, + "source": [ + "Note: throughout this notebook, you may see a few ModuleNotFoundErrors. As long as the cell successfully executes, you can ignore that error." ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "plt.title('Learning Curve')\n", - "plt.xlabel('Wall Clock Time (s)')\n", - "plt.ylabel('Validation Accuracy')\n", - "print(len(valid_loss_history))\n", - "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", - "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Spooky-author-identification example" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ + }, { - "data": { - "text/html": [ - "== Status ==
Current time: 2022-07-22 07:20:52 (running for 00:30:13.14)
Memory usage on this node: 19.0/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/4 CPUs, 0/4 GPUs, 0.0/252.65 GiB heap, 0.0/112.27 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: 504afb96 with val_loss=0.11011235955056176 and parameters={'learning_rate': 4.567279255636039e-05, 'num_train_epochs': 5, 'per_device_train_batch_size': 32, 'seed': 37, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38
Number of trials: 12/1000000 (12 TERMINATED)

" + "cell_type": "markdown", + "metadata": { + "id": "ZBr83DYlTjhD" + }, + "source": [ + "## 2. Sentiment Classification Example\n", + "### Load data and preprocess\n", + "\n", + "The Stanford Sentiment treebank (SST-2) dataset is a dataset for sentiment classification. First, let's load this dataset into pandas dataframes:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 106, + "referenced_widgets": [ + "16fba9eb9e4542bc9d34eca00d71cc14", + "c81f11a99b9d4d1d95533f24bea1d5ac", + "e4ce5cf6ea174583a14675a75d31992d", + "0c8473019e434db0ae34d58b69605a69", + "814d3f2b7212461ca51f8635b5106783", + "129488cadbb9477ca593ae106ee8e9f7", + "c7aaa1fbd10942649c90044c0c901d99", + "6fe78cfd377b4c10a626588f46a569cf", + "2aa0f33b8d3a4bc7bedf3b66e06b62f0", + "1794a34790b647b3a8a845c55e5e0744", + "13d813116e1846a3a0e42a5e8423f80e", + "2aa02213244048fead33ea157c17837b", + "dfee126a02ef4934a0f654a101dadc1b", + "8e58d795d528405f8d1c48bfc2afe399", + "e23212ae504e493a85e4b2524f0217e1", + "b84af540de4741ceb206456d2f05fa4b", + "9ba620aa4e51456c9b3b2469c2c887c3", + "cc8f1bc7322542828205777903530f1e", + "c5f0ba4cda014a63a99ecc989f72f731", + "44803ba97fd54ceab2afe0555e21dfe8", + "f4ca1b7b5868446da945574f4db4373b", + "f7070757d4784c8099ef7dd9bd280ed3", + "6a8be136bafe40eea3430dda4063e6db", + "f28c1b9dee064db99c389beb98306f86", + "b2a5550cd6474de7a46eab6a973305e0", + "a4e9b7b28055406c9569e585296850c6", + "ac534b02efb34100b53999031767e8a3", + "5f9f744825e44fdbae9c126837e40efe", + "c3f7a2bb90b44a21a43939f78914f9b8", + "4ec28fbc9433413f8355e0c976839a94", + "fab424da92b541cfac6b3bb05ee4e17b", + "8cfa8c0a28f549c19649ec9b390aa528", + "470b17af8c8442e49757dc4e385d16f0", + "2c31d2eb9ae44ffbb0f02ad1b1e7937a", + "cd5ec1bd9cf54dbfbfd577a096a9e588", + "d1ba6870db484696879f0d6e5d3a9d70", + "e7405147ca374cc4998ca947be069652", + "fb72bbb82aec4184a8e0a510177433cf", + "f37c66b863854588b7a8891720372dc6", + "f48c7243c49a43acac4d1ba3a6fe674f", + "95ad5a87c66f405599a710b8a5fa0a9d", + "de103ee4780843db8502ebe64f5d2b28", + "428db79c8cd74257ad09539518a21835", + "3c95b6d54b294fe2a958056c463ce541" + ] + }, + "id": "hGP2eqTBTjhD", + "outputId": "56245bac-5fe2-4dcc-c7ae-714d2a11be5d" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading and preparing dataset glue/sst2 (download: 7.09 MiB, generated: 4.81 MiB, post-processed: Unknown size, total: 11.90 MiB) to /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad...\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading data: 0%| | 0.00/7.44M [00:00" + "source": [ + "from datasets import load_dataset\n", + "\n", + "train_dataset = load_dataset(\"glue\", \"sst2\", split=\"train\").to_pandas()\n", + "dev_dataset = load_dataset(\"glue\", \"sst2\", split=\"validation\").to_pandas()\n", + "test_dataset = load_dataset(\"glue\", \"sst2\", split=\"test\").to_pandas()" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m {'train_runtime': 675.882, 'train_samples_per_second': 108.628, 'train_steps_per_second': 6.791, 'train_loss': 0.15587145374491324, 'epoch': 5.0}\n" - ] + "cell_type": "markdown", + "metadata": { + "id": "Nb7SAWVLTjhE" + }, + "source": [ + "Take a look at the first 5 examples of this dataset:" + ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m Num examples = 4895\n", - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m Batch size = 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m {'eval_loss': 0.47973549365997314, 'eval_automl_metric': 0.12134831460674156, 'eval_runtime': 42.4421, 'eval_samples_per_second': 115.334, 'eval_steps_per_second': 115.334, 'epoch': 4.0}\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m {'train_runtime': 482.2078, 'train_samples_per_second': 121.806, 'train_steps_per_second': 1.908, 'train_loss': 0.23729169679724652, 'epoch': 4.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m Num examples = 4895\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m Batch size = 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'loss': 0.1844, 'learning_rate': 2.7020242630660653e-06, 'epoch': 3.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/vocab.txt\n", - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=78542)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/tokenizer_config.json\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/vocab.txt\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=78793)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/tokenizer_config.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'eval_loss': 0.7813186049461365, 'eval_automl_metric': 0.13564862104187947, 'eval_runtime': 38.0461, 'eval_samples_per_second': 128.66, 'eval_steps_per_second': 128.66, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'loss': 0.0829, 'learning_rate': 2.3353000145500413e-06, 'epoch': 3.13}\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m {'eval_loss': 0.5877022743225098, 'eval_automl_metric': 0.11848825331971402, 'eval_runtime': 40.6459, 'eval_samples_per_second': 120.43, 'eval_steps_per_second': 120.43, 'epoch': 4.0}\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'loss': 0.0714, 'learning_rate': 1.9685757660340173e-06, 'epoch': 3.27}\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m {'loss': 0.0237, 'learning_rate': 3.91430499829805e-06, 'epoch': 4.36}\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'loss': 0.0584, 'learning_rate': 1.6018515175179931e-06, 'epoch': 3.41}\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'loss': 0.0836, 'learning_rate': 1.235127269001969e-06, 'epoch': 3.54}\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'loss': 0.0335, 'learning_rate': 8.684030204859451e-07, 'epoch': 3.68}\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m {'eval_loss': 0.6322397589683533, 'eval_automl_metric': 0.11664964249233911, 'eval_runtime': 39.775, 'eval_samples_per_second': 123.067, 'eval_steps_per_second': 123.067, 'epoch': 5.0}\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m {'train_runtime': 607.6324, 'train_samples_per_second': 120.83, 'train_steps_per_second': 3.777, 'train_loss': 0.17213530145699163, 'epoch': 5.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m Num examples = 4895\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m Batch size = 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'loss': 0.0562, 'learning_rate': 5.01678771969921e-07, 'epoch': 3.81}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/vocab.txt\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=79025)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/tokenizer_config.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'loss': 0.0527, 'learning_rate': 1.3495452345389687e-07, 'epoch': 3.95}\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'eval_loss': 0.8104404211044312, 'eval_automl_metric': 0.12625127681307458, 'eval_runtime': 37.4885, 'eval_samples_per_second': 130.573, 'eval_steps_per_second': 130.573, 'epoch': 4.0}\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m {'train_runtime': 1051.1951, 'train_samples_per_second': 55.875, 'train_steps_per_second': 13.969, 'train_loss': 0.2927411694969607, 'epoch': 4.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m Num examples = 4895\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m Batch size = 1\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/vocab.txt\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=78225)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/tokenizer_config.json\n", - "2022-07-22 07:27:20,126\tINFO tune.py:747 -- Total run time: 2201.85 seconds (1801.97 seconds for the tuning loop).\n", - "[flaml.automl: 07-22 07:27:25] {3314} INFO - selected model: None\n", - "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 0.2317, 'learning_rate': 5.95732076822092e-06, 'epoch': 4.35}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[flaml.automl: 07-22 07:35:19] {3457} INFO - retrain transformer for 474.4s\n", - "[flaml.automl: 07-22 07:35:19] {3464} INFO - retrained model: None\n", - "[flaml.automl: 07-22 07:35:19] {2742} INFO - fit succeeded\n", - "[flaml.automl: 07-22 07:35:19] {2743} INFO - Time taken to find the best model: 1118.247492313385\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'train_runtime': 463.5873, 'train_samples_per_second': 158.374, 'train_steps_per_second': 1.24, 'train_loss': 0.20362980179164722, 'epoch': 5.0}\n" - ] - } - ], - "source": [ - "from flaml import AutoML\n", - "import ray\n", - "import pandas as pd\n", - "from sklearn.model_selection import train_test_split\n", - "ray.init(num_cpus=4, num_gpus=4, ignore_reinit_error=True)\n", - "\n", - "df = pd.read_csv('/data/xliu127/projects/hyperopt/FLAML/data/spooky-author-identification.csv')\n", - "X, y = df.drop('author', axis=1), df['author']\n", - "\n", - "X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=123)\n", - "\n", - "\n", - "print(len(X_train), len(X_val))\n", - "automl_model = AutoML()\n", - "\n", - "automl_settings = {\n", - " \"time_budget\": 1800, \n", - " \"task\": \"seq-classification\", \n", - " \"fit_kwargs_by_estimator\": {\n", - " \"transformer\": {\n", - " \"output_dir\": \"data/output/\", \n", - " \"model_path\": \"bert-base-uncased\", \n", - " }\n", - " },\n", - " \"metric\": \"accuracy\",\n", - " \"gpu_per_trial\": 1, \n", - " \"log_file_name\": \"spooky_bert.log\", \n", - " \"log_type\": \"all\", \n", - " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n", - " \"n_concurrent_trials\": 4,\n", - " \"keep_search_state\": True, # keeping the search state\n", - "}\n", - "\n", - "automl_model.fit(X_train=X_train, y_train=y_train,X_val=X_val, y_val=y_val, **automl_settings)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "the best loss for spooky author identification: 0.11133810010214507\n" - ] - } - ], - "source": [ - "print(\"the best loss for spooky author identification: {}\".format(automl_model.best_loss))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "== Status ==
Current time: 2022-07-21 21:21:15 (running for 00:30:10.30)
Memory usage on this node: 20.5/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/4 CPUs, 0/4 GPUs, 0.0/252.62 GiB heap, 0.0/112.26 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: 84d3be85 with val_loss=0.12951991828396325 and parameters={'learning_rate': 4.486769916716146e-05, 'num_train_epochs': 4, 'per_device_train_batch_size': 8, 'seed': 28, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05
Number of trials: 12/1000000 (12 TERMINATED)

" + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "65mLkoJhTjhE", + "outputId": "2ec2ba75-caeb-4e6e-e1f8-78ee900f525d" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sentencelabelidx
0hide new secretions from the parental units00
1contains no wit , only labored gags01
2that loves its characters and communicates som...12
3remains utterly satisfied to remain the same t...03
4on the worst revenge-of-the-nerds clichés the ...04
\n", + "
" + ], + "text/plain": [ + " sentence label idx\n", + "0 hide new secretions from the parental units 0 0\n", + "1 contains no wit , only labored gags 0 1\n", + "2 that loves its characters and communicates som... 1 2\n", + "3 remains utterly satisfied to remain the same t... 0 3\n", + "4 on the worst revenge-of-the-nerds clichés the ... 0 4" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "train_dataset.head(5)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m {'eval_loss': 0.7418951392173767, 'eval_automl_metric': 0.1284984678243105, 'eval_runtime': 37.3935, 'eval_samples_per_second': 130.905, 'eval_steps_per_second': 130.905, 'epoch': 4.0}\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m {'train_runtime': 565.7729, 'train_samples_per_second': 103.816, 'train_steps_per_second': 6.49, 'train_loss': 0.2802804773409642, 'epoch': 4.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m Num examples = 4895\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m Batch size = 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m {'eval_loss': 1.0893423557281494, 'eval_automl_metric': 0.6024514811031665, 'eval_runtime': 39.7178, 'eval_samples_per_second': 123.245, 'eval_steps_per_second': 123.245, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.2369, 'learning_rate': 1.4090340380281214e-05, 'epoch': 2.72}\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m {'train_runtime': 566.9953, 'train_samples_per_second': 77.694, 'train_steps_per_second': 9.714, 'train_loss': 1.0928592581461545, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'eval_loss': 1.092341661453247, 'eval_automl_metric': 0.6024514811031665, 'eval_runtime': 38.0057, 'eval_samples_per_second': 128.797, 'eval_steps_per_second': 128.797, 'epoch': 3.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m Num examples = 4895\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m Batch size = 1\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/vocab.json\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/merges.txt\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=50245)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/tokenizer_config.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'loss': 1.0896, 'learning_rate': 1.5104688589428795e-05, 'epoch': 3.13}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/vocab.json\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/merges.txt\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=50412)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/tokenizer_config.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.2195, 'learning_rate': 1.2404892966371977e-05, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'loss': 1.0907, 'learning_rate': 1.2732721160184323e-05, 'epoch': 3.27}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.1252, 'learning_rate': 1.0719445552462741e-05, 'epoch': 3.27}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'loss': 1.0926, 'learning_rate': 1.0360753730939852e-05, 'epoch': 3.41}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.1093, 'learning_rate': 9.033998138553504e-06, 'epoch': 3.54}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'loss': 1.0908, 'learning_rate': 7.988786301695379e-06, 'epoch': 3.54}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.1166, 'learning_rate': 7.348550724644269e-06, 'epoch': 3.81}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'loss': 1.0899, 'learning_rate': 5.616818872450909e-06, 'epoch': 3.68}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'loss': 1.0923, 'learning_rate': 3.244851443206437e-06, 'epoch': 3.81}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'eval_loss': 0.7831101417541504, 'eval_automl_metric': 0.13462717058222673, 'eval_runtime': 37.9679, 'eval_samples_per_second': 128.925, 'eval_steps_per_second': 128.925, 'epoch': 4.0}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'loss': 1.0862, 'learning_rate': 8.728840139619655e-07, 'epoch': 3.95}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.1164, 'learning_rate': 5.663103310735033e-06, 'epoch': 4.08}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'eval_loss': 1.0893481969833374, 'eval_automl_metric': 0.6024514811031665, 'eval_runtime': 36.2865, 'eval_samples_per_second': 134.899, 'eval_steps_per_second': 134.899, 'epoch': 4.0}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m {'train_runtime': 1069.9104, 'train_samples_per_second': 54.898, 'train_steps_per_second': 13.725, 'train_loss': 1.0960205283875493, 'epoch': 4.0}\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m Num examples = 4895\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m Batch size = 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.0542, 'learning_rate': 3.977655896825797e-06, 'epoch': 4.36}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/vocab.json\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/merges.txt\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=49988)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/tokenizer_config.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.0618, 'learning_rate': 2.2922084829165607e-06, 'epoch': 4.63}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'loss': 0.0494, 'learning_rate': 6.06761069007325e-07, 'epoch': 4.9}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'eval_loss': 0.88468998670578, 'eval_automl_metric': 0.12972420837589382, 'eval_runtime': 37.9519, 'eval_samples_per_second': 128.979, 'eval_steps_per_second': 128.979, 'epoch': 5.0}\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m {'train_runtime': 873.0679, 'train_samples_per_second': 84.094, 'train_steps_per_second': 10.515, 'train_loss': 0.27977710040306475, 'epoch': 5.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m Num examples = 4895\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m Batch size = 1\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/added_tokens.json. We won't load it.\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/vocab.json\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/merges.txt\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/tokenizer.json\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m loading file None\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/special_tokens_map.json\n", - "\u001b[2m\u001b[36m(train pid=50658)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/tokenizer_config.json\n", - "2022-07-21 21:29:43,228\tINFO tune.py:747 -- Total run time: 2317.81 seconds (1801.93 seconds for the tuning loop).\n", - "[flaml.automl: 07-21 21:29:46] {3314} INFO - selected model: None\n", - "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 0.5742, 'learning_rate': 3.264882684494973e-05, 'epoch': 1.09}\n" - ] - } - ], - "source": [ - "automl_settings[\"fit_kwargs_by_estimator\"][\"transformer\"][\"model_path\"] = \"roberta-base\"\n", - "automl_settings[\"log_file_name\"] = \"spooky_roberta.log\"\n", - "automl_model.fit(X_train=X_train, y_train=y_train,X_val=X_val, y_val=y_val, **automl_settings)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "8\n", - "8\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaA0lEQVR4nO3dfXRV9Z3v8ffXyEOKCEpShQRNdDEMj0Jv5KG0V1uHgq2AWBfC1XtLO63tdaF36QwtXLuQyXWqlVntHbvoVG+rtLN8gKEUgzorVdB22TKaAMpDMBIeCgl2jCj4FCQk3/vH2Qkn6Ulyguecfc7O57VWVvb+7c05v192+GSf3/7t3zZ3R0REct85YVdARERSQ4EuIhIRCnQRkYhQoIuIRIQCXUQkIs4N640LCgq8pKQkrLcXEclJ27Zte9vdCxNtCy3QS0pKqK6uDuvtRURykpn9qatt6nIREYmIpALdzGabWa2Z1ZnZsgTbLzWzzWa208xeNLPi1FdVRES602Ogm1kesBq4FhgLLDKzsZ12+yfgV+4+ESgH7kt1RUVEpHvJnKFPAerc/YC7nwKeBOZ12mcssCVYfiHBdhERSbNkAr0IOBK3Xh+UxXsNuCFYng8MNrNhnV/IzG41s2ozq25sbDyb+oqISBdSdVH074GrzGwHcBXQALR03sndH3b3MncvKyxMOOpGRLLQxh0NzLh/C6XLnmHG/VvYuKMh7CpJAskMW2wARsatFwdl7dz9KMEZupmdB3zV3Y+nqI4iEqKNOxpYvmEXTc2xc7SG400s37ALgOsnd/6wLmFKJtCrgFFmVkosyBcC/y1+BzMrAN5x91ZgOfBIqisqIuFYVVnbHuZtmppb+O76nTzxyuGQapXbxo44n3vmjEv56/bY5eLup4ElQCWwF1jn7nvMrNzM5ga7XQ3UmtkbwEXAP6a8piISiqPHmxKWn2ppzXBNpCdJ3Snq7s8Cz3YqWxG3vB5Yn9qqiUg2GDE0n4YEoV40NJ+1354eQo2kK7pTVES6tXTWaPL75XUoy++Xx9JZo0OqkXQltLlcRCQ3tF34/O76nZxqaaVoaD5LZ43WBdEspEAXkR5dP7mo/QKoulmyl7pcJPftXAc/Hg8rh8a+71wXdo1EQqEzdMltO9fBpjugObhod+JIbB1g4oLw6pVjNu5oYFVlLUePNzFCXSo5y9w9lDcuKytzzYcun9iPx8dCvLO8AVB8Zebrk4Pe/uBjDrz9Ia1xWXCOGZcVDKLgvAHtZXvePAHAuOFDMl7HrHfxBLj2/oy8lZltc/eyRNt0hi657UR94vKWjzNbjxx2+J2POoQ5QKs7+xs/4D/fP9le9tGpFj7VP6/zP5csokCX3DakOPEZ+pCR8PVnMl+fHPTVZc/Q1ef0qUUXdlifN6mIcVMvSX+l5Kwo0CW3XbOiYx86QL/8WLkkRTcORYdGuUhum7gA5jwY6zOH2Jn5nAd1QbQXdONQdOgMXXLfxAWw7ZexZXWz9JpuHIoOBbqI6MahiFCgZ4DG+IpIJijQ00wPB5CM2rkONpfHhnMOKY5dHNb1hD5DgZ5mejhAZqw4FrvppfyhrSHXJEQfNsKxI9D6tdj6W8DaI/B8BQzq+ZGPLx98h6mlF/a4n8TJsj+gGuWSZno4gGTMu4egtdPvVWtrrDwJU0svZN4kfWpMWtu0EyeOAH5m2okQ5xLSGXqaaYxvhjwaux197df78M905bWQl+gWIYNvH890baJvc3nH+x8gtr65PLSzdJ2hp1mPY3w1U6CkypDi3pXLJ9PVtBNdlWeAAj3Nrp9cxH03TKB/XuxHXTQ0n/tumBC7IJqFH9kkh12zInaXbDzdNZs+WfgHVF0uGdDlGN+uPrI9teTMjTKSnD/vis1415e1fczPoot0kZaF004o0MOkmQJT5+IJMOHGsGsRvokLFOCZkoV/QBXoYdJMgSK5Lcv+gKoPPUzq8xSRFFKgh0kzBYpICqnLJWyaKVBEUiS3ztA1ZltEpEu5c4aup7uLiHQrdwI9E2O2P3wL3v1TbNhg3gC44FIY9OmUvHTb5FFtt6h3oDHUIhkT5emscyfQ0z1m+8O34FgdeOuZ1z1WF1tOUah3SWOoRTIi6tNZ506gp3vM9o/HnwnzNt4a+xSQgtdvm9a1T08eJRKyrqazXlVZG4lAz52Loukes52FE+2ISGp1NZ11V+W5JqlAN7PZZlZrZnVmtizB9kvM7AUz22FmO83syymvaduY7SEjAUv9mO0snGhHRFJrxND8XpWn2sYdDcy4fwuly55hxv1b2LijIaWv32OXi5nlAauBmUA9UGVmFe5eE7fb94F17v4vZjYWeBYoSWlNIb232WbhRDsiklpLZ43u0IcOnaazTqNM9N8n04c+Bahz9wMAZvYkMA+ID3QHzg+WhwBHU1K7TMrCiXZEMiHKoz46a2tXGO3NRP99MoFeBMRfjawHpnbaZyXwWzO7HRgE/E2iFzKzW4FbAS655JLe1jX9smyiHZF0i/qoj0Sun1wUStsy0X+fqouii4A17l4MfBn4VzP7i9d294fdvczdywoLe35orYikV3dnjZJamei/TybQG4CRcevFQVm8vwXWAbj7VmAgUJCKCopI+kR91Ec26fFxlCmQTKBXAaPMrNTM+gMLgYpO+xwGrgEwszHEAr0xZbUUkbQIe9RHX9L2OMqiofkYnR5HmSI99qG7+2kzWwJUAnnAI+6+x8zKgWp3rwD+Dvh/ZnYnsQuki9090ePHRSSLhDnqoy9Kd/99UneKuvuzxIYixpetiFuuAWaktmoikm5hjvqQ1MudW/9FJC3CGvUhqZc7t/6LiEi3FOgiIhGhQBcRiQgFuohIRCjQRUQiQoEuIhIRCnQRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEQp0EZGI6FPzoW/c0aCJ/EUksvpMoG/c0dDhUVsNx5tYvmEXgEJdJNvsXAeby+FEPQwphmtWwMQFYdcq6/WZQF9VWdvhuYkATc0tfHf9Tp545XDa37/mzfcYO/z8tL+PSM7buQ423QHNTbH1E0di66BQ70Gf6UM/erwpYfmpltaMvP/Y4eczb5I+CYj0aHP5mTBv09wUK5du9Zkz9BFD82lIEOpFQ/NZ++3pIdRIRBI6Ud+7cmnXZ87Ql84aTX6/vA5l+f3yWDprdEg1EpGEhhT3rlza9ZlAv35yEffdMIGiofkYsTPz+26YoAuiItnmmhXQL79jWb/8WLl0q890uUAs1BXgIlmu7cKnRrn0Wp8KdBHJERMXKMDPQp/pchERiToFuohIRCjQRUQiQoEuIhIRCnQRkYhQoIuIRERSgW5ms82s1szqzGxZgu0/NrNXg683zOx4ymsqIiLd6nEcupnlAauBmUA9UGVmFe5e07aPu98Zt//twOQ01FVERLqRzBn6FKDO3Q+4+yngSWBeN/svAp5IReVERCR5yQR6EXAkbr0+KPsLZnYpUAps6WL7rWZWbWbVjY2Nva2riIh0I9UXRRcC6929JdFGd3/Y3cvcvaywsDDFby0i0rclE+gNwMi49eKgLJGFqLtFRCQUyQR6FTDKzErNrD+x0K7ovJOZ/TVwAbA1tVUUEZFk9Bjo7n4aWAJUAnuBde6+x8zKzWxu3K4LgSfd3dNTVRER6U5S0+e6+7PAs53KVnRaX5m6aomISG/pTlERkYhQoIuIRIQCXUQkIhToIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEUndWCS9t3FHA6sqazl6vIkRQ/NZOms0109OOEmliEhKKNDTYOOOBpZv2EVTc2zSyYbjTSzfsAtAoS4iaaMulzRYVVnbHuZtmppbWFVZG1KNRKQvUKCnwdHjTb0qFxFJBQV6GowYmt+rchGRVFCgp8HSWaPJ75fXoSy/Xx5LZ40OqUYi0hfoomgatF341CgXEckkBXqaXD+5SAEuIhmlLhcRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAFxGJCAW6iEhEKNBFRCIiqUA3s9lmVmtmdWa2rIt9FphZjZntMbPHU1tNERHpSY8PuDCzPGA1MBOoB6rMrMLda+L2GQUsB2a4+7tm9ul0VVhERBJL5gx9ClDn7gfc/RTwJDCv0z7fAla7+7sA7v5WaqspIiI9SSbQi4Ajcev1QVm8vwL+ysz+YGb/YWazU1VBERFJTqqeKXouMAq4GigGfm9mE9z9ePxOZnYrcCvAJZdckqK3FhERSO4MvQEYGbdeHJTFqwcq3L3Z3Q8CbxAL+A7c/WF3L3P3ssLCwrOts4iIJJBMoFcBo8ys1Mz6AwuBik77bCR2do6ZFRDrgjmQumqKiEhPegx0dz8NLAEqgb3AOnffY2blZjY32K0SOGZmNcALwFJ3P5auSouIyF8ydw/ljcvKyry6ujqU9xYRyVVmts3dyxJt052iIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAFxGJCAW6iEhEKNBFRCJCgS4iEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGhQBcRiQgFuohIRCjQRUQiQoEuIhIRCnQRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEQp0EZGIUKCLiESEAl1EJCKSCnQzm21mtWZWZ2bLEmxfbGaNZvZq8PXN1FdVRES6c25PO5hZHrAamAnUA1VmVuHuNZ12XevuS9JQRxERSUIyZ+hTgDp3P+Dup4AngXnprZaIiPRWMoFeBByJW68Pyjr7qpntNLP1ZjYy0QuZ2a1mVm1m1Y2NjWdRXRER6UqqLopuAkrcfSLwHPDLRDu5+8PuXubuZYWFhSl6axERgeQCvQGIP+MuDsraufsxd/84WP058F9SUz0REUlWMoFeBYwys1Iz6w8sBCridzCz4XGrc4G9qauiiIgko8dRLu5+2syWAJVAHvCIu+8xs3Kg2t0rgDvMbC5wGngHWJzGOouISALm7qG8cVlZmVdXV4fy3iIiucrMtrl7WaJtulNURCQiFOgiIhGhQBcRiQgFuohIRCjQRUQiQoEuIhIRCnQRkYjo8caiTGpubqa+vp6TJ0+GXZWsM3DgQIqLi+nXr1/YVRGRLJVVgV5fX8/gwYMpKSnBzMKuTtZwd44dO0Z9fT2lpaVhV0dEslRWdbmcPHmSYcOGKcw7MTOGDRumTy4i0q2sCnRAYd4F/VxEpCdZF+giInJ2FOidHDp0iPHjx5/1v9+4cSM1NZ0ftyoikn4K9BQ6ffq0Al1EQpNVo1zi/cOmPdQcfS+lrzl2xPncM2dcj/udPn2am2++me3btzNu3Dh+9atfsXfvXu666y4++OADCgoKWLNmDcOHD+fqq69m0qRJvPTSS8yfP5+Kigp+97vfce+99/LrX/+ayy+/PKVtEBHpStYGephqa2v5xS9+wYwZM/jGN77B6tWr+c1vfsNTTz1FYWEha9eu5e677+aRRx4B4NSpU7TN7b5v3z6uu+46brzxxjCbICJ9UNYGejJn0ukycuRIZsyYAcAtt9zCD37wA3bv3s3MmTMBaGlpYfjwM0/du+mmm0Kpp4hIvKwN9DB1HiI4ePBgxo0bx9atWxPuP2jQoExUS0SkW7oomsDhw4fbw/vxxx9n2rRpNDY2tpc1NzezZ8+ehP928ODBvP/++xmrq4hIGwV6AqNHj2b16tWMGTOGd999l9tvv53169fzve99jyuuuIJJkybxxz/+MeG/XbhwIatWrWLy5Mns378/wzUXkb4sqx4SvXfvXsaMGRNKfXKBfj4ioodEi4j0AQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAFxGJCAX6WXjxxRe57rrrPtFrrFmzhqNHj6aoRiIiCvRuuTutra0pf92WlhYFuoikXPbO5fLvy+DPu1L7mhdPgGvv73aXQ4cOMWvWLKZOncq2bduYMmUKVVVVmBnf//732yfieu+99/jKV75CXV0dX/jCF/jpT3/KOeecw29/+1vuuecePv74Yy6//HIeffRRzjvvPEpKSrjpppt47rnnuOuuu6iurubmm28mPz+frVu3smrVKjZt2kRTUxOf/exneeihh/TYORHpFZ2hJ7Bv3z5uu+02ysvLqa+v57XXXuP5559n6dKlvPnmmwC88sor/OQnP6Gmpob9+/ezYcMG3n77be69916ef/55tm/fTllZGT/60Y/aX3fYsGFs376dW265hbKyMh577DFeffVV8vPzWbJkCVVVVezevZumpiaefvrpsJovIjkqqTN0M5sN/DOQB/zc3ROe5prZV4H1wJXuXp1on6T1cCadTpdeeinTpk3jzjvvZNGiReTl5XHRRRdx1VVXUVVVxfnnn8+UKVO47LLLAFi0aBEvvfQSAwcOpKampn3q3VOnTjF9+vT21+1umt0XXniBBx54gI8++oh33nmHcePGMWfOnPQ2VEQipcdAN7M8YDUwE6gHqsyswt1rOu03GPhfwMvpqGgmJTMdbufuEDPD3Zk5cyZPPPFEr1735MmT3HbbbVRXVzNy5EhWrlzJyZMne19xkajauQ42l8OJehhSDNesgIkLwq5V1kmmy2UKUOfuB9z9FPAkMC/Bfv8H+CEQmST6/Oc/z9q1a2lpaaGxsZHf//73TJkyBYh1uRw8eJDW1lbWrl3L5z73OaZNm8Yf/vAH6urqAPjwww954403Er52/DS7beFdUFDABx98wPr16zPQOpEcsXMdbLoDThwBPPZ90x2xcukgmUAvAo7ErdcHZe3M7DPASHd/prsXMrNbzazazKobGxt7XdlMmz9/PhMnTuSKK67gi1/8Ig888AAXX3wxAFdeeSVLlixhzJgxlJaWMn/+fAoLC1mzZg2LFi1i4sSJTJ8+nddffz3hay9evJjvfOc7TJo0iQEDBvCtb32L8ePHM2vWLK688spMNlMku20uh+amjmXNTbFy6aDH6XPN7EZgtrt/M1j/78BUd18SrJ8DbAEWu/shM3sR+Pue+tA1fW7v6ecjfdLKoUCinDJYeTyzdckCn3T63AZgZNx6cVDWZjAwHnjRzA4B04AKM0v4hiIivTKkuHflfVgygV4FjDKzUjPrDywEKto2uvsJdy9w9xJ3LwH+A5j7iUe5iIhA7AJov/yOZf3yY+XSQY+B7u6ngSVAJbAXWOfue8ys3MzmprpCYT1BKdvp5yJ91sQFMOdBGDISsNj3OQ9qlEsCWfUIuoMHDzJ48GCGDRumuyTjuDvHjh3j/fffp7S0NOzqiEiIuutDz6pb/4uLi6mvrycXRsBk2sCBAykuVp+hiHQtqwK9X79+OgMVETlLmstFRCQiFOgiIhGhQBcRiYjQRrmYWSPwp1DePHUKgLfDrkSKRa1NUWsPRK9Nak/vXOruhYk2hBboUWBm1V0NH8pVUWtT1NoD0WuT2pM66nIREYkIBbqISEQo0D+Zh8OuQBpErU1Raw9Er01qT4qoD11EJCJ0hi4iEhEKdBGRiFCg98DM8sxsh5k9HayXmtnLZlZnZmuDOeIxswHBel2wvSTUinfBzIaa2Xoze93M9prZdDO70MyeM7N9wfcLgn3NzB4M2rQzeNRgVjGzO81sj5ntNrMnzGxgrh0jM3vEzN4ys91xZb0+Jmb2tWD/fWb2tTDaEtQjUXtWBb9zO83sN2Y2NG7b8qA9tWY2K658dlBWZ2bLMtyMDhK1KW7b35mZm1lBsB7eMXJ3fXXzBdwFPA48HayvAxYGyz8D/mewfBvws2B5IbA27Lp30Z5fAt8MlvsDQ4EHgGVB2TLgh8Hyl4F/B4zYk6heDrv+ndpSBBwE8uOOzeJcO0bAfwU+A+yOK+vVMQEuBA4E3y8Ili/IovZ8CTg3WP5hXHvGAq8BA4BSYD+QF3ztBy4Lfk9fA8Zm0zEKykcSe1bEn4CCsI9R6L/M2fxF7HF7m4EvAk8HB+jtuF/M6UBlsFwJTA+Wzw32s7Db0Kk9Q4IAtE7ltcDwYHk4UBssPwQsSrRfNnxx5gHmFwY/86eBWbl4jICSTgHYq2MCLAIeiivvsF/Y7em0bT7wWLC8HFget60yOGbtxy3RftnSJmA9cAVwKC7QQztG6nLp3v8Fvgu0BuvDgOMee4oTQD2xUIEz4UKw/USwfzYpBRqBR4NupJ+b2SDgInd/M9jnz8BFwXJ7mwLx7Q2duzcA/wQcBt4k9jPfRm4foza9PSZZfaw6+QaxM1jI4faY2Tygwd1f67QptDYp0LtgZtcBb7n7trDrkkLnEvvY+C/uPhn4kNjH+XYeO3XIibGsQb/yPGJ/qEYAg4DZoVYqDXLpmPTEzO4GTgOPhV2XT8LMPgX8byCrHmyqQO/aDGCumR0CniTW7fLPwFAza3swSDHQECw3EOtPI9g+BDiWyQonoR6od/eXg/X1xAL+P81sOEDw/a1ge3ubAvHtzQZ/Axx090Z3bwY2EDtuuXyM2vT2mGT7scLMFgPXATcHf6Qgd9tzObETideCjCgGtpvZxYTYJgV6F9x9ubsXu3sJsQtoW9z9ZuAF4MZgt68BTwXLFcE6wfYtcb+0WcHd/wwcMbPRQdE1QA0d6965Tf8juGo/DTgR1w2QDQ4D08zsU2ZmnGlPzh6jOL09JpXAl8zsguCTy5eCsqxgZrOJdV/OdfeP4jZVAAuDEUilwCjgFaAKGBWMWOpP7P9gRabr3RV33+Xun3b3kiAj6oHPBP/HwjtGYV5kyJUv4GrOjHK5jNgvXB3wb8CAoHxgsF4XbL8s7Hp30ZZJQDWwE9hI7Gr7MGIXf/cBzwMXBvsasJrYaINdQFnY9U/Qnn8AXgd2A/9KbLRETh0j4Ali1wCaiQXD357NMSHWN10XfH09y9pTR6z/+NXg62dx+98dtKcWuDau/MvAG8G2u7PtGHXafogzF0VDO0a69V9EJCLU5SIiEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGhQBcRiQgFuohIRPx/cNXA2tMFSnEAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" + "cell_type": "markdown", + "metadata": { + "id": "ENcUQbOgTjhE" + }, + "source": [ + "Separate the data into X and y:" ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "from flaml.data import get_output_from_log\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "for each_file_name in ['bert', 'roberta']:\n", - " time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", - " get_output_from_log(filename='spooky_' + each_file_name + '.log', time_budget=3000)\n", - " print(len(valid_loss_history))\n", - " plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", - " plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", - "\n", - "plt.legend(['bert', 'roberta'])\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Other Tasks" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Besides sequence classification, FLAML currently also supports four other tasks (more tasks are to be supported, which can be found on FLAML's documentation website https://microsoft.github.io/FLAML/docs/Examples/AutoML-NLP):\n", - "\n", - "- sequence regression: predicting a float number from the input sequence, e.g., predicting the rating of a hotel review based on the text content;\n", - "- token classification: predicting the label of each token in a sequence, e.g., named entity recognition;\n", - "- multiple choice: predicting the best second half of a sentence that comes next to the first part of a sentence based on common sensen reasoning. An example is seen below;\n", - "- (abstractive) summarization: generating the textual summarization of an input paragraph;\n", - "\n", - "For each task, you only have to change the \"Load data and preprocess\" with the corresponding data loading process. For example:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.1 Multiple Choice Example" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Multiple choice is a task of predicting the best second half of a sentence that follows the first half based on common sense reasoning. An example of multiple-choice classification problem is:\n", - "\n", - "On stage, a woman takes a seat at the piano. She\n", - "a) sits on a bench as her sister plays with the doll.\n", - "b) smiles with someone as the music plays.\n", - "c) is in the crowd, watching the dancers.\n", - "d) *nervously sets her fingers on the keys*." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ + }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "No config specified, defaulting to: swag/regular\n", - "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n", - "No config specified, defaulting to: swag/regular\n", - "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n", - "No config specified, defaulting to: swag/regular\n", - "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n" - ] - } - ], - "source": [ - "from datasets import load_dataset\n", - "\n", - "train_dataset = load_dataset(\"swag\", split=\"train\").to_pandas().iloc[:10000]\n", - "dev_dataset = load_dataset(\"swag\", split=\"validation\").to_pandas().iloc[:10000]\n", - "test_dataset = load_dataset(\"swag\", split=\"test\").to_pandas()\n", - "\n", - "custom_sent_keys = [\n", - " \"sent1\",\n", - " \"sent2\",\n", - " \"ending0\",\n", - " \"ending1\",\n", - " \"ending2\",\n", - " \"ending3\",\n", - " \"gold-source\",\n", - " \"video-id\",\n", - " \"startphrase\",\n", - " \"fold-ind\",\n", - " ] # specify the column names of the input sentences\n", - "label_key = \"label\" # specify the column name of the label\n", - "\n", - "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n", - "X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]\n", - "X_test = test_dataset[custom_sent_keys]" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Members of the procession walk down the street holding small horn brass instruments.'" + "cell_type": "code", + "execution_count": 13, + "metadata": { + "id": "GA0VH9URTjhF" + }, + "outputs": [], + "source": [ + "custom_sent_keys = [\"sentence\"] # specify the column names of the input sentences\n", + "label_key = \"label\" # specify the column name of the label\n", + "\n", + "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n", + "X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]\n", + "X_test = test_dataset[custom_sent_keys]" ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "train_dataset.iloc[0][\"sent1\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ + }, { - "data": { - "text/html": [ - "== Status ==
Current time: 2022-03-19 14:39:29 (running for 00:08:29.94)
Memory usage on this node: 33.0/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/250.17 GiB heap, 0.0/111.21 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: de45e672 with val_loss=0.18300000000000005 and parameters={'learning_rate': 6.104513714676502e-06, 'num_train_epochs': 2.3743291981165893, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.23610846764298543, 'weight_decay': 0.20205904544254147, 'adam_epsilon': 5.752964074991208e-08, 'seed': 41, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-03-19_14-30-59
Number of trials: 10/1000000 (10 TERMINATED)

" + "cell_type": "markdown", + "metadata": { + "id": "NpRqB153TjhF" + }, + "source": [ + "### Run FLAML" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2kXabqxZuzQl" + }, + "source": [ + "Now we can run AutoML with FLAML:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "id": "asYbkzrXTjhF" + }, + "outputs": [], + "source": [ + "from flaml import AutoML\n", + "automl = AutoML()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2XZmrBRru_A0" + }, + "source": [ + "Let's run FLAML for 30 mins. Here we use Electra's [small model](https://huggingface.co/google/electra-small-discriminator) for the tuning. We set gpu_per_trial to 1, and n_concurrent_trials to 1 (the number of trials running at the same time). Make sure gpu_per_trial * n_concurrent_trials does not exceed the GPU number you have. While running you can observe the resource usage (including the GPU) on the right. " + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "id": "QEvR2bZiTjhG" + }, + "outputs": [], + "source": [ + "TIME_BUDGET=1800\n", + "automl_settings = {\n", + " \"time_budget\": TIME_BUDGET, # setting the time budget\n", + " \"task\": \"seq-classification\", # setting the task as seq-classification\n", + " \"fit_kwargs_by_estimator\": {\n", + " \"transformer\": {\n", + " \"output_dir\": \"data/output/\", # setting the output directory\n", + " \"model_path\": \"google/electra-small-discriminator\", # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base\n", + " }\n", + " },\n", + " \"gpu_per_trial\": 1, # using 1 GPU for each trial\n", + " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n", + " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n", + " \"use_ray\": False, # If parallel tuning, set \"use_ray\" to {\"local_dir\": \"data/output/\"}\n", + " \"n_concurrent_trials\": 1, # How many trials to run at the same time, n_concurrent_trials * gpu_per_trial must not exceed the total number of GPUs\n", + " \"keep_search_state\": True, # keeping the search state\n", + " \"fp16\": False # whether to use fp16, this option is True by default. \n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "EXjF65hOTjhG", + "outputId": "e706d9e0-c890-41a0-cf9e-fb308d7c9533" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:3641: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " self[k1] = value[k2]\n", + "[flaml.automl: 08-21 02:50:27] {2565} INFO - task = seq-classification\n", + "INFO:flaml.automl:task = seq-classification\n", + "[flaml.automl: 08-21 02:50:27] {2567} INFO - Data split method: stratified\n", + "INFO:flaml.automl:Data split method: stratified\n", + "[flaml.automl: 08-21 02:50:27] {2570} INFO - Evaluation method: holdout\n", + "INFO:flaml.automl:Evaluation method: holdout\n", + "[flaml.automl: 08-21 02:50:27] {2689} INFO - Minimizing error metric: 1-accuracy\n", + "INFO:flaml.automl:Minimizing error metric: 1-accuracy\n", + "[flaml.automl: 08-21 02:50:27] {2831} INFO - List of ML learners in AutoML Run: ['transformer']\n", + "INFO:flaml.automl:List of ML learners in AutoML Run: ['transformer']\n", + "[flaml.automl: 08-21 02:50:27] {3133} INFO - iteration 0, current learner transformer\n", + "INFO:flaml.automl:iteration 0, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.5665, 'learning_rate': 4.6751863684771026e-06, 'epoch': 1.6}\n", + "{'eval_loss': 0.42372775077819824, 'eval_automl_metric': 0.1754587155963303, 'eval_runtime': 10.818, 'eval_samples_per_second': 80.606, 'eval_steps_per_second': 80.606, 'epoch': 2.0}\n", + "{'eval_loss': 0.4013938903808594, 'eval_automl_metric': 0.16399082568807344, 'eval_runtime': 10.8291, 'eval_samples_per_second': 80.524, 'eval_steps_per_second': 80.524, 'epoch': 3.0}\n", + "{'train_runtime': 81.4429, 'train_samples_per_second': 368.356, 'train_steps_per_second': 11.53, 'train_loss': 0.4875296855759951, 'epoch': 3.0}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_02-50-27/train_036eed6c_15_s=9223372036854775807,e=1e-05,s=-1,s=3,e=32,d=20_2022-08-21_02-50-27/checkpoint-939/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_02-50-27/train_036eed6c_15_s=9223372036854775807,e=1e-05,s=-1,s=3,e=32,d=20_2022-08-21_02-50-27/checkpoint-939/vocab.txt\n", + "loading file data/output/train_2022-08-21_02-50-27/train_036eed6c_15_s=9223372036854775807,e=1e-05,s=-1,s=3,e=32,d=20_2022-08-21_02-50-27/checkpoint-939/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_02-50-27/train_036eed6c_15_s=9223372036854775807,e=1e-05,s=-1,s=3,e=32,d=20_2022-08-21_02-50-27/checkpoint-939/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_02-50-27/train_036eed6c_15_s=9223372036854775807,e=1e-05,s=-1,s=3,e=32,d=20_2022-08-21_02-50-27/checkpoint-939/tokenizer_config.json\n", + "[flaml.automl: 08-21 02:52:09] {3267} INFO - Estimated sufficient time budget=6862714s. Estimated necessary time budget=6863s.\n", + "INFO:flaml.automl:Estimated sufficient time budget=6862714s. Estimated necessary time budget=6863s.\n", + "[flaml.automl: 08-21 02:52:09] {3319} INFO - at 102.1s,\testimator transformer's best error=0.1640,\tbest estimator transformer's best error=0.1640\n", + "INFO:flaml.automl: at 102.1s,\testimator transformer's best error=0.1640,\tbest estimator transformer's best error=0.1640\n", + "[flaml.automl: 08-21 02:52:09] {3133} INFO - iteration 1, current learner transformer\n", + "INFO:flaml.automl:iteration 1, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'eval_loss': 0.4843567907810211, 'eval_automl_metric': 0.18233944954128445, 'eval_runtime': 10.5457, 'eval_samples_per_second': 82.688, 'eval_steps_per_second': 82.688, 'epoch': 2.0}\n", + "{'eval_loss': 0.4618026912212372, 'eval_automl_metric': 0.17889908256880738, 'eval_runtime': 10.645, 'eval_samples_per_second': 81.916, 'eval_steps_per_second': 81.916, 'epoch': 3.0}\n", + "{'train_runtime': 65.5885, 'train_samples_per_second': 457.397, 'train_steps_per_second': 7.181, 'train_loss': 0.5575582905180135, 'epoch': 3.0}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_02-52-09/train_402e7e16_16_s=9223372036854775807,e=9.7119e-06,s=-1,s=3,e=64,d=14_2022-08-21_02-52-09/checkpoint-471/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_02-52-09/train_402e7e16_16_s=9223372036854775807,e=9.7119e-06,s=-1,s=3,e=64,d=14_2022-08-21_02-52-09/checkpoint-471/vocab.txt\n", + "loading file data/output/train_2022-08-21_02-52-09/train_402e7e16_16_s=9223372036854775807,e=9.7119e-06,s=-1,s=3,e=64,d=14_2022-08-21_02-52-09/checkpoint-471/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_02-52-09/train_402e7e16_16_s=9223372036854775807,e=9.7119e-06,s=-1,s=3,e=64,d=14_2022-08-21_02-52-09/checkpoint-471/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_02-52-09/train_402e7e16_16_s=9223372036854775807,e=9.7119e-06,s=-1,s=3,e=64,d=14_2022-08-21_02-52-09/checkpoint-471/tokenizer_config.json\n", + "[flaml.automl: 08-21 02:53:35] {3319} INFO - at 188.2s,\testimator transformer's best error=0.1640,\tbest estimator transformer's best error=0.1640\n", + "INFO:flaml.automl: at 188.2s,\testimator transformer's best error=0.1640,\tbest estimator transformer's best error=0.1640\n", + "[flaml.automl: 08-21 02:53:36] {3133} INFO - iteration 2, current learner transformer\n", + "INFO:flaml.automl:iteration 2, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.5778, 'learning_rate': 7.550901222797876e-06, 'epoch': 0.8}\n", + "{'loss': 0.3836, 'learning_rate': 4.805118959962285e-06, 'epoch': 1.6}\n", + "{'eval_loss': 0.3749224543571472, 'eval_automl_metric': 0.15596330275229353, 'eval_runtime': 10.5464, 'eval_samples_per_second': 82.682, 'eval_steps_per_second': 82.682, 'epoch': 2.0}\n", + "{'loss': 0.3399, 'learning_rate': 2.0593366971266936e-06, 'epoch': 2.4}\n", + "{'eval_loss': 0.37013810873031616, 'eval_automl_metric': 0.1513761467889908, 'eval_runtime': 10.6222, 'eval_samples_per_second': 82.092, 'eval_steps_per_second': 82.092, 'epoch': 3.0}\n", + "{'train_runtime': 126.516, 'train_samples_per_second': 237.124, 'train_steps_per_second': 14.82, 'train_loss': 0.40950755208333334, 'epoch': 3.0}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_02-53-36/train_7382565c_17_s=9223372036854775807,e=1.0297e-05,s=-1,s=3,e=16,d=26_2022-08-21_02-53-36/checkpoint-1875/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_02-53-36/train_7382565c_17_s=9223372036854775807,e=1.0297e-05,s=-1,s=3,e=16,d=26_2022-08-21_02-53-36/checkpoint-1875/vocab.txt\n", + "loading file data/output/train_2022-08-21_02-53-36/train_7382565c_17_s=9223372036854775807,e=1.0297e-05,s=-1,s=3,e=16,d=26_2022-08-21_02-53-36/checkpoint-1875/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_02-53-36/train_7382565c_17_s=9223372036854775807,e=1.0297e-05,s=-1,s=3,e=16,d=26_2022-08-21_02-53-36/checkpoint-1875/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_02-53-36/train_7382565c_17_s=9223372036854775807,e=1.0297e-05,s=-1,s=3,e=16,d=26_2022-08-21_02-53-36/checkpoint-1875/tokenizer_config.json\n", + "[flaml.automl: 08-21 02:56:03] {729} WARNING - checkpoint data/output/train_2022-08-21_02-50-27/train_036eed6c_15_s=9223372036854775807,e=1e-05,s=-1,s=3,e=32,d=20_2022-08-21_02-50-27/checkpoint-939 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-21_02-50-27/train_036eed6c_15_s=9223372036854775807,e=1e-05,s=-1,s=3,e=32,d=20_2022-08-21_02-50-27/checkpoint-939 not found\n", + "[flaml.automl: 08-21 02:56:03] {3319} INFO - at 335.5s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "INFO:flaml.automl: at 335.5s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "[flaml.automl: 08-21 02:56:03] {3133} INFO - iteration 3, current learner transformer\n", + "INFO:flaml.automl:iteration 3, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.5362, 'learning_rate': 8.879996750213199e-06, 'epoch': 0.8}\n", + "{'eval_loss': 0.3863365948200226, 'eval_automl_metric': 0.1594036697247706, 'eval_runtime': 10.6255, 'eval_samples_per_second': 82.067, 'eval_steps_per_second': 82.067, 'epoch': 1.0}\n", + "{'loss': 0.3654, 'learning_rate': 2.959998916737733e-06, 'epoch': 1.6}\n", + "{'eval_loss': 0.375693142414093, 'eval_automl_metric': 0.15596330275229353, 'eval_runtime': 10.6464, 'eval_samples_per_second': 81.906, 'eval_steps_per_second': 81.906, 'epoch': 2.0}\n", + "{'train_runtime': 91.5445, 'train_samples_per_second': 218.473, 'train_steps_per_second': 13.655, 'train_loss': 0.42628193359375, 'epoch': 2.0}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_02-56-03/train_cb570bac_18_s=9223372036854775807,e=1.48e-05,s=-1,s=2,e=16,d=25_2022-08-21_02-56-03/checkpoint-1250/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_02-56-03/train_cb570bac_18_s=9223372036854775807,e=1.48e-05,s=-1,s=2,e=16,d=25_2022-08-21_02-56-03/checkpoint-1250/vocab.txt\n", + "loading file data/output/train_2022-08-21_02-56-03/train_cb570bac_18_s=9223372036854775807,e=1.48e-05,s=-1,s=2,e=16,d=25_2022-08-21_02-56-03/checkpoint-1250/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_02-56-03/train_cb570bac_18_s=9223372036854775807,e=1.48e-05,s=-1,s=2,e=16,d=25_2022-08-21_02-56-03/checkpoint-1250/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_02-56-03/train_cb570bac_18_s=9223372036854775807,e=1.48e-05,s=-1,s=2,e=16,d=25_2022-08-21_02-56-03/checkpoint-1250/tokenizer_config.json\n", + "[flaml.automl: 08-21 02:57:55] {3319} INFO - at 447.3s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "INFO:flaml.automl: at 447.3s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "[flaml.automl: 08-21 02:57:55] {3133} INFO - iteration 4, current learner transformer\n", + "INFO:flaml.automl:iteration 4, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.6402, 'learning_rate': 5.730904302906456e-06, 'epoch': 0.8}\n", + "{'loss': 0.4537, 'learning_rate': 4.298178227179842e-06, 'epoch': 1.6}\n", + "{'loss': 0.3716, 'learning_rate': 2.865452151453228e-06, 'epoch': 2.4}\n", + "{'eval_loss': 0.4031089246273041, 'eval_automl_metric': 0.16284403669724767, 'eval_runtime': 10.6207, 'eval_samples_per_second': 82.104, 'eval_steps_per_second': 82.104, 'epoch': 2.88}\n", + "{'eval_loss': 0.4031089246273041, 'eval_automl_metric': 0.16284403669724767, 'eval_runtime': 10.663, 'eval_samples_per_second': 81.778, 'eval_steps_per_second': 81.778, 'epoch': 2.88}\n", + "{'train_runtime': 122.6118, 'train_samples_per_second': 326.233, 'train_steps_per_second': 20.39, 'train_loss': 0.46601707301930595, 'epoch': 2.88}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_02-57-55/train_0df42e0e_19_s=9223372036854775807,e=7.1636e-06,s=-1,s=4,e=16,d=27_2022-08-21_02-57-55/checkpoint-1803/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_02-57-55/train_0df42e0e_19_s=9223372036854775807,e=7.1636e-06,s=-1,s=4,e=16,d=27_2022-08-21_02-57-55/checkpoint-1803/vocab.txt\n", + "loading file data/output/train_2022-08-21_02-57-55/train_0df42e0e_19_s=9223372036854775807,e=7.1636e-06,s=-1,s=4,e=16,d=27_2022-08-21_02-57-55/checkpoint-1803/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_02-57-55/train_0df42e0e_19_s=9223372036854775807,e=7.1636e-06,s=-1,s=4,e=16,d=27_2022-08-21_02-57-55/checkpoint-1803/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_02-57-55/train_0df42e0e_19_s=9223372036854775807,e=7.1636e-06,s=-1,s=4,e=16,d=27_2022-08-21_02-57-55/checkpoint-1803/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:00:18] {3319} INFO - at 590.4s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "INFO:flaml.automl: at 590.4s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "[flaml.automl: 08-21 03:00:18] {3133} INFO - iteration 5, current learner transformer\n", + "INFO:flaml.automl:iteration 5, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.5223, 'learning_rate': 1.3121346786922505e-05, 'epoch': 0.8}\n", + "{'loss': 0.333, 'learning_rate': 8.349947955314322e-06, 'epoch': 1.6}\n", + "{'eval_loss': 0.37441486120224, 'eval_automl_metric': 0.16169724770642202, 'eval_runtime': 10.5419, 'eval_samples_per_second': 82.717, 'eval_steps_per_second': 82.717, 'epoch': 2.0}\n", + "{'eval_loss': 0.3761043846607208, 'eval_automl_metric': 0.15481651376146788, 'eval_runtime': 10.512, 'eval_samples_per_second': 82.953, 'eval_steps_per_second': 82.953, 'epoch': 2.23}\n", + "{'eval_loss': 0.3761043846607208, 'eval_automl_metric': 0.15481651376146788, 'eval_runtime': 10.5934, 'eval_samples_per_second': 82.316, 'eval_steps_per_second': 82.316, 'epoch': 2.23}\n", + "{'train_runtime': 111.7637, 'train_samples_per_second': 268.423, 'train_steps_per_second': 16.776, 'train_loss': 0.39087216824957316, 'epoch': 2.23}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-00-18/train_634289e6_20_s=9223372036854775807,e=1.7893e-05,s=-1,s=3,e=16,d=32_2022-08-21_03-00-18/checkpoint-1391/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-00-18/train_634289e6_20_s=9223372036854775807,e=1.7893e-05,s=-1,s=3,e=16,d=32_2022-08-21_03-00-18/checkpoint-1391/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-00-18/train_634289e6_20_s=9223372036854775807,e=1.7893e-05,s=-1,s=3,e=16,d=32_2022-08-21_03-00-18/checkpoint-1391/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-00-18/train_634289e6_20_s=9223372036854775807,e=1.7893e-05,s=-1,s=3,e=16,d=32_2022-08-21_03-00-18/checkpoint-1391/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-00-18/train_634289e6_20_s=9223372036854775807,e=1.7893e-05,s=-1,s=3,e=16,d=32_2022-08-21_03-00-18/checkpoint-1391/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:02:30] {3319} INFO - at 722.7s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "INFO:flaml.automl: at 722.7s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "[flaml.automl: 08-21 03:02:30] {3133} INFO - iteration 6, current learner transformer\n", + "INFO:flaml.automl:iteration 6, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.6593, 'learning_rate': 4.3452939856201385e-06, 'epoch': 0.8}\n", + "{'loss': 0.5039, 'learning_rate': 2.76518708175827e-06, 'epoch': 1.6}\n", + "{'eval_loss': 0.4441715180873871, 'eval_automl_metric': 0.18463302752293576, 'eval_runtime': 10.6465, 'eval_samples_per_second': 81.905, 'eval_steps_per_second': 81.905, 'epoch': 2.0}\n", + "{'eval_loss': 0.444117933511734, 'eval_automl_metric': 0.18463302752293576, 'eval_runtime': 10.6581, 'eval_samples_per_second': 81.815, 'eval_steps_per_second': 81.815, 'epoch': 2.0}\n", + "{'eval_loss': 0.444117933511734, 'eval_automl_metric': 0.18463302752293576, 'eval_runtime': 10.659, 'eval_samples_per_second': 81.809, 'eval_steps_per_second': 81.809, 'epoch': 2.0}\n", + "{'train_runtime': 103.8522, 'train_samples_per_second': 288.872, 'train_steps_per_second': 18.055, 'train_loss': 0.5531763736959651, 'epoch': 2.0}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-02-30/train_b2206e84_21_s=9223372036854775807,e=5.9254e-06,s=-1,s=3,e=16,d=20_2022-08-21_03-02-30/checkpoint-1250/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-02-30/train_b2206e84_21_s=9223372036854775807,e=5.9254e-06,s=-1,s=3,e=16,d=20_2022-08-21_03-02-30/checkpoint-1250/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-02-30/train_b2206e84_21_s=9223372036854775807,e=5.9254e-06,s=-1,s=3,e=16,d=20_2022-08-21_03-02-30/checkpoint-1250/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-02-30/train_b2206e84_21_s=9223372036854775807,e=5.9254e-06,s=-1,s=3,e=16,d=20_2022-08-21_03-02-30/checkpoint-1250/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-02-30/train_b2206e84_21_s=9223372036854775807,e=5.9254e-06,s=-1,s=3,e=16,d=20_2022-08-21_03-02-30/checkpoint-1250/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:04:34] {3319} INFO - at 847.0s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "INFO:flaml.automl: at 847.0s,\testimator transformer's best error=0.1514,\tbest estimator transformer's best error=0.1514\n", + "[flaml.automl: 08-21 03:04:34] {3133} INFO - iteration 7, current learner transformer\n", + "INFO:flaml.automl:iteration 7, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.4949, 'learning_rate': 1.624682269684853e-05, 'epoch': 0.8}\n", + "{'loss': 0.3234, 'learning_rate': 1.0338887170721792e-05, 'epoch': 1.6}\n", + "{'eval_loss': 0.34439605474472046, 'eval_automl_metric': 0.13188073394495414, 'eval_runtime': 10.6017, 'eval_samples_per_second': 82.251, 'eval_steps_per_second': 82.251, 'epoch': 2.0}\n", + "{'eval_loss': 0.3457942605018616, 'eval_automl_metric': 0.13188073394495414, 'eval_runtime': 10.539, 'eval_samples_per_second': 82.74, 'eval_steps_per_second': 82.74, 'epoch': 2.0}\n", + "{'eval_loss': 0.3457942605018616, 'eval_automl_metric': 0.13188073394495414, 'eval_runtime': 10.661, 'eval_samples_per_second': 81.794, 'eval_steps_per_second': 81.794, 'epoch': 2.0}\n", + "{'train_runtime': 102.8292, 'train_samples_per_second': 291.746, 'train_steps_per_second': 18.234, 'train_loss': 0.39010055993291304, 'epoch': 2.0}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-04-34/train_fc3698e0_22_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-04-34/checkpoint-1250/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-04-34/train_fc3698e0_22_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-04-34/checkpoint-1250/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-04-34/train_fc3698e0_22_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-04-34/checkpoint-1250/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-04-34/train_fc3698e0_22_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-04-34/checkpoint-1250/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-04-34/train_fc3698e0_22_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-04-34/checkpoint-1250/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:06:38] {729} WARNING - checkpoint data/output/train_2022-08-21_02-53-36/train_7382565c_17_s=9223372036854775807,e=1.0297e-05,s=-1,s=3,e=16,d=26_2022-08-21_02-53-36/checkpoint-1875 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-21_02-53-36/train_7382565c_17_s=9223372036854775807,e=1.0297e-05,s=-1,s=3,e=16,d=26_2022-08-21_02-53-36/checkpoint-1875 not found\n", + "[flaml.automl: 08-21 03:06:38] {3319} INFO - at 971.1s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "INFO:flaml.automl: at 971.1s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "[flaml.automl: 08-21 03:06:38] {3133} INFO - iteration 8, current learner transformer\n", + "INFO:flaml.automl:iteration 8, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.5516, 'learning_rate': 2.1517120038185796e-05, 'epoch': 0.4}\n", + "{'loss': 0.3741, 'learning_rate': 1.8206793878464904e-05, 'epoch': 0.8}\n", + "{'eval_loss': 0.4729900658130646, 'eval_automl_metric': 0.1674311926605505, 'eval_runtime': 10.6305, 'eval_samples_per_second': 82.028, 'eval_steps_per_second': 82.028, 'epoch': 0.92}\n", + "{'eval_loss': 0.4729900658130646, 'eval_automl_metric': 0.1674311926605505, 'eval_runtime': 10.6789, 'eval_samples_per_second': 81.656, 'eval_steps_per_second': 81.656, 'epoch': 0.92}\n", + "{'train_runtime': 83.7502, 'train_samples_per_second': 358.208, 'train_steps_per_second': 44.776, 'train_loss': 0.450164735005164, 'epoch': 0.92}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-06-38/train_462df83a_23_s=9223372036854775807,e=2.4827e-05,s=-1,s=3,e=8,d=24_2022-08-21_03-06-38/checkpoint-1146/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-06-38/train_462df83a_23_s=9223372036854775807,e=2.4827e-05,s=-1,s=3,e=8,d=24_2022-08-21_03-06-38/checkpoint-1146/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-06-38/train_462df83a_23_s=9223372036854775807,e=2.4827e-05,s=-1,s=3,e=8,d=24_2022-08-21_03-06-38/checkpoint-1146/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-06-38/train_462df83a_23_s=9223372036854775807,e=2.4827e-05,s=-1,s=3,e=8,d=24_2022-08-21_03-06-38/checkpoint-1146/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-06-38/train_462df83a_23_s=9223372036854775807,e=2.4827e-05,s=-1,s=3,e=8,d=24_2022-08-21_03-06-38/checkpoint-1146/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:08:23] {3319} INFO - at 1076.1s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "INFO:flaml.automl: at 1076.1s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "[flaml.automl: 08-21 03:08:23] {3133} INFO - iteration 9, current learner transformer\n", + "INFO:flaml.automl:iteration 9, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.4588, 'learning_rate': 9.24274365870653e-06, 'epoch': 1.6}\n", + "{'eval_loss': 0.35305526852607727, 'eval_automl_metric': 0.14220183486238536, 'eval_runtime': 10.7298, 'eval_samples_per_second': 81.269, 'eval_steps_per_second': 81.269, 'epoch': 2.0}\n", + "{'eval_loss': 0.3390190303325653, 'eval_automl_metric': 0.13876146788990829, 'eval_runtime': 10.6766, 'eval_samples_per_second': 81.674, 'eval_steps_per_second': 81.674, 'epoch': 2.22}\n", + "{'eval_loss': 0.3390190303325653, 'eval_automl_metric': 0.13876146788990829, 'eval_runtime': 10.7321, 'eval_samples_per_second': 81.252, 'eval_steps_per_second': 81.252, 'epoch': 2.22}\n", + "{'train_runtime': 76.0261, 'train_samples_per_second': 394.602, 'train_steps_per_second': 12.351, 'train_loss': 0.42033653918879177, 'epoch': 2.22}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-08-23/train_84bb5b06_24_s=9223372036854775807,e=1.977e-05,s=-1,s=3,e=32,d=24_2022-08-21_03-08-23/checkpoint-694/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-08-23/train_84bb5b06_24_s=9223372036854775807,e=1.977e-05,s=-1,s=3,e=32,d=24_2022-08-21_03-08-23/checkpoint-694/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-08-23/train_84bb5b06_24_s=9223372036854775807,e=1.977e-05,s=-1,s=3,e=32,d=24_2022-08-21_03-08-23/checkpoint-694/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-08-23/train_84bb5b06_24_s=9223372036854775807,e=1.977e-05,s=-1,s=3,e=32,d=24_2022-08-21_03-08-23/checkpoint-694/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-08-23/train_84bb5b06_24_s=9223372036854775807,e=1.977e-05,s=-1,s=3,e=32,d=24_2022-08-21_03-08-23/checkpoint-694/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:10:00] {3319} INFO - at 1172.6s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "INFO:flaml.automl: at 1172.6s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "[flaml.automl: 08-21 03:10:00] {3133} INFO - iteration 10, current learner transformer\n", + "INFO:flaml.automl:iteration 10, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.5476, 'learning_rate': 1.0987792912546241e-05, 'epoch': 0.8}\n", + "{'eval_loss': 0.41232776641845703, 'eval_automl_metric': 0.1594036697247706, 'eval_runtime': 10.5607, 'eval_samples_per_second': 82.57, 'eval_steps_per_second': 82.57, 'epoch': 1.35}\n", + "{'eval_loss': 0.41232776641845703, 'eval_automl_metric': 0.1594036697247706, 'eval_runtime': 10.5165, 'eval_samples_per_second': 82.918, 'eval_steps_per_second': 82.918, 'epoch': 1.35}\n", + "{'train_runtime': 68.5081, 'train_samples_per_second': 437.905, 'train_steps_per_second': 27.369, 'train_loss': 0.47965226870796485, 'epoch': 1.35}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-10-00/train_be42d944_25_s=9223372036854775807,e=1.4983e-05,s=-1,s=3,e=16,d=18_2022-08-21_03-10-00/checkpoint-841/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-10-00/train_be42d944_25_s=9223372036854775807,e=1.4983e-05,s=-1,s=3,e=16,d=18_2022-08-21_03-10-00/checkpoint-841/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-10-00/train_be42d944_25_s=9223372036854775807,e=1.4983e-05,s=-1,s=3,e=16,d=18_2022-08-21_03-10-00/checkpoint-841/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-10-00/train_be42d944_25_s=9223372036854775807,e=1.4983e-05,s=-1,s=3,e=16,d=18_2022-08-21_03-10-00/checkpoint-841/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-10-00/train_be42d944_25_s=9223372036854775807,e=1.4983e-05,s=-1,s=3,e=16,d=18_2022-08-21_03-10-00/checkpoint-841/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:11:29] {3319} INFO - at 1262.0s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "INFO:flaml.automl: at 1262.0s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "[flaml.automl: 08-21 03:11:29] {3133} INFO - iteration 11, current learner transformer\n", + "INFO:flaml.automl:iteration 11, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.4679, 'learning_rate': 2.402295436797273e-05, 'epoch': 0.8}\n", + "{'eval_loss': 0.3937930166721344, 'eval_automl_metric': 0.1513761467889908, 'eval_runtime': 10.5403, 'eval_samples_per_second': 82.73, 'eval_steps_per_second': 82.73, 'epoch': 1.14}\n", + "{'eval_loss': 0.3937930166721344, 'eval_automl_metric': 0.1513761467889908, 'eval_runtime': 10.5525, 'eval_samples_per_second': 82.634, 'eval_steps_per_second': 82.634, 'epoch': 1.14}\n", + "{'train_runtime': 61.8987, 'train_samples_per_second': 484.663, 'train_steps_per_second': 30.291, 'train_loss': 0.4275143780285799, 'epoch': 1.14}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-11-29/train_f38a7ed6_26_s=9223372036854775807,e=3.2759e-05,s=-1,s=3,e=16,d=30_2022-08-21_03-11-29/checkpoint-711/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-11-29/train_f38a7ed6_26_s=9223372036854775807,e=3.2759e-05,s=-1,s=3,e=16,d=30_2022-08-21_03-11-29/checkpoint-711/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-11-29/train_f38a7ed6_26_s=9223372036854775807,e=3.2759e-05,s=-1,s=3,e=16,d=30_2022-08-21_03-11-29/checkpoint-711/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-11-29/train_f38a7ed6_26_s=9223372036854775807,e=3.2759e-05,s=-1,s=3,e=16,d=30_2022-08-21_03-11-29/checkpoint-711/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-11-29/train_f38a7ed6_26_s=9223372036854775807,e=3.2759e-05,s=-1,s=3,e=16,d=30_2022-08-21_03-11-29/checkpoint-711/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:12:52] {3319} INFO - at 1344.3s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "INFO:flaml.automl: at 1344.3s,\testimator transformer's best error=0.1319,\tbest estimator transformer's best error=0.1319\n", + "[flaml.automl: 08-21 03:12:52] {3133} INFO - iteration 12, current learner transformer\n", + "INFO:flaml.automl:iteration 12, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'loss': 0.4939, 'learning_rate': 2.1277689409714175e-05, 'epoch': 0.12}\n", + "{'loss': 0.3558, 'learning_rate': 2.040062059645308e-05, 'epoch': 0.24}\n", + "{'loss': 0.3081, 'learning_rate': 1.9523551783191982e-05, 'epoch': 0.36}\n", + "{'loss': 0.2893, 'learning_rate': 1.8646482969930888e-05, 'epoch': 0.48}\n", + "{'loss': 0.2667, 'learning_rate': 1.776941415666979e-05, 'epoch': 0.59}\n", + "{'loss': 0.2576, 'learning_rate': 1.6892345343408696e-05, 'epoch': 0.71}\n", + "{'loss': 0.2435, 'learning_rate': 1.60152765301476e-05, 'epoch': 0.83}\n", + "{'loss': 0.2409, 'learning_rate': 1.5138207716886507e-05, 'epoch': 0.95}\n", + "{'loss': 0.2148, 'learning_rate': 1.4261138903625411e-05, 'epoch': 1.07}\n", + "{'loss': 0.2032, 'learning_rate': 1.3384070090364317e-05, 'epoch': 1.19}\n", + "{'loss': 0.1991, 'learning_rate': 1.2507001277103219e-05, 'epoch': 1.31}\n", + "{'loss': 0.2109, 'learning_rate': 1.1629932463842124e-05, 'epoch': 1.43}\n", + "{'loss': 0.1921, 'learning_rate': 1.0752863650581028e-05, 'epoch': 1.54}\n", + "{'loss': 0.1924, 'learning_rate': 9.875794837319934e-06, 'epoch': 1.66}\n", + "{'loss': 0.1903, 'learning_rate': 8.99872602405884e-06, 'epoch': 1.78}\n", + "{'loss': 0.1865, 'learning_rate': 8.121657210797743e-06, 'epoch': 1.9}\n", + "{'eval_loss': 0.317385196685791, 'eval_automl_metric': 0.08944954128440363, 'eval_runtime': 10.4635, 'eval_samples_per_second': 83.338, 'eval_steps_per_second': 83.338, 'epoch': 1.93}\n", + "{'eval_loss': 0.317385196685791, 'eval_automl_metric': 0.08944954128440363, 'eval_runtime': 10.5911, 'eval_samples_per_second': 82.333, 'eval_steps_per_second': 82.333, 'epoch': 1.93}\n", + "{'train_runtime': 477.591, 'train_samples_per_second': 423.054, 'train_steps_per_second': 26.445, 'train_loss': 0.2517916716532359, 'epoch': 1.93}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/tokenizer_config.json\n", + "[flaml.automl: 08-21 03:21:35] {729} WARNING - checkpoint data/output/train_2022-08-21_03-04-34/train_fc3698e0_22_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-04-34/checkpoint-1250 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-21_03-04-34/train_fc3698e0_22_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-04-34/checkpoint-1250 not found\n", + "[flaml.automl: 08-21 03:21:35] {3319} INFO - at 1867.4s,\testimator transformer's best error=0.0894,\tbest estimator transformer's best error=0.0894\n", + "INFO:flaml.automl: at 1867.4s,\testimator transformer's best error=0.0894,\tbest estimator transformer's best error=0.0894\n", + "[flaml.automl: 08-21 03:21:35] {3434} INFO - selected model: None\n", + "INFO:flaml.automl:selected model: None\n", + "[flaml.automl: 08-21 03:21:35] {2862} INFO - fit succeeded\n", + "INFO:flaml.automl:fit succeeded\n", + "[flaml.automl: 08-21 03:21:35] {2864} INFO - Time taken to find the best model: 1867.4163627624512\n", + "INFO:flaml.automl:Time taken to find the best model: 1867.4163627624512\n", + "[flaml.automl: 08-21 03:21:35] {2878} WARNING - Time taken to find the best model is 104% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n", + "WARNING:flaml.automl:Time taken to find the best model is 104% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "'''The main flaml automl API'''\n", + "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m {'eval_loss': 0.6315866112709045, 'eval_automl_metric': 0.18779999999999997, 'eval_runtime': 15.4883, 'eval_samples_per_second': 645.648, 'eval_steps_per_second': 40.353, 'epoch': 1.66}\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m {'train_runtime': 190.7625, 'train_samples_per_second': 87.254, 'train_steps_per_second': 10.909, 'train_loss': 0.5091343906738046, 'epoch': 1.66}\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m {'eval_loss': 1.2118068933486938, 'eval_automl_metric': 0.2015, 'eval_runtime': 15.2585, 'eval_samples_per_second': 655.374, 'eval_steps_per_second': 40.961, 'epoch': 2.87}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: ending3, ending1, video-id, sent1, ending0, sent2, fold-ind, ending2, startphrase, gold-source. If ending3, ending1, video-id, sent1, ending0, sent2, fold-ind, ending2, startphrase, gold-source are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m Num examples = 10000\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m Batch size = 16\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m {'eval_loss': 1.2118068933486938, 'eval_automl_metric': 0.2015, 'eval_runtime': 15.1369, 'eval_samples_per_second': 660.639, 'eval_steps_per_second': 41.29, 'epoch': 2.87}\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m {'train_runtime': 546.3809, 'train_samples_per_second': 156.658, 'train_steps_per_second': 39.165, 'train_loss': 0.5030154804349909, 'epoch': 2.87}\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'loss': 0.4854, 'learning_rate': 1.3592147782116173e-06, 'epoch': 2.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: fold-ind, sent2, gold-source, ending1, startphrase, sent1, ending0, video-id, ending2, ending3. If fold-ind, sent2, gold-source, ending1, startphrase, sent1, ending0, video-id, ending2, ending3 are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m Num examples = 10000\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m Batch size = 16\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'eval_loss': 0.49709731340408325, 'eval_automl_metric': 0.17600000000000005, 'eval_runtime': 15.4983, 'eval_samples_per_second': 645.232, 'eval_steps_per_second': 40.327, 'epoch': 2.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:41:56,719\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'eval_loss': 0.5254333019256592, 'eval_automl_metric': 0.17800000000000005, 'eval_runtime': 15.45, 'eval_samples_per_second': 647.251, 'eval_steps_per_second': 40.453, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'loss': 0.3989, 'learning_rate': 3.8051750127352887e-07, 'epoch': 3.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:42:56,729\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'eval_loss': 0.5254867076873779, 'eval_automl_metric': 0.17789999999999995, 'eval_runtime': 15.424, 'eval_samples_per_second': 648.341, 'eval_steps_per_second': 40.521, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'eval_loss': 0.5332269072532654, 'eval_automl_metric': 0.17830000000000001, 'eval_runtime': 15.4452, 'eval_samples_per_second': 647.45, 'eval_steps_per_second': 40.466, 'epoch': 3.39}\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'train_runtime': 382.2827, 'train_samples_per_second': 88.597, 'train_steps_per_second': 11.076, 'train_loss': 0.5299136270370808, 'epoch': 3.39}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:43:56,739\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: ending2, sent1, ending0, sent2, ending3, video-id, gold-source, ending1, startphrase, fold-ind. If ending2, sent1, ending0, sent2, ending3, video-id, gold-source, ending1, startphrase, fold-ind are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m Num examples = 10000\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m Batch size = 16\n", - "2022-03-19 14:44:14,271\tINFO tune.py:639 -- Total run time: 795.18 seconds (504.18 seconds for the tuning loop).\n", - "[flaml.automl: 03-19 14:44:19] {2837} INFO - selected model: None\n", - "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 0.6603, 'learning_rate': 4.631567529441369e-06, 'epoch': 1.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[flaml.automl: 03-19 14:46:08] {2947} INFO - retrain transformer for 109.2s\n", - "[flaml.automl: 03-19 14:46:08] {2954} INFO - retrained model: None\n", - "[flaml.automl: 03-19 14:46:08] {2283} INFO - fit succeeded\n", - "[flaml.automl: 03-19 14:46:08] {2284} INFO - Time taken to find the best model: 319.927033662796\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'train_runtime': 96.899, 'train_samples_per_second': 245.031, 'train_steps_per_second': 30.63, 'train_loss': 0.6602518278346073, 'epoch': 1.0}\n" - ] - } - ], - "source": [ - "''' import AutoML class from flaml package '''\n", - "from flaml import AutoML\n", - "automl = AutoML()\n", - "\n", - "if not ray.is_initialized():\n", - " ray.init()\n", - "\n", - "automl_settings = {\n", - " \"time_budget\": 500, # setting the time budget\n", - " \"task\": \"multichoice-classification\", # setting the task as multiplechoice-classification\n", - " \"fit_kwargs_by_estimator\": { # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base\n", - " \"transformer\": {\n", - " \"output_dir\": \"data/output/\", # setting the output directory\n", - " \"per_device_eval_batch_size\": 16, # the batch size for validation (inference)\n", - " }\n", - " },\n", - " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n", - " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n", - " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n", - " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n", - " \"n_concurrent_trials\": 4\n", - "}\n", - "\n", - "'''The main flaml automl API'''\n", - "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.00021956991427751982, 'num_train_epochs': 0.3549576494055084, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.07425273520338253, 'weight_decay': 0.03879221030529465, 'adam_epsilon': 3.7880482987985576e-08, 'seed': 43, 'global_max_steps': 444, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00021956991427751982, 'num_train_epochs': 0.3549576494055084, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.07425273520338253, 'weight_decay': 0.03879221030529465, 'adam_epsilon': 3.7880482987985576e-08, 'seed': 43, 'global_max_steps': 444, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.3241899893349513e-06, 'num_train_epochs': 0.4379128434860086, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.257055208282222, 'weight_decay': 0.012652183020312091, 'adam_epsilon': 1.0189125195705357e-07, 'seed': 43, 'global_max_steps': 274, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.0002562922748967212, 'num_train_epochs': 0.1802995999606059, 'per_device_train_batch_size': 4, 'warmup_ratio': 0.1809477882684876, 'weight_decay': 0.10305626005953175, 'adam_epsilon': 5.536776887412208e-08, 'seed': 42, 'global_max_steps': 451, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 6.104513714676502e-06, 'num_train_epochs': 2.3743291981165893, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.23610846764298543, 'weight_decay': 0.20205904544254147, 'adam_epsilon': 5.752964074991208e-08, 'seed': 41, 'global_max_steps': 1251, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 6.104513714676502e-06, 'num_train_epochs': 2.3743291981165893, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.23610846764298543, 'weight_decay': 0.20205904544254147, 'adam_epsilon': 5.752964074991208e-08, 'seed': 41, 'global_max_steps': 1251, 'learner': 'transformer'}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.306519250357542e-06, 'num_train_epochs': 0.4664878701006166, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 5.931759315303309e-07, 'seed': 43, 'global_max_steps': 147, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 6.104513714676502e-06, 'num_train_epochs': 2.3743291981165893, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.23610846764298543, 'weight_decay': 0.20205904544254147, 'adam_epsilon': 5.752964074991208e-08, 'seed': 41, 'global_max_steps': 1251, 'learner': 'transformer'}}\n", - "6\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAd8klEQVR4nO3df9wVZZ3/8dfbWxQqFY27VhGEEinNEiN/9GMzy8XcFDIzbLc126S2NMuiZEtjbd0yV3vUY9l8oF/zxzdDZc3IKHLTrBQF/IlgGCkqaIo/UFIUwc/+MdfB8XjuwwBnzrnve97Px+M8mLnmOjOfM8x9Pmeua+YaRQRmZlZdW3U6ADMz6ywnAjOzinMiMDOrOCcCM7OKcyIwM6s4JwIzs4pzIjBrQtJ7JC3pdBxmZXIisF5L0jJJH+hkDBHx+4gYXdb6JY2T9DtJqyWtlHS9pCPK2p5ZI04EVmmSujq47aOAK4CLgV2B1wOnAYdvxrokyX/Ptll84FifI2krSadI+rOkxyVdLmmn3PIrJP1F0lPp1/ZeuWUXSvqhpNmSngHel848viLpzvSeyyQNTPUPkrQ89/4e66blX5X0sKSHJH1aUkjavcFnEHAO8K2IOD8inoqIFyPi+og4PtWZKun/594zIq1v6zT/W0lnSLoBeBaYLGlB3Xa+JGlWmt5W0n9KekDSI5LOlTRoC/87rB9wIrC+6ERgAvBeYBfgSWBabvkvgVHA64BbgR/Xvf/jwBnAdsAfUtnRwKHASOCtwCebbL9hXUmHAicDHwB2Bw5qso7RwDBgZpM6RXwCmET2Wc4FRksalVv+ceDSNP0dYA9gnxTfULIzEKs4JwLriz4LfD0ilkfE88BU4KjaL+WIuCAiVueWvU3SDrn3/ywibki/wJ9LZT+IiIci4gng52Rflj3pqe7RwI8iYlFEPJu23ZPXpn8fLvaRe3Rh2t66iHgK+BlwDEBKCG8CZqUzkEnAlyLiiYhYDfwHMHELt2/9gBOB9UW7AT+VtErSKuBuYD3wekldkr6Tmo2eBpal9wzJvf/BBuv8S276WeA1TbbfU91d6tbdaDs1j6d/d25Sp4j6bVxKSgRkZwNXpaTUDbwKuCW3336Vyq3inAisL3oQ+GBEDM69BkbECrIvv/FkzTM7ACPSe5R7f1lD7j5M1ulbM6xJ3SVkn+MjTeo8Q/blXfM3DerUf5ZrgG5J+5AlhFqz0GPAGmCv3D7bISKaJTyrCCcC6+0GSBqYe21N1hZ+hqTdACR1Sxqf6m8HPE/2i/tVZM0f7XI5cJykN0t6FXBqTxUjG//9ZOBUScdJ2j51gr9b0vRU7XbgbyUNT01bUzYWQES8QHYl0lnATmSJgYh4ETgP+J6k1wFIGipp3OZ+WOs/nAist5tN9ku29poKfB+YBfxa0mrgJmD/VP9i4H5gBbA4LWuLiPgl8APgOmBpbtvP91B/JvAx4FPAQ8AjwL+TtfMTEdcAlwF3ArcAVxcM5VKyM6IrImJdrvxrtbhSs9n/knVaW8XJD6YxK4ekNwN3AdvWfSGb9So+IzBrIUkfTtfr7wicCfzcScB6OycCs9b6DPAo8GeyK5n+pbPhmG2cm4bMzCrOZwRmZhW3dacD2FRDhgyJESNGdDoMM7M+5ZZbbnksIhreQNjnEsGIESNYsGDBxiuamdkGku7vaZmbhszMKs6JwMys4pwIzMwqzonAzKzinAjMzCquz101ZGa2Oa66bQVnzVnCQ6vWsMvgQUweN5oJY4Z2OqxewYmgBXyAmfVuV922gilXLmTNC+sBWLFqDVOuXAjgv1WcCLaYDzCz3u+sOUs2/I3WrHlhPV+deSc/mfdAh6LadHvusj3fPHyvlq/XiWAL9ZcDzKw/W7FqTcPytetfbHMkvZMTwRZ6yAeYWa+3TddWDf8mhw4exGWfObADEW2aWvPzvPue4NeLHml587MTwRbaZfCghr82+soBZlYF9U24AIMGdDF5XO9/QFs7mp9LvXxU0qGSlkhaKumUBsuHS7pO0m2S7pR0WJnxlGHyuNEMGtD1srK+coCZVcWEMUP59pF7M3TwIET2Q+3bR+7dJ/rxemp+PmvOkpZto7QzAkldwDTgEGA5MF/SrIhYnKv2DeDyiPihpD3Jnk87oqyYylA7kL46807Wrn+Rob5qyKxXmjBmaJ/8u+yp+bmn8s1RZtPQfsDSiLgXQNIMYDzZA8VrAtg+Te9A9gDvPmfCmKEbOobdHGRmrdRT8/Mugwe1bBtlNg0NBR7MzS9PZXlTgX+UtJzsbODERiuSNEnSAkkLVq5cWUasZma9Ujuanzs9xMQxwIURsStwGHCJpFfEFBHTI2JsRIzt7m74XAUzs36pHf0bZTYNrQCG5eZ3TWV5/wwcChARcyUNBIaQPfzbzMwov3+jzDOC+cAoSSMlbQNMBGbV1XkAeD+ApDcDAwG3/ZiZtVFpiSAi1gEnAHOAu8muDlok6XRJR6RqXwaOl3QH8BPgkxERZcVkZmavVOoNZRExm6wTOF92Wm56MfCuMmMwazcPQmh9je8sNmshD0JofVGnrxoy61facReoWas5EZi1UDvuAjVrNScCsxbq6W7PVt4FatZqTgRmLeRBCK0vcmexWQvVOoR91ZD1JU4EZi3WV0e5tOpy05CZWcU5EZiZVZwTgZlZxTkRmJlVnBOBmVnFORGYmVWcE4GZWcU5EZiZVZwTgZlZxTkRmJlVnBOBmVnFORGYmVWcE4GZWcU5EZiZVZwTgZlZxTkRmJlVnBOBmVnFORGYmVWcE4GZWcWVmggkHSppiaSlkk5psPx7km5Pr3skrSozHjMze6XSHl4vqQuYBhwCLAfmS5oVEYtrdSLiS7n6JwJjyorHzMwaK/OMYD9gaUTcGxFrgRnA+Cb1jwF+UmI8ZmbWQJmJYCjwYG5+eSp7BUm7ASOBa3tYPknSAkkLVq5c2fJAzcyqrLd0Fk8EZkbE+kYLI2J6RIyNiLHd3d1tDs3MrH8rMxGsAIbl5ndNZY1MxM1CZmYdUWYimA+MkjRS0jZkX/az6itJehOwIzC3xFjMzKwHpSWCiFgHnADMAe4GLo+IRZJOl3RErupEYEZERFmxmJlZz0q7fBQgImYDs+vKTqubn1pmDGZm1lxv6Sw2M7MOcSIwM6s4JwIzs4pzIjAzq7iNJgJJr21HIGZm1hlFzghuknSFpMMkqfSIzMysrYokgj2A6cAngD9J+g9Je5QblpmZtctGE0FkromIY4DjgWOBeZKul3Rg6RGamVmpNnpDWeoj+EeyM4JHgBPJhorYB7iCbNRQMzPro4rcWTwXuASYEBHLc+ULJJ1bTlhmZtYuRRLB6J7GAYqIM1scj5mZtVmRzuJfSxpcm5G0o6Q55YVkZmbtVCQRdEfEqtpMRDwJvK60iMzMrK2KJIL1kobXZtJjJT1ktJlZP1Gkj+DrwB8kXQ8IeA8wqdSozMysbTaaCCLiV5L2BQ5IRV+MiMfKDcvMzNql6INp1gOPAgOBPSUREb8rLywzM2uXIjeUfRo4iezh87eTnRnMBQ4uNTIzM2uLIp3FJwHvAO6PiPcBY4BVZQZlZmbtUyQRPBcRzwFI2jYi/giMLjcsMzNrlyJ9BMvTDWVXAddIehK4v8ygzMysfYpcNfThNDlV0nXADsCvSo3KzMzapmkikNQFLIqINwFExPVticrMzNqmaR9BRKwHluTvLDYzs/6lSB/BjsAiSfOAZ2qFEXFEaVGZmVnbFEkEp27uyiUdCnwf6ALOj4jvNKhzNDCVbPyiOyLi45u7PTMz23RFOos3q18g9S9MAw4BlgPzJc2KiMW5OqOAKcC7IuJJSR7V1MyszTZ6H4Gk1ZKeTq/nJK2X9HSBde8HLI2IeyNiLTADGF9X53hgWhramoh4dFM/gJmZbZkiZwTb1aYliezL/ICe37HBUODB3PxyYP+6Onuk9d5A1nw0NSJecWmqpEmkEU+HD3e/tZlZKxW5s3iDyFwFjGvR9rcGRgEHAccA5+Wfhpbb7vSIGBsRY7u7u1u0aTMzg2KDzh2Zm90KGAs8V2DdK4BhufldU1necuDmiHgBuE/SPWSJYX6B9ZuZWQsUuWro8Nz0OmAZr2zrb2Q+MErSSLIEMBGovyLoKrIzgR9JGkLWVHRvgXWbmVmLFOkjOG5zVhwR6ySdAMwha/+/ICIWSTodWBARs9Kyv5O0mOyZB5Mj4vHN2Z6ZmW2eIk1DFwEn1R5gL2lH4OyI+NTG3hsRs4HZdWWn5aYDODm9zMysA4p0Fr+1lgQA0qWeY0qLyMzM2qpIItgqnQUAIGknij/i0szMerkiX+hnA3MlXZHmPwqcUV5IZmbWTkU6iy+WtICXnlF8ZH6YCDMz69uKdBYfQPZMgv9K89tL2j8ibi49OjMzK12RPoIfAn/Nzf81lZmZWT9QJBEoXeYJQES8iDuLzcz6jSKJ4F5JX5A0IL1Ownf/mpn1G0USwWeBd5INE1EbQfT4MoMyM7P2KXLV0KNk4wQBIGkQ8CHgih7fZGZmfUahYagldUk6TNIlwH3Ax8oNy8zM2qXpGYGk95KNGHoYMA94F/CGiHi2DbGZmVkb9JgIJC0HHiC7VPQrEbFa0n1OAmZm/UuzpqGZwC5kzUCHS3o1EE3qm5lZH9RjIoiILwIjycYaOghYAnRLOlrSa9oSnZmZla5pZ3F6RvF1ETGJLCkcQ/Z0smVtiM3MzNqg8B3C6bnCVwNXp0tIzcysHyh0+Wi9iFjT6kDMzKwzNisRmJlZ/+FEYGZWcUWeR7AHMBnYLV8/Ig7u8U1mZtZnFOksvgI4FzgPWF9uOGZm1m5FEsG6iPCDaMzM+qkifQQ/l/Q5STtL2qn2Kj0yMzNriyJnBMemfyfnygJ4Q+vDMTOzdivyPIKR7QjEzMw6Y6NNQ+nxlF+QNDO9TpA0oMjKJR0qaYmkpZJOabD8k5JWSro9vT69OR/CzMw2X5GmoR8CA4D/TvOfSGVNv7QldQHTgEPIHnE5X9KsiFhcV/WyiDhhk6I2M7OWKZII3hERb8vNXyvpjgLv2w9YGhH3AkiaQTZgXX0iMDOzDipy1dB6SW+szUh6A8XuJxgKPJibX57K6n1E0p2p2WlYoxVJmiRpgaQFK1euLLBpMzMrqkgimAxcJ+m3kq4HrgW+3KLt/xwYERFvBa4BLmpUKSKmR8TYiBjb3d3dok2bmRkUu2roN5JGAaNT0ZKIeL7AulcA+V/4u6ay/Lofz82eD3y3wHrNzKyFmj2z+OCIuFbSkXWLdpdERFy5kXXPB0ZJGkmWACYCH6/bxs4R8XCaPQK4e9PCNzOzLdXsjOC9ZM1AhzdYFkDTRBAR6ySdAMwBuoALImKRpNOBBRExC/iCpCOAdcATwCc3/SOYmdmW6DERRMQ30+TpEXFffln6lb9RETEbmF1XdlpuegowpXC0ZmbWckU6i/+nQdnMVgdiZmad0ayP4E3AXsAOdf0E2wMDyw7MzMzao1kfwWjgQ8BgXt5PsBo4vsSYzMysjZr1EfwM+JmkAyNibhtjMjOzNioyxMRtkj5P1ky0oUkoIj5VWlRmZtY2RTqLLwH+BhgHXE92Y9jqMoMyM7P2KZIIdo+IU4FnIuIi4O+B/csNy8zM2qVIIngh/btK0luAHYDXlReSmZm1U5E+gumSdgROBWYBrwFOa/4WMzPrK4oMOnd+mrweP6fYzKzfaXZD2cnN3hgR57Q+HDMza7dmZwTbpX9HA+8gaxaC7OayeWUGZWZm7dPshrJ/A5D0O2DfiFid5qcCv2hLdGZmVroiVw29Hlibm1+byszMrB8octXQxcA8ST9N8xOAC8sKyMzM2qvIVUNnSPol8J5UdFxE3FZuWGZm1i7NrhraPiKelrQTsCy9ast2iognyg/PzMzK1uyM4FKyYahvIXs0ZY3SvO8pMDPrB5pdNfSh9G+hx1KamVnf1KxpaN9mb4yIW1sfjpmZtVuzpqGzmywL4OAWx2JmZh3QrGnofe0MxMzMOqPIfQSk4af35OVPKLu4rKDMzKx9NpoIJH0TOIgsEcwGPgj8gexGMzMz6+OKDDFxFPB+4C8RcRzwNrKH05iZWT9QJBGsiYgXgXWStgceBYYVWbmkQyUtkbRU0ilN6n1EUkgaWyxsMzNrlSJ9BAskDQbOI7u57K/A3I29SVIXMA04BFgOzJc0KyIW19XbDjgJuHnTQjczs1bo8YxA0jRJ74qIz0XEqog4l+xL/djURLQx+wFLI+LeiFgLzADGN6j3LeBM4LnNiN/MzLZQs6ahe4D/lLRM0ncljYmIZRFxZ8F1DwUezM0vT2UbpJvWhkVE0+cbSJokaYGkBStXriy4eTMzK6LHRBAR34+IA4H3Ao8DF0j6o6RvStpjSzcsaSvgHODLG6sbEdMjYmxEjO3u7t7STZuZWc5GO4sj4v6IODMixgDHkD2P4O4C617ByzuVd01lNdsBbwF+K2kZcAAwyx3GZmbttdFEIGlrSYdL+jHwS2AJcGSBdc8HRkkaKWkbYCIvPfeYiHgqIoZExIiIGAHcBBwREQs254OYmdnmaTbo3CFkZwCHkT2sfgYwKSKeKbLiiFgn6QRgDtAFXBARiySdDiyIiFnN12BmZu3Q7PLRKWTPJPhyRDy5OSuPiNlkdyPny07roe5Bm7MNMzPbMs0GnfPoomZmFVDkzmIzM+vHnAjMzCrOicDMrOKcCMzMKs6JwMys4pwIzMwqzonAzKzinAjMzCrOicDMrOKcCMzMKs6JwMys4pwIzMwqzonAzKzinAjMzCrOicDMrOKcCMzMKs6JwMys4pwIzMwqzonAzKzinAjMzCrOicDMrOKcCMzMKs6JwMys4pwIzMwqzonAzKziSk0Ekg6VtETSUkmnNFj+WUkLJd0u6Q+S9iwzHjMze6XSEoGkLmAa8EFgT+CYBl/0l0bE3hGxD/Bd4Jyy4jEzs8bKPCPYD1gaEfdGxFpgBjA+XyEins7NvhqIEuMxM7MGti5x3UOBB3Pzy4H96ytJ+jxwMrANcHCjFUmaBEwCGD58eMsDNTOrso53FkfEtIh4I/A14Bs91JkeEWMjYmx3d3d7AzQz6+fKTAQrgGG5+V1TWU9mABNKjMfMzBooMxHMB0ZJGilpG2AiMCtfQdKo3OzfA38qMR4zM2ugtD6CiFgn6QRgDtAFXBARiySdDiyIiFnACZI+ALwAPAkcW1Y8ZmbWWJmdxUTEbGB2XdlpuemTyty+mZltXMc7i83MrLOcCMzMKs6JwMys4pwIzMwqzonAzKzinAjMzCrOicDMrOKcCMzMKs6JwMys4kq9s7i3uOq2FZw1ZwkPrVrDLoMHMXncaCaMGdrpsMzMeoV+nwiuum0FU65cyJoX1gOwYtUaply5EMDJwCzxj6Vq6/eJ4Kw5SzYkgZo1L6znqzPv5CfzHmjZdhY//DR77rx9y9Zn1i7+sWT9vo/goVVrGpavXf9iS7ez587bM34f/9FY39PTj6Wz5izpUETWbv3+jGCXwYNY0SAZDB08iMs+c2AHIjLrXXr6sdRTufU//f6MYPK40Qwa0PWyskEDupg8bnSHIjLrXXYZPGiTyq3/6feJYMKYoXz7yL0ZOngQIjsT+PaRe7vt0yzxjyXr901DkCUDf/GbNVb72/BVQ9VViURgZs35x1K19fumITMza86JwMys4pwIzMwqzonAzKzinAjMzCpOEdHpGDaJpJXA/Z2OYwsNAR7rdBC9gPdDxvsh4/1Q7j7YLSK6Gy3oc4mgP5C0ICLGdjqOTvN+yHg/ZLwfOrcP3DRkZlZxTgRmZhXnRNAZ0zsdQC/h/ZDxfsh4P3RoH7iPwMys4nxGYGZWcU4EZmYV50RQAkkXSHpU0l25sqmSVki6Pb0Oyy2bImmppCWSxnUm6taSNEzSdZIWS1ok6aRUvpOkayT9Kf27YyqXpB+k/XCnpH07+wlao8l+qNrxMFDSPEl3pP3wb6l8pKSb0+e9TNI2qXzbNL80LR/R0Q/QIk32w4WS7ssdD/uk8vb8XUSEXy1+AX8L7AvclSubCnylQd09gTuAbYGRwJ+Brk5/hhbsg52BfdP0dsA96bN+FzgllZ8CnJmmDwN+CQg4ALi505+h5P1QteNBwGvS9ADg5vT/fDkwMZWfC/xLmv4ccG6anghc1unPUPJ+uBA4qkH9tvxd+IygBBHxO+CJgtXHAzMi4vmIuA9YCuxXWnBtEhEPR8StaXo1cDcwlOzzXpSqXQRMSNPjgYsjcxMwWNLO7Y269Zrsh5701+MhIuKvaXZAegVwMDAzldcfD7XjZCbwfklqT7TlabIfetKWvwsngvY6IZ3eXVBrEiH7UngwV2c5zb8o+px0Wj+G7NfP6yPi4bToL8Dr03TV9gNU7HiQ1CXpduBR4Bqys51VEbEuVcl/1g37IS1/CnhtWwMuSf1+iIja8XBGOh6+J2nbVNaW48GJoH1+CLwR2Ad4GDi7o9G0iaTXAP8DfDEins4vi+zctxLXLzfYD5U7HiJifUTsA+xKdpbzps5G1Bn1+0HSW4ApZPvjHcBOwNfaGZMTQZtExCPpAHgROI+XTvdXAMNyVXdNZX2epAFkX34/jogrU/EjtVPb9O+jqbxS+6GKx0NNRKwCrgMOJGvqqD0yN/9ZN+yHtHwH4PH2Rlqu3H44NDUhRkQ8D/yINh8PTgRtUteu92GgdkXRLGBiukpiJDAKmNfu+Fottef+P+DuiDgnt2gWcGyaPhb4Wa78n9JVEgcAT+WakPqsnvZDBY+HbkmD0/Qg4BCy/pLrgKNStfrjoXacHAVcm84g+7Qe9sMfcz+ORNZPkj8eSv+78MPrSyDpJ8BBwBBJy4FvAgelS8ICWAZ8BiAiFkm6HFgMrAM+HxHrOxB2q70L+ASwMLWHAvwr8B3gckn/TDac+NFp2WyyKySWAs8Cx7U12vL0tB+OqdjxsDNwkaQush+gl0fE1ZIWAzMk/TtwG1nSJP17iaSlZBdeTOxE0CXoaT9cK6mb7Oqg24HPpvpt+bvwEBNmZhXnpiEzs4pzIjAzqzgnAjOzinMiMDOrOCcCM7OKcyKwXiXdXv/F3PwcSefn5s+WdHKT918o6ag0/VtJr3gQuKQBkr6jbATUWyXNlfTBtGyZpCGbEfeG7fawfFoaVXKxpDW5USaPkjS7dm15K0naWdLVTZZvI+l3uRu6rKKcCKy3uQF4J4CkrYAhwF655e8EbtzCbXyL7Hrut0TEvmQ38Gy3hetsKiI+n4YVOAz4c0Tsk14zI+KwdJdpq51MdtdyTzGtBX4DfKyEbVsf4kRgvc2NZEMPQJYA7gJWS9oxDcT1ZuBWSadJmi/pLknTi45MKelVwPHAiel2/tpwD5c3qHtyWv9ddWcp/5QGB7tD0iUN3vetdIbQVTCmZZKGSBoh6Y/pvfdI+rGkD0i6IZ297Jfqv1rZQHXzJN0maXwPq/4I8Kv0nr1S/dtT7KNSnauAfygSp/VfPiW0XiUiHpK0TtJwsl//c8lGWzyQbATKhRGxVtJ/RcTpAOnL+EPAzwtsYnfggfoB8OpJejvZXZz7k93tebOk64G1wDeAd0bEY5J2qnvfWWRnF8dt5pAIuwMfBT4FzAc+DrwbOILsjuQJwNfJhlz4VGpSmifpfyPimVwcI4Ena8mO7E7V70fEj5U9/KWWpO4iG+jMKsxnBNYb3UiWBGqJYG5u/oZU533Knly1kGxM+70arWgLvBv4aUQ8k8aPvxJ4T9rWFRHxGEBE5J87cSqwQ0R8dgvGxbkvIhamwegWAb9J61oIjEh1/g44JQ1Z8VtgIDC8bj07Aytz83OBf5X0NWC3iFiT4l8PrJVUatOY9W5OBNYb1foJ9ib7xXoT2RnBO4EbJQ0E/pvsiU57k7WDDyy47qXAcEnbtzzq7Bf82+vPEjbR87npF3PzL/LSGbyAj+T6GYZHxN1161lDbp9ExKVkZxVrgNmSDs7V3RZ4bgtitj7OicB6oxvJmnqeSEM1PwEMJksGN/LSF9xjysb57/FqnXoR8SzZgGbf10vPx+2W9NG6qr8HJkh6laRXk40Q+nvgWuCjkl6b3pv/0v8V2aB6vyj5F/Yc4MRav4ikMQ3q3MNLZxBIegNwb0T8gGyEz7em8tcCj0XECyXGa72cE4H1RgvJrha6qa7sqYh4LF1hcx7Z2cIcsl/im+IbZM0miyXdBVwN1D8051ay58jOI3ui2PkRcVtELALOAK6XdAdwTt37rkixzVI2zHAZvkX2iMM7JS1K8y+T+gv+LGn3VHQ0cFdqTnoLcHEqfx/wi5LitD7Co4+a9VOSPgy8PSK+0aTOlcApEXFP+yKz3sZXDZn1UxHx01oTViOpaewqJwHzGYGZWcW5j8DMrOKcCMzMKs6JwMys4pwIzMwqzonAzKzi/g/E4r9dHt+9nQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" + "cell_type": "markdown", + "metadata": { + "id": "Ehn1SDb5xAH9" + }, + "source": [ + "The run searched for 9 trials. We can print the best trial's loss, which is 1-the accuracy. The accuracy we got is 91.0% which is close to 91.2% reported by [the Electra model github](https://github.com/google-research/electra). " ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "from flaml.data import get_output_from_log\n", - "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", - " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n", - "for config in config_history:\n", - " print(config)\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "plt.title('Learning Curve')\n", - "plt.xlabel('Wall Clock Time (s)')\n", - "plt.ylabel('Validation Accuracy')\n", - "print(len(valid_loss_history))\n", - "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", - "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.2 Text Summarization Example" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The text summarization task summarizes a long text into a short sentence. For example:\n", - "\n", - "- Document: Army explosives experts were called out to deal with a suspect package at the offices on the Newtownards Road on Friday night. Roads were sealed off and traffic diverted as a controlled explosion was carried out. The premises, used by East Belfast MP Naomi Long, have been targeted a number of times. Most recently, petrol bomb attacks were carried out on the offices on consecutive nights in April and May. The attacks began following a Belfast City Council vote in December 2012 restricting the flying of the union flag at the City Hall. Condemning the latest hoax, Alliance MLA Chris Lyttle said: \"It is a serious incident for the local area, it causes serious disruption, it puts people's lives at risk, it can prevent emergency services reaching the area. \"Ultimately we need people with information to share that with the police in order for them to do their job and bring these people to justice.\n", - "\n", - "- Summary: A suspicious package left outside an Alliance Party office in east Belfast has been declared a hoax.\n", - "\n", - "In this example, we use FLAML to perform *abstractive summarization* using the t5-small language model, i.e., the summary is generated word-by-word. " - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using custom data configuration default\n", - "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n" - ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "204045\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using custom data configuration default\n", - "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n", - "Using custom data configuration default\n", - "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n" - ] - } - ], - "source": [ - "from datasets import load_dataset\n", - "\n", - "train_dataset = load_dataset(\"xsum\", split=\"train\").to_pandas()\n", - "print(len(train_dataset))\n", - "dev_dataset = load_dataset(\"xsum\", split=\"validation\").to_pandas()\n", - "test_dataset = load_dataset(\"xsum\", split=\"test\").to_pandas()\n", - "\n", - "custom_sent_keys = [\"document\"] # specify the column names of the input sentences\n", - "label_key = \"summary\" # specify the column name of the label \n", - "\n", - "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n", - "X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]\n", - "X_test = test_dataset[custom_sent_keys]" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "== Status ==
Current time: 2022-03-19 14:55:00 (running for 00:08:31.38)
Memory usage on this node: 23.1/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/250.17 GiB heap, 0.0/111.21 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: 08b6571c with val_loss=0.8569452656271894 and parameters={'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 9223372036854775807, 'learner': 'transformer', 'FLAML_sample_size': 10000}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-03-19_14-46-29
Number of trials: 8/1000000 (8 TERMINATED)

" + "cell_type": "code", + "execution_count": 25, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qbTAqBsnTjhG", + "outputId": "08c88e0e-0fd8-4b76-8275-e8893c4fe0b1" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "The best loss by FLAML: 0.9105504587155964\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "print(\"The best loss by FLAML: {}\".format(1-automl.best_loss))" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m {'loss': 8.7635, 'learning_rate': 1.2308416834153697e-05, 'epoch': 0.11}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m warnings.warn(\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m warnings.warn(\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m warnings.warn(\n", - "2022-03-19 14:56:00,679\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m {'eval_loss': 6.893245697021484, 'eval_automl_metric': 0.8537338408275918, 'eval_runtime': 102.2734, 'eval_samples_per_second': 110.801, 'eval_steps_per_second': 6.932, 'epoch': 0.11}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:57:00,687\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m {'eval_loss': 7.381210803985596, 'eval_automl_metric': 0.8475751825208984, 'eval_runtime': 107.4032, 'eval_samples_per_second': 105.509, 'eval_steps_per_second': 6.601, 'epoch': 0.16}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m {'eval_loss': 10.150897979736328, 'eval_automl_metric': 0.8566791839938478, 'eval_runtime': 108.2143, 'eval_samples_per_second': 104.718, 'eval_steps_per_second': 6.552, 'epoch': 0.36}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:58:00,697\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m {'eval_loss': 11.665904998779297, 'eval_automl_metric': 0.858011676038827, 'eval_runtime': 109.4667, 'eval_samples_per_second': 103.52, 'eval_steps_per_second': 6.477, 'epoch': 0.38}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m {'eval_loss': 6.893245697021484, 'eval_automl_metric': 0.8537338408275918, 'eval_runtime': 110.7246, 'eval_samples_per_second': 102.344, 'eval_steps_per_second': 6.403, 'epoch': 0.11}\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m {'train_runtime': 220.8946, 'train_samples_per_second': 4.648, 'train_steps_per_second': 0.149, 'train_loss': 8.763471198804451, 'epoch': 0.11}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:59:00,706\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m Num examples = 11332\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m Batch size = 16\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m {'eval_loss': 7.381210803985596, 'eval_automl_metric': 0.8475751825208984, 'eval_runtime': 109.1975, 'eval_samples_per_second': 103.775, 'eval_steps_per_second': 6.493, 'epoch': 0.16}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m {'train_runtime': 232.9303, 'train_samples_per_second': 10.067, 'train_steps_per_second': 1.262, 'train_loss': 9.880440506280637, 'epoch': 0.16}\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m {'eval_loss': 10.150897979736328, 'eval_automl_metric': 0.8566791839938478, 'eval_runtime': 108.3182, 'eval_samples_per_second': 104.618, 'eval_steps_per_second': 6.546, 'epoch': 0.36}\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m {'train_runtime': 232.4568, 'train_samples_per_second': 92.218, 'train_steps_per_second': 2.887, 'train_loss': 11.215172903878349, 'epoch': 0.36}\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m {'eval_loss': 11.665904998779297, 'eval_automl_metric': 0.858011676038827, 'eval_runtime': 110.526, 'eval_samples_per_second': 102.528, 'eval_steps_per_second': 6.415, 'epoch': 0.38}\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m {'train_runtime': 236.6253, 'train_samples_per_second': 19.714, 'train_steps_per_second': 0.621, 'train_loss': 11.549961930614407, 'epoch': 0.38}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 15:00:00,942\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m Num examples = 11332\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m Batch size = 16\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m Num examples = 11332\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m Batch size = 16\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m Num examples = 11332\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m Batch size = 16\n", - "2022-03-19 15:01:00,948\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "2022-03-19 15:02:20,150\tINFO tune.py:639 -- Total run time: 950.87 seconds (500.36 seconds for the tuning loop).\n", - "[flaml.automl: 03-19 15:02:25] {2837} INFO - selected model: None\n", - "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "[flaml.automl: 03-19 15:14:54] {2947} INFO - retrain transformer for 748.2s\n", - "[flaml.automl: 03-19 15:14:54] {2954} INFO - retrained model: None\n", - "[flaml.automl: 03-19 15:14:54] {2283} INFO - fit succeeded\n", - "[flaml.automl: 03-19 15:14:54] {2284} INFO - Time taken to find the best model: 472.3055913448334\n", - "[flaml.automl: 03-19 15:14:54] {2295} WARNING - Time taken to find the best model is 94% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'train_runtime': 14.6848, 'train_samples_per_second': 13894.959, 'train_steps_per_second': 434.258, 'train_loss': 10.199760437011719, 'epoch': 0.02}\n" - ] - } - ], - "source": [ - "''' import AutoML class from flaml package '''\n", - "from flaml import AutoML\n", - "automl = AutoML()\n", - "\n", - "import ray\n", - "if not ray.is_initialized():\n", - " ray.init()\n", - "\n", - "automl_settings = {\n", - " \"time_budget\": 500, # setting the time budget\n", - " \"task\": \"summarization\", # setting the task as summarization\n", - " \"fit_kwargs_by_estimator\": { # if model_path is not set, the default model is t5-small: https://huggingface.co/t5-small\n", - " \"transformer\": {\n", - " \"output_dir\": \"data/output/\", # setting the output directory\n", - " \"model_path\": \"t5-small\",\n", - " \"per_device_eval_batch_size\": 16, # the batch size for validation (inference)\n", - " }\n", - " },\n", - " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n", - " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n", - " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n", - " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n", - " \"metric\": \"rouge1\",\n", - " \"n_concurrent_trials\": 4, # sample: False # if the time is sufficient (e.g., longer than one trial's running time), you can set \n", - "}\n", - "\n", - "'''The main flaml automl API'''\n", - "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 3.6439277745413994e-06, 'num_train_epochs': 0.454119690781029, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.04654549348562217, 'weight_decay': 0.06669806327326033, 'adam_epsilon': 2.5833461668835812e-08, 'seed': 42, 'global_max_steps': 125, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 3.6439277745413994e-06, 'num_train_epochs': 0.454119690781029, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.04654549348562217, 'weight_decay': 0.06669806327326033, 'adam_epsilon': 2.5833461668835812e-08, 'seed': 42, 'global_max_steps': 125, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 112, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 112, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 3.4236378229097798e-06, 'num_train_epochs': 8.919336644807531, 'per_device_train_batch_size': 4, 'warmup_ratio': 0.022492820063166875, 'weight_decay': 0.27013721375576616, 'adam_epsilon': 6.366959214432801e-08, 'seed': 43, 'global_max_steps': 180, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 112, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", - "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 2.83823390666728e-06, 'num_train_epochs': 1.6667827812145841, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.04013366246992448, 'weight_decay': 0.2945152447208819, 'adam_epsilon': 4.694476379503266e-08, 'seed': 43, 'global_max_steps': 163, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 112, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", - "4\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEWCAYAAACjYXoKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAcdUlEQVR4nO3de7gddX3v8feHcAtFiEqkEEBQKApVQQIUKx5vPUBVwAoKeNpGq6gU2+ojFmq1FsoRS2nVSlX0WGorClKM2EajvSlF0UTAQKDRSFNIQm1QuWkkBL7nj5mNi83eO2tn9tqX5P16nvUw85vfzHz3hL0++zez1kyqCkmSuthmqguQJM18hokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0ykCZTk6CQrproOabIZJtpiJFmV5MVTWUNVXVNVBw5q+0mOSfLVJPclWZfkK0mOH9T+pH4ZJtI4JJk1hfs+CfgM8AlgL2B34F3AyzZjW0ni778mjP8zaYuXZJskZyf5XpIfJLkiyRN6ln8myX8nuaf9q//gnmWXJvlQkkVJfgy8oB0BvS3Jsnady5Ps2PZ/fpLVPeuP2rdd/vYkdyZZm+R1SSrJ/iP8DAH+HDivqj5WVfdU1cNV9ZWqen3b591J/q5nnX3b7W3bzv9bkvOTXAv8BDgrydJh+3lLkqvb6R2S/FmS25N8P8mHk8zu+M+hLZRhoq3Bm4ETgf8F7An8CLi4Z/kXgAOAJwHXA58ctv5pwPnA44B/b9teCRwL7Ac8E1gwxv5H7JvkWOCtwIuB/YHnj7GNA4G9gSvH6NOPXwdOp/lZPgwcmOSAnuWnAZe10xcAvwAc0tY3j2YkJD2GYaKtwRuBd1TV6qp6AHg3cNLQX+xV9fGquq9n2bOS7Nqz/ueq6tp2JPDTtu0DVbW2qn4IfJ7mDXc0o/V9JfDXVbW8qn7S7ns0T2z/e2d/P/KoLm33t7Gq7gE+B5wK0IbK04Cr25HQ6cBbquqHVXUf8H+BUzruX1sow0RbgycDn01yd5K7gVuBh4Ddk8xKckF7CuxeYFW7zm49698xwjb/u2f6J8DOY+x/tL57Dtv2SPsZ8oP2v3uM0acfw/dxGW2Y0IxKFrbBNhfYCfhWz3H7YtsuPYZhoq3BHcBxVTWn57VjVa2heQM9geZU067Avu066Vl/ULfWvpPmQvqQvcfou4Lm53jFGH1+TBMAQ35+hD7Df5YvA3OTHEITKkOnuO4C1gMH9xyzXatqrNDUVsww0ZZmuyQ79ry2pbk2cH6SJwMkmZvkhLb/44AHaP7y34nmVM5kuQJ4TZKnJ9kJeOdoHat5VsRbgXcmeU2SXdoPFjw3ySVttxuB5yXZpz1Nd86mCqiqB2k+IXYh8ASacKGqHgY+CvxFkicBJJmX5JjN/WG1ZTNMtKVZRPMX9dDr3cD7gauBLyW5D7gOOLLt/wngv4A1wC3tsklRVV8APgD8K7CyZ98PjNL/SuBVwGuBtcD3gT+hue5BVX0ZuBxYBnwL+Ic+S7mMZmT2mara2NP++0N1tacA/4nmgwDSY8SHY0nTQ5KnAzcDOwx7U5emPUcm0hRK8vL2+xyPB94LfN4g0UxkmEhT6w3A/wDfo/mE2Zumthxp83iaS5LUmSMTSVJn2051AZNht912q3333Xeqy5CkGWO33XZj8eLFi6vq2H76bxVhsu+++7J06dJNd5QkPSLJbpvu1fA0lySpM8NEktSZYSJJ6swwkSR1ZphIkjrbKj7NJUnjsfCGNVy4eAVr717PnnNmc9YxB3LiofOmuqxpzTCRpB4Lb1jDOVfdxPoHHwJgzd3rOeeqmwAMlDEYJpLU48LFKx4JkiHrH3yIt1+5jE998/YpqmrzHLTnLvzRyw6elH0N9JpJkmOTrEiyMsnZIyx/XpLrk2xMctIIy3dJsjrJB3vavpjk20mWJ/lwklmD/BkkbV3W3r1+xPYNDz08yZXMLAMbmbRv8hcDvwKsBpYkubqqbunpdjuwAHjbKJs5D/jqsLZXVtW9SQJcCZwMfHoia5e09dpzzmzWjBAo8+bM5vI3HDUFFc0MgxyZHAGsrKrbqmoDzRv+Cb0dqmpVVS0DHhP5SQ4Ddge+NGyde9vJbYHtGdzzuSVthc465kBmb/foEx6zt5vFWcf4kMmxDDJM5gF39Myvbts2Kck2wEWMMmJJspjmGRD30YxORupzepKlSZauW7duPHVL2oqdeOg83vNrz2D7Wc3b47w5s3nPrz3Di++bMF2/Z3IGsKiqVo+0sKqOAfYAdgBeOEqfS6pqflXNnzt37uAqlbTFOfHQeRy6zxyO3O8JXHv2Cw2SPgzy01xrgL175vdq2/pxFHB0kjOAnYHtk9xfVY9cxK+qnyb5HM2psy9PUM2SpM0wyDBZAhyQZD+aEDkFOK2fFavq1UPTSRYA86vq7CQ7A4+rqjuTbAu8BLhmwiuXJI3LwE5zVdVG4ExgMXArcEVVLU9ybpLjAZIcnmQ1zSeyPpJk+SY2+3PA1UmWATfSXDf58KB+BklSfwb6pcWqWgQsGtb2rp7pJTSnv8baxqXApe3094HDJ7pOSVI30/UCvCRpBjFMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4GGiZJjk2yIsnKJGePsPx5Sa5PsjHJSSMs3yXJ6iQfbOd3SvKPSf4jyfIkFwyyfklSfwYWJklmARcDxwEHAacmOWhYt9uBBcBlo2zmPOCrw9r+rKqeBhwK/HKS4yasaEnSZhnkyOQIYGVV3VZVG4BPAyf0dqiqVVW1DHh4+MpJDgN2B77U0/8nVfWv7fQG4Hpgr8H9CJKkfgwyTOYBd/TMr27bNinJNsBFwNvG6DMHeBnwz6MsPz3J0iRL161b12/NkqTNMF0vwJ8BLKqq1SMtTLIt8CngA1V120h9quqSqppfVfPnzp07wFIlSdsOcNtrgL175vdq2/pxFHB0kjOAnYHtk9xfVUMX8S8BvltV75uoYiVJm2+QYbIEOCDJfjQhcgpwWj8rVtWrh6aTLADmDwVJkj8BdgVeN9EFS5I2z8BOc1XVRuBMYDFwK3BFVS1Pcm6S4wGSHJ5kNXAy8JEky8faZpK9gHfQfDrs+iQ3JjFUJGmKDXJkQlUtAhYNa3tXz/QSNvFprKq6FLi0nV4NZKLrlCR1M10vwEuSZhDDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdbZZYZLkCxNdiCRp5tp2tAVJnj3aIuCQgVQjSZqRRg0TYAnwFZrwGG7OQKqRJM1IY4XJrcAbquq7wxckuWNwJUmSZpqxrpm8e4zlb574UiRJM9WoI5OqunKMZQsHUo0kaUbyo8GSpM4ME0lSZwMNkyTHJlmRZGWSs0dY/rwk1yfZmOSkEZbvkmR1kg/2tJ2f5I4k9w+ydklS/zYZJkl2SvLOJB9t5w9I8tI+1psFXAwcBxwEnJrkoGHdbgcWAJeNspnzgK8Oa/s8cMSm9i9Jmjz9jEz+GngAOKqdXwP8SR/rHQGsrKrbqmoD8GnghN4OVbWqqpYBDw9fOclhwO7Al4atc11V3dnH/iVJk6SfMHlqVf0p8CBAVf2Ekb/IONw8oPf7KKvbtk1Ksg1wEfC2fvqPso3TkyxNsnTdunWbuxlJUh/6CZMNSWYDBZDkqTQjlUE6A1hUVas3dwNVdUlVza+q+XPnzp3A0iRJw431DfghfwR8Edg7ySeBX6a5zrEpa4C9e+b3atv6cRRwdJIzgJ2B7ZPcX1WPuYgvSZp6mwyTqvpykuuBX6I5vfW7VXVXH9teAhyQZD+aEDkFOK2foqrq1UPTSRYA8w0SSZq++vk017OBJwN3AmuBfZI8NcmYQVRVG4EzgcU09/m6oqqWJzk3yfHttg9Psho4GfhIkuV91POn7To7tR8bfvem1pEkDVY/p7n+Cng2sIxmZPKLwHJg1yRvqqovjbZiVS0CFg1re1fP9BKa01+jqqpLgUt75t8OvL2PuiVJk6SfC/BrgUPbi9mHAYcCtwG/AvzpIIuTJM0M/YTJL1TVI6efquoW4GlVddvgypIkzST9nOZanuRDNF86BHgVcEuSHWi/eyJJ2rr1MzJZAKwEfq993da2PQi8YDBlSZJmkn4+Grye5tvoF42w2JstSpI2HSZJ/pP22++9quopA6lIkjTj9HPNZH7P9I403wl5wmDKkSTNRJu8ZlJVP+h5ramq9wEvGXxpkqSZop/TXM/umd2GZqTSz4hGkrSV6CcUei+8bwRWAa8cSDWSpBmpn09z+fFfSdKY+rnR465J/nzoQVNJLkqy62QUJ0maGfr50uLHgftoTm29EriX5lG+kiQB/V0zeWpVvaJn/o+T3DigeiRJM1A/I5P1SZ47NJPkl4H1gytJkjTT9DMyeRPwN+11kgA/BH5zoFVJkmaUfj7NdSPwrCS7tE0/pnkE77IB1iVJmkFGPc2VZJck5yT5YJJfobkI/xs0dxD2eyaSpEeMNTL5W+BHwNeB1wPvoDnN9fJ2tCJJEjB2mDylqp4BkORjwJ3APlX100mpTJK0WRbesIYLF69g7d3r2XPObM465kBOPHTeQPc5Vpg88hTFqnooyWqDRJKmt4U3rOGcq25i/YMPAbDm7vWcc9VNAAMNlLE+GvysJPe2r/uAZw5NJ7l3YBVJkjbbhYtXPBIkQ9Y/+BAXLl4x0P2OOjKpqlkD3bMkacKtvXvkrwGO1j5R+vnSoiRphthzzuxxtU8Uw0SStiBnHXMgs7d79Iml2dvN4qxjDhzofn3IlSRtQYYusk+nT3NJkmagEw+dN/DwGM7TXJKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgYaJkmOTbIiycokZ4+w/HlJrk+yMclJIyzfJcnqJB/saTssyU3tNj+QJIP8GSRJmzawMEkyC7gYOA44CDg1yUHDut0OLAAuG2Uz5wFfHdb2IZrHCB/Qvo6doJIlSZtpkCOTI4CVVXVbVW0APg2c0NuhqlZV1TLg4eErJzkM2B34Uk/bHsAuVXVdVRXwCeDEwf0IkqR+DPJGj/OAO3rmVwNH9rNikm2Ai4D/A7x42DZXD9vmiHczS3I6cDrAPvvs03fRQ6biGcqSNFNN1wvwZwCLqmr1JnuOoqouqar5VTV/7ty541p36BnKa+5eT/GzZygvvGHN5pYjSVu0QY5M1gB798zv1bb14yjg6CRnADsD2ye5H3h/u53N2WbfRnuG8tuvXManvnn7RO9O0jR0y533ctAeu0x1GTPGIMNkCXBAkv1o3vBPAU7rZ8WqevXQdJIFwPyqOrudvzfJLwHfAH4D+MsJrnvUZyVveOgxl3YkbaEO2mMXTjjEU9v9GliYVNXGJGcCi4FZwMeranmSc4GlVXV1ksOBzwKPB16W5I+r6uBNbPoM4FJgNvCF9jWh9pwzmzUjBMq8ObO5/A1HTfTuJGnGS/OhqC3b/Pnza+nSpX33H7pm0nuqa/Z2s3jPrz3Di/CSthpJvlVV8/vp62N7RzBVz1CWpJnKMBnFVDxDWZJmqun60WBJ0gximEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHU20DBJcmySFUlWJjl7hOXPS3J9ko1JTuppf3LbfmOS5Une2LPsVUmWte3vHWT9kqT+DCxMkswCLgaOAw4CTk1y0LButwMLgMuGtd8JHFVVhwBHAmcn2TPJE4ELgRdV1cHAzyd50aB+BklSfwY5MjkCWFlVt1XVBuDTwAm9HapqVVUtAx4e1r6hqh5oZ3foqfMpwHeral07/0/AKwb1A0iS+jPIMJkH3NEzv7pt60uSvZMsa7fx3qpaC6wEDkyyb5JtgROBvSeuZEnS5pi2F+Cr6o6qeiawP/CbSXavqh8BbwIuB64BVgEPjbR+ktOTLE2ydN26dSN1kSRNkEGGyRoePWrYq20bl3ZEcjNwdDv/+ao6sqqOAlYA3xllvUuqan5VzZ87d+64i5ck9W+QYbIEOCDJfkm2B04Bru5nxSR7JZndTj8eeC5NcJDkST3tZwAfG0DtkqRxGFiYVNVG4ExgMXArcEVVLU9ybpLjAZIcnmQ1cDLwkSTL29WfDnwjybeBrwB/VlU3tcven+QW4FrggqoacWQiSZo8qaqprmHg5s+fX0uXLp3qMiRpRknyraqa30/faXsBXpI0cxgmkqTODBNJUmeGiSSpM8NEktSZYSJJ6swwkSR1ZphIkjrbdqoLkCRNnIU3rOHCxStYe/d69pwzm7OOOZATD+37hu2bzTCRpC3EwhvWcM5VN7H+weZm6mvuXs85VzV3ohp0oHiaS5K2EBcuXvFIkAxZ/+BDXLh4xcD3bZhI0hZi7d3rx9U+kQwTSdpC7Dln9rjaJ5JhIklbiLOOOZDZ2816VNvs7WZx1jEHDnzfXoCXpC3E0EV2P80lSerkxEPnTUp4DOdpLklSZ4aJJKkzw0SS1JlhIknqzDCRJHWWqprqGgYuyTrgvwa4i92Auwa4/S6ma23WNT7WNT7WNX7Da7sLoKqO7WflrSJMBi3J0qqaP9V1jGS61mZd42Nd42Nd49e1Nk9zSZI6M0wkSZ0ZJhPjkqkuYAzTtTbrGh/rGh/rGr9OtXnNRJLUmSMTSVJnhokkqTPDpE9JZiW5Ick/DGv/QJL7e+Z3SHJ5kpVJvpFk32lU24Ik65Lc2L5eN5l1Jbk0yX/27P+Qtj1trSuTLEvy7GlS1/OT3NPT/q5JritJzk/ynSS3JvmdnvapPF6j1TXVx+uann2vTbKwp96pPF6j1TWpx2uU2l6U5Pp2//+eZP+2fdzvY96Cvn+/C9wK7DLUkGQ+8Phh/X4L+FFV7Z/kFOC9wKumSW0Al1fVmQOuZ9S6gLOq6sph/Y4DDmhfRwIfav871XUBXFNVLx1gLb2G17UA2Bt4WlU9nORJbftUH6/R6oIpPF5VdfTQgiR/D3yunZ3S4zVGXTC5x+sxtdEcixOq6tYkZwB/SPPvO+73MUcmfUiyF/AS4GM9bbOAC4G3D+t+AvA37fSVwIuSZJrUNmlGqmsMJwCfqMZ1wJwke0yDuibNKHW9CTi3qh4GqKr/adun+niNVtekGevfMckuwAuBhW3TVB+v0eqaVKPUVvwsWHYF1rbT434fM0z68z6aN+aHe9rOBK6uqjuH9Z0H3AFQVRuBe4AnTpPaAF7RDvWvTLL3JNcFcH67/79IskPb9sgxa61u26a6LoCjknw7yReSHDygmkar66nAq5Isbfd/QNs+1cdrtLpgao/XkBOBf66qe9v5qT5eo9UFk3e8RqvtdcCiJKuBXwcuaNvH/T5mmGxCkpcC/1NV3+pp2xM4GfjLKSuMzart88C+VfVM4Mv87C+PgdfVOgd4GnA48ATg9wex/wms63rgyVX1LJrjuXCS69oB+Gl7i4uPAh8fxP4nsK6pPl5DTgU+NYh9j2Uz6pqU47WJ2t4C/GpV7QX8NfDnm72TqvI1xgt4D81fMquA/wZ+AvyonV7Vvh4GVrb9FwNHtdPb0twsLdOhtmHrzgLumcS6/m5Yn+cD/9BOfwQ4tWfZCmCPqa5rhPVXAbtNVl3AfwD7tX0y9O811cdrtLqm+ni1y3YDfgDs2NN/yv//GqmuyTpeY9T2j8D3evrsA9zSTo/7fWzCi96SX6O90QD390z/NvDhdvoU4IppVNsePdMvB66bzLqG9t++Ab0PuKCdfwnwhbb9l4BvTpO6fn7oFwg4Arh9U79QE1zXBcBre9qXTJPjNVpdU3q82vk3An8zrM+UHq8x6pr049VbW09I/ELb/lvA37fT434f89NcE+//AX+bZCXwQ5p/iOnid5IcD2ykqW3BJO//k0nm0vxS30jzCwawCPhVYCXNX0yvmSZ1nQS8KclGYD1wSrW/XZPkgra2twD305zfhqk/XqPVNdXHC5rftwuGtU318YKR65rS41VVG5O8Hvj7JA/TnNV4bbt43O9j3k5FktSZF+AlSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWGiGa+9Bcrv9cwvTtJ7r7KLkrx1jPUvTXJSO/1v7U0yh/fZLskFSb7b3mX160mOa5etSrLbZtT9yH5HWX5xezfXW5Ks77m77ElJFiWZM9599lHTHhl29+lhy7dP8tUkfq1Aj2KYaEtwLfAcgCTb0HzbuPc+R88BvtZxH+cBewC/WFXPprnP0uM6bnNMVfXbVXUIzXckvldVh7SvK6vqV6vq7gHs9q00t0gZraYNwD8z+Dtha4YxTLQl+BpwVDt9MHAzcF+Sx7c3bXw6cH2SdyVZkuTmJJf0ezfnJDsBrwfeXFUPAFTV96vqihH6vrXd/s3DRku/0d5I8ttJ/naE9c5rRyqz+qxpVZLdkuyb5D/adb+T5JNJXpzk2nYUdUTb/+eSfDzJN9M8z+KEUTb9CuCL7ToHt/1vbGsfuqHjQuDV/dSprYdDVc14VbU2ycYk+9CMQr5Oc9fTo2judnpTVW1I8sGqOhegfUN/Kc3NLzdlf+D2evTdXh8jyWE0364+kubb9N9I8hVgA81zIp5TVXclecKw9S6kGeW8ZjO/Ab0/zc09XwssAU4DngscD/wBzSjqHcC/VNVr29Nj30zyT1X145469qN5hsUDbdMbgfdX1SeTbE9zPzdowvrwzahTWzBHJtpSfI0mSIbC5Os989e2fV6Q5qlxN9E8V2Kib/n9XOCzVfXjqrofuAo4ut3XZ6rqLoCq+mHPOu8Edq2qN3a4lcZ/VtVN1TxfZDnNbc4LuAnYt+3zv4Gzk9wI/BuwI82N/XrtAazrmf868AdJfp/m7rbr2/ofAjYkGehpPs0shom2FEPXTZ5B85fzdTQjk+cAX0uyI/BXwElV9Qya6wI79rntlcA+aR5uNNGWAIcNH62M0wM90w/3zD/Mz84+BHhFz3WXfarq1mHbWU/PMamqy2hGN+tpnnnxwp6+OwA/7VCztjCGibYUX6M5bfXDqnqo/et/Dk2gfI2fvUnelWRnmpvs9aWqfkJz47v3t6d7SDI3ycnDul4DnJhkpyQ/R3Nn5muAfwFOTvLEdt3e4PgizQ0A/3HAf+kvBt48dJ0oyaEj9PkOPxvJkOQpwG1V9QGaR80+s21/InBXVT04wHo1wxgm2lLcRPMpruuGtd1TVXe1n3z6KM2oZTHNiGA8/pDmFNAtSW6muYX3o66hVNX1wKXAN4FvAB+rqhuqajlwPvCVJN9m2AOIquozbW1XJ5k9zrr6dR6wHbAsyfJ2/lHa6yffS7J/2/RK4Ob21NgvAp9o219A8ywM6RHeNVjSI5K8HDisqv5wjD5XAWdX1XcmrzJNd36aS9IjquqzQ6fjRtKe5ltokGg4RyaSpM68ZiJJ6swwkSR1ZphIkjozTCRJnRkmkqTO/j/XoXUXWf2mTgAAAABJRU5ErkJggg==", - "text/plain": [ - "
" + "cell_type": "markdown", + "metadata": { + "id": "wcO2th5M6AIu" + }, + "source": [ + "If you have more GPUs on your server, you can use flaml.tune with the ray tune option, which will often give you a better score. For example, with the 4 NVIDIA V100 GPUs, the accuracy was 92.2%. For that experiment, you can open this notebook on your GPU server and set \"use_ray\" to {\"local_dir\": \"data/output/\"} and n_concurrent_trials to more than 1. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QFP5JNdPTjhG" + }, + "source": [ + "### Best model and metric" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mY07pTY_xlIV" + }, + "source": [ + "Next, we can print the best hyperparameter and the best score:" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "sbnhP3WrTjhG", + "outputId": "e7be276a-d30f-4dde-acea-d7b00107a161" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Best hyperparmeter config: {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 8112, 'FLAML_sample_size': 67349}\n", + "Best accuracy on validation data: 0.9106\n", + "Training duration of best run: 523.1 s\n" + ] + } + ], + "source": [ + "'''retrieve best config and best learner'''\n", + "print('Best hyperparmeter config:', automl.best_config)\n", + "print('Best accuracy on validation data: {0:.4g}'.format(1-automl.best_loss))\n", + "print('Training duration of best run: {0:.4g} s'.format(automl.best_config_train_time))" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Save and load the model:" + ], + "metadata": { + "id": "MqIpmxl0dKWu" + } + }, + { + "cell_type": "code", + "source": [ + "import pickle\n", + "automl.pickle(\"automl.pkl\")\n", + "\n", + "with open(\"automl.pkl\", \"rb\") as f:\n", + " automl = pickle.load(f)" + ], + "metadata": { + "id": "gfUNXfcNTBA2" + }, + "execution_count": 27, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6mdBURdexxJS" + }, + "source": [ + "Run the prediction:\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "kRl7pnEKTjhH", + "outputId": "19b17fb3-cf01-472c-958d-1511be54379d", + "scrolled": true + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Prediction *****\n", + " Num examples = 872\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/vocab.txt\n", + "loading file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/special_tokens_map.json\n", + "loading file data/output/train_2022-08-21_03-12-52/train_249deb52_27_s=9223372036854775807,e=2.2155e-05,s=-1,s=3,e=16,d=24_2022-08-21_03-12-52/checkpoint-8112/tokenizer_config.json\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Predicted labels [1 1 1 1 0 1 0 0 1 0 1 0 0 0 0 1 1 1 0 0 0 0 0 1 1 0 0 1 0 0 1 0 1 0 0 0 1\n", + " 0 1 1 1 1 1 1 0 0 0 1 1 0 0 1 1 1 0 1 0 0 0 0 1 0 1 1 1 0 1 1 1 0 0 1 1 1\n", + " 0 1 0 1 1 0 1 0 0 1 1 1 0 1 1 1 1 1 1 0 1 1 1 1 0 0 1 0 1 1 1 0 1 0 0 1 0\n", + " 0 1 0 1 1 1 1 0 0 1 1 1 0 1 1 0 0 1 1 0 0 1 0 0 1 0 0 1 0 0 0 1 1 0 0 1 0\n", + " 0 1 1 1 1 0 1 0 1 0 1 1 0 0 0 0 1 0 0 0 1 1 1 1 1 0 1 1 1 0 0 1 0 0 0 1 0\n", + " 1 1 1 0 0 0 1 1 1 1 1 1 0 1 0 1 1 0 0 1 1 1 1 0 0 1 0 0 1 0 0 1 0 1 1 1 0\n", + " 1 1 1 1 0 1 1 0 1 1 0 0 1 1 1 0 0 1 1 0 0 1 1 1 1 0 0 1 1 0 1 1 0 0 0 0 0\n", + " 1 0 1 0 1 0 0 0 0 0 0 1 0 0 1 1 1 1 1 0 1 1 0 0 1 0 0 1 1 1 1 1 0 1 1 1 1\n", + " 0 1 0 1 1 1 1 1 1 0 1 1 1 1 1 1 0 1 1 0 1 1 0 0 1 0 0 1 0 1 1 1 0 0 0 1 1\n", + " 1 1 0 1 0 0 1 0 1 0 0 1 1 0 0 0 0 0 1 1 1 1 0 0 0 0 1 1 0 1 0 0 0 1 1 0 1\n", + " 0 1 1 0 0 0 0 0 0 0 0 1 0 1 0 1 1 1 0 1 0 0 0 1 1 1 1 1 1 0 1 1 1 0 1 0 1\n", + " 0 0 0 1 1 0 0 1 1 1 1 1 0 0 1 1 0 1 1 1 1 1 0 1 0 0 1 0 1 0 1 1 1 1 0 1 1\n", + " 0 1 1 1 0 1 1 1 0 1 0 1 0 1 1 1 1 0 0 0 0 0 1 1 1 0 1 0 1 1 0 1 0 1 1 1 1\n", + " 1 1 1 1 1 0 0 1 0 0 0 1 0 1 1 1 0 1 1 0 0 0 0 1 1 1 1 1 1 1 1 0 1 1 0 0 1\n", + " 1 1 1 0 0 1 1 1 0 0 1 0 1 0 1 1 1 1 0 1 0 0 1 1 0 0 1 0 1 0 0 0 0 0 0 1 0\n", + " 0 0 1 1 0 0 0 0 1 0 1 0 1 0 1 1 1 0 1 0 0 0 1 0 0 1 0 0 0 0 0 0 0 1 1 0 0\n", + " 0 1 1 0 0 0 0 0 1 0 1 1 1 1 0 1 1 1 0 1 1 1 1 0 1 1 1 0 0 1 1 1 1 1 1 0 1\n", + " 0 1 0 0 1 1 0 1 0 1 0 1 1 0 0 1 0 1 0 1 0 0 0 0 0 0 1 1 0 1 0 1 0 0 1 1 1\n", + " 1 1 0 1 0 1 0 0 1 1 0 1 1 1 1 0 0 1 1 0 1 0 0 1 0 1 1 0 0 1 1 1 1 0 0 0 0\n", + " 0 1 0 0 0 1 0 0 0 0 0 1 0 1 1 1 0 1 1 0 1 0 1 1 0 1 1 0 0 0 1 0 0 1 1 1 1\n", + " 1 1 1 0 1 0 1 1 0 0 0 0 0 1 1 1 0 0 1 0 0 0 1 1 0 0 1 1 1 1 0 1 1 1 0 0 0\n", + " 1 1 0 1 0 1 1 1 1 0 0 1 0 0 1 1 1 1 0 1 0 0 1 0 0 0 1 0 1 1 1 1 1 1 0 1 1\n", + " 0 1 1 1 0 0 1 1 0 1 0 1 1 1 0 1 1 1 0 0 1 1 0 1 0 0 0 1 0 0 0 0 1 1 1 1 0\n", + " 0 0 1 0 1 1 1 1 1 1 1 1 0 0 0 1 0 1 0 1 1]\n" + ] + } + ], + "source": [ + "'''compute predictions of testing dataset''' \n", + "y_pred = automl.predict(X_val, **{\"per_device_eval_batch_size\": 1})\n", + "print('Predicted labels', y_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QThcVssKTjhH" + }, + "source": [ + "### Log history" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OEFqWAuLyYIQ" + }, + "source": [ + "You can also save and plot the history:" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "58wpj4vPTjhH", + "outputId": "bbd9850e-dc0e-416b-d9eb-342bb4a3a052" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 939, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 939, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.711865003865157e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 64, 'seed': 14, 'global_max_steps': 471, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 939, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.0296683485633468e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 26, 'global_max_steps': 1875, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0296683485633468e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 26, 'global_max_steps': 1875, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.4799994583688665e-05, 'num_train_epochs': 2, 'per_device_train_batch_size': 16, 'seed': 25, 'global_max_steps': 1250, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0296683485633468e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 26, 'global_max_steps': 1875, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 7.163630378633069e-06, 'num_train_epochs': 4, 'per_device_train_batch_size': 16, 'seed': 27, 'global_max_steps': 1803, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0296683485633468e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 26, 'global_max_steps': 1875, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.789274561853069e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 32, 'global_max_steps': 1391, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0296683485633468e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 26, 'global_max_steps': 1875, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 5.925400889482007e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 20, 'global_max_steps': 1250, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0296683485633468e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 26, 'global_max_steps': 1875, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 1250, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 1250, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 2.4827446197906688e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 8, 'seed': 24, 'global_max_steps': 1146, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 1250, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.9769786550171826e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 24, 'global_max_steps': 694, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 1250, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.4983353971653967e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 18, 'global_max_steps': 841, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 1250, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 3.2758574138144633e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 30, 'global_max_steps': 711, 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 1250, 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 67349, 'Current Hyper-parameters': {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 8112, 'FLAML_sample_size': 67349}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 2.215475822297527e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 16, 'seed': 24, 'global_max_steps': 8112, 'FLAML_sample_size': 67349}}\n" + ] + } + ], + "source": [ + "from flaml.data import get_output_from_log\n", + "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", + " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n", + "for config in config_history:\n", + " print(config)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 312 + }, + "id": "dtWSrLsdTjhH", + "outputId": "dfe4f9c5-f9b7-4a7d-d519-4d4a24647aa4" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "13\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5hdVX3/8feHEMiAhACJNCSExAIpQSzBCGJVEC+J+QlERAvWimhB28pPRUNJQUyx1EuqffSR4g8sRVCuKYSokUgRsYUgCQQSEgwGjJAJQhAiF0dy+/7+2OuEncOeM3smc24zn9fznGf2XvtyvmfPzPnuvdbaaysiMDMzq7ZTswMwM7PW5ARhZmaFnCDMzKyQE4SZmRVygjAzs0JOEGZmVsgJwqwPJL1F0qpmx2FWT04Q1nYkrZH0jmbGEBH/ExET67V/SVMl/VzS85LWS7pD0gn1ej+zIk4QZgUkDWnie58M3ABcCYwF9gUuAI7vw74kyf/n1if+w7EBQ9JOks6V9Iik30m6XtLeueU3SPqtpN+ns/NDc8uukHSJpAWSXgTelq5UPidpWdrmOknD0vrHSlqb277bddPycyQ9IWmdpL+RFJIOLPgMAr4OfDEivhMRv4+IrRFxR0SckdaZLel7uW3Gp/3tnOZ/JukiSXcCfwBmSlpS9T6fkTQ/Te8q6V8lPSbpSUnfltSxg78OGwCcIGwgOQuYARwD7Ac8C1ycW/5j4CDg1cB9wPertv8gcBGwB/C/qewDwDRgAvA64CM13r9wXUnTgLOBdwAHAsfW2MdEYH9gbo11yvhr4Eyyz/JtYKKkg3LLPwhcnaa/DBwMHJ7iG0N2xWKDnBOEDSSfAM6LiLUR8RIwGzi5cmYdEZdHxPO5ZX8uac/c9jdHxJ3pjP2PqeybEbEuIp4BfkD2Jdqd7tb9APCfEbEiIv6Q3rs7+6SfT5T90N24Ir3f5oj4PXAzcCpAShR/BsxPVyxnAp+JiGci4nngX4BTdvD9bQBwgrCB5ADgJkkbJG0AHgK2APtKGiLpy6n66TlgTdpmZG77xwv2+dvc9B+AV9V4/+7W3a9q30XvU/G79HN0jXXKqH6Pq0kJguzqYV5KVqOA3YB7c8ftllRug5wThA0kjwPvjogRudewiOgk+1I8kayaZ09gfNpGue3rNbTxE2SNzRX711h3FdnneF+NdV4k+1Kv+JOCdao/y63AKEmHkyWKSvXS00AXcGjumO0ZEbUSoQ0SThDWroZKGpZ77UxW136RpAMAJI2SdGJafw/gJbIz9N3IqlEa5XrgdEmHSNoN+Hx3K0Y2/v7ZwOclnS5peGp8f7OkS9Nq9wNvlTQuVZHN6imAiNhE1jNqDrA3WcIgIrYClwH/JunVAJLGSJra509rA4YThLWrBWRnvpXXbOAbwHzgJ5KeB+4GjkrrXwn8BugEVqZlDRERPwa+CdwOrM6990vdrD8X+Evgo8A64Engn8naEYiIW4HrgGXAvcAPS4ZyNdkV1A0RsTlX/g+VuFL123+TNZbbICc/MMissSQdAjwI7Fr1RW3WUnwFYdYAkt6b7jfYC/gK8AMnB2t1ThBmjfFx4CngEbKeVX/b3HDMeuYqJjMzK+QrCDMzK7RzswPoLyNHjozx48c3Owwzs7Zy7733Ph0RhTdGDpgEMX78eJYsWdLzimZmto2k33S3zFVMZmZWyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoUGTC8mM7PBZt7STuYsXMW6DV3sN6KDmVMnMmPymH7bvxOEmVkbmre0k1k3Lqdr0xYAOjd0MevG5QD9liRcxWRm1obmLFy1LTlUdG3awpyFq/rtPZwgzMza0LoNXb0q7wsnCDOzNrTfiI5elfeFE4SZWRuaOXUiHUOHbFfWMXQIM6f238MA3UhtZtaGKg3R58xdxsYtWxnjXkxmZlYxY/IYrrnnMQCu+/jR/b5/VzGZmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFXKCMDOzQk4QZmZWyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFXKCMDOzQnVNEJKmSVolabWkcwuWHyDpNknLJP1M0tjcstMk/Sq9TqtnnGZm9kp1SxCShgAXA+8GJgGnSppUtdq/AldGxOuAC4EvpW33Br4AHAUcCXxB0l71itXMzF6pnlcQRwKrI+LRiNgIXAucWLXOJOCnafr23PKpwK0R8UxEPAvcCkyrY6xmZlalngliDPB4bn5tKst7ADgpTb8X2EPSPiW3RdKZkpZIWrJ+/fp+C9zMzJrfSP054BhJS4FjgE5gS9mNI+LSiJgSEVNGjRpVrxjNzAalej6TuhPYPzc/NpVtExHrSFcQkl4FvC8iNkjqBI6t2vZndYzVzMyq1PMKYjFwkKQJknYBTgHm51eQNFJSJYZZwOVpeiHwLkl7pcbpd6UyMzNrkLoliIjYDHyS7Iv9IeD6iFgh6UJJJ6TVjgVWSXoY2Be4KG37DPBFsiSzGLgwlZmZWYPUs4qJiFgALKgquyA3PReY2822l/PyFYWZmTVYsxupzcysRTlBmJlZIScIMzMr5ARhZmaFnCDMzKyQE4SZmRVygjAzs0JOEGZmVsgJwszMCjlBmJlZIScIMzMr5ARhZmaFnCDMzKyQE4SZmRVygjAzs0JOEGZmVsgJwszMCjlBmJlZIScIMzMr5ARhZmaFnCDMzKyQE4SZmRVygjAzs0I9JghJ+zQiEDMzay1lriDulnSDpOmSVPeIzMysJZRJEAcDlwJ/DfxK0r9IOri+YZmZWbP1mCAic2tEnAqcAZwG3CPpDklH1z1CMzNrip17WiG1QXyI7AriSeAsYD5wOHADMKGeAZqZWXP0mCCARcBVwIyIWJsrXyLp2/UJy8zMmq1MgpgYEVG0ICK+0s/xmJlZiyjTSP0TSSMqM5L2krSwjjGZmVkLKJMgRkXEhspMRDwLvLp+IZmZWSsokyC2SBpXmZF0AFBY5WRmZgNHmQRxHvC/kq6S9D3g58CsMjuXNE3SKkmrJZ1bsHycpNslLZW0TNL0VD5U0nclLZf0kKRS72dmZv2nx0bqiLhF0hHAG1PRpyPi6Z62kzQEuBh4J7AWWCxpfkSszK12PnB9RFwiaRKwABgPvB/YNSIOk7QbsFLSNRGxphefzczMdkDZwfq2AE8BzwGTJL21xDZHAqsj4tGI2AhcC5xYtU4Aw9P0nsC6XPnuknYGOoCN6b3NzKxBytwo9zfAp4CxwP1kVxKLgON62HQM8Hhufi1wVNU6s8l6SZ0F7A68I5XPJUsmTwC7AZ+JiGcKYjsTOBNg3Lhx1YvNzGwHlLmC+BTwBuA3EfE2YDKwofYmpZ0KXBERY4HpwFWSdiK7+tgC7Ed2p/ZnJb2meuOIuDQipkTElFGjRvVTSGZmBuUSxB8j4o8AknaNiF8CE0ts1wnsn5sfm8ryPgZcDxARi4BhwEjgg8AtEbEpIp4C7gSmlHhPMzPrJ2USxNp0o9w84FZJNwO/KbHdYuAgSRMk7QKcQjaGU95jwNsBJB1CliDWp/LjUvnuZNVavyzxnmZm1k/K9GJ6b5qcLel2ssbkW0pst1nSJ4GFwBDg8ohYIelCYElEzAc+C1wm6TNkDdMfiYiQdDHwn5JWAAL+MyKW9eUDmplZ39RMEKmr6oqI+DOAiLijNzuPiAVkXVfzZRfkplcCf1Gw3QtkXV3NzKxJalYxRcQWYFX+TmozMxscyozmuhewQtI9wIuVwog4oW5RmZlZ05VJEJ+vexRmZtZyyjRS96rdwczMBoYyd1I/z8ujt+4CDAVejIjh3W9lZmbtrswVxB6VaUkiGwLjjd1vYWZmA0HZwfoAiMw8YGqd4jEzsxZRporppNzsTmRDXvyxbhGZmVlLKNOL6fjc9GZgDa8cttvMzAaYMm0QpzciEDMzay09tkGkR3+OyM3vJeny+oZlZmbNVqaR+nURse35DxHxLNkzIczMbAArkyB2krRXZUbS3pRruzAzszZW5ov+a8AiSTek+fcDF9UvJDMzawVlGqmvlLSEl59BfVIaptvMzAawMvdBvJHsmRDfSvPDJR0VEb+oe3RmbWje0k7mLFzFug1d7Deig5lTJzJj8phmh2XWa2XaIC4BXsjNv5DKzKzKvKWdzLpxOZ0bugigc0MXs25czryl1Y9jN2t9ZdogFBGVwfqIiK2S3EhtVmDOwlV0bdqyXVnXpi2cM3cZ19zzWJOisoFs5RPPMWl0fcZOLXMF8aik/ytpaHp9Cni0LtGYtbl1G7oKyzdu2drgSGywmDR6OCceXp8qzDJXAp8AvgmcTzbs923AGXWJxqzN7Teig86CJDFmRAfXffzoJkRk1nc9XkFExFMRcUpEvDoi9gU+Bhxb98jM2tDMqRPpGDpku7KOoUOYOXVikyIy67tSw31LGiJpuqSrgF8Df1nfsMza04zJY/jSSYexy5DsX2vMiA6+dNJh7sVkbalmFZOkY4APAtOBe4C/AF4TEX9oQGxmbWnG5DHbGqRdrWTtrNsEIWkt8BhZl9bPRcTzkn7t5GBmNjjUqmKaC+xHVp10vKTdefnZ1GZmNsB1myAi4tPABLKxmI4FVgGjJH1A0qsaE56ZmTVLzUbq9Azq2yPiTLJkcSrZ0+TWNCA2MzNrotJ3REfEJuCHwA8lddQvJDMzawWlurlWi4ji20XNzGzA6FOCMDOzgc+D7pkNYB563HZEmedBHAzMBA7Irx8Rx3W7kZk1XWXo8crospWhxwEnCSulTBXTDcB9ZIP1zcy9eiRpmqRVklZLOrdg+ThJt0taKmmZpOm5Za+TtEjSCknLJQ0r95HMDLofenzOwlVNisjaTZkqps0R0esHBEkaAlwMvBNYCyyWNL/qcaXnA9dHxCWSJgELgPHpeRPfA/46Ih6QtA+wqbcxmA1m3Q093l25WbUyVxA/kPR3kkZL2rvyKrHdkcDqiHg0IjYC15LdQ5EXQOVJF3sC69L0u4BlEfEAQET8LiK2YGal7TeiuDd6d+Vm1cokiNPIqpTuAu5NryUlthsDPJ6bX5vK8mYDH0rjPi0AzkrlBwMhaaGk+ySdU/QGks6UtETSkvXr15cIyWzw8NDjtqN6rGKKiAl1fP9TgSsi4muSjgaukvTaFNebgTcAfwBuk3RvRNxWFdulwKUAU6ZM8ThRZjmVhmj3YrK+KtOLaSjwt8BbU9HPgP+X7qyupRPYPzc/NpXlfQyYBhARi1JD9Eiyq42fR8TTKYYFwBFkT7Mzs5JmTB7jhGB9VqaK6RLg9cC/p9frU1lPFgMHSZogaRfgFGB+1TqPAW8HkHQIMAxYDywEDpO0W2qwPgZYiZmZNUyZXkxviIg/z83/VNIDPW0UEZslfZLsy34IcHlErJB0IbAkIuYDnwUuk/QZsgbrj0REAM9K+jpZkglgQUT8qHcfzczMdkSZBLFF0p9GxCMAkl4DlOpRFBELyBqf82UX5KZXkj2lrmjb75F1dTUzsyYokyBmArdLehQQ2R3Vp9c1KjMza7oyvZhuk3QQUOkbtyoiXqpvWGZm1my1nkl9XET8VNJJVYsOlERE3Fjn2MzMrIlqXUEcA/wUOL5gWQBOEGZmA1i3CSIivpAmL4yIX+eXSarnzXNmZtYCytwH8V8FZXP7OxAzM2sttdog/gw4FNizqh1iONkNbWZmNoDVaoOYCLwHGMH27RDPA2fUMygzM2u+Wm0QNwM3Szo6IhY1MCYzM2sBZW6UWyrp78mqm7ZVLUXER+sWlZmZNV2ZRuqrgD8BpgJ3kI3K+nw9gzIzs+YrkyAOjIjPAy9GxHeB/wMcVd+wzMys2cokiMpzHzakh/nsCby6fiGZmVkrKNMGcamkvYDPkz3P4VXABbU3MTOzdldmsL7vpMk7gNfUNxwzM2sVtW6UO7vWhhHx9f4Px6rNW9rpZwpb2/Hf7cBQ6wpij/RzIvAGXn5c6PHAPfUMyjLzlnYy68bldG3Kns/UuaGLWTcuB/A/m7Us/90OHLVulPsnAEk/B46IiOfT/GzAj/9sgDkLV237J6vo2rSFc+Yu45p7HmtSVFbGyieeY9Lo4c0Ooym6+7uds3CVE0SbKdOLaV9gY25+YyqzOlu3oauwfOOWrQ2OxHpr0ujhnHj44Pwy7O7vtrtya11lejFdCdwj6aY0PwO4om4R2Tb7jeigs+CfasyIDq77+NFNiMisZ9393e43oqMJ0diO6PEKIiIuInsG9bPpdXpEfKnegRnMnDqRjqFDtivrGDqEmVMndrOFWfP573bgqNWLaXhEPCdpb2BNelWW7R0Rz9Q/vMGtUl97ztxlbNyylTHuDWJtoPL36V5M7a9WFdPVZMN930v2iNEKpXnfE9EAMyaP2dYgPdirldx1sn3MmDzGv5sBoFYvpvekn368qDWdu06aNV6tKqYjam0YEff1fzhWT+18Bu6uk2aNV6uK6Ws1lgVwXD/HYnXU7mfg7jpp1ni1qpje1shArL7a/QzcXSfNGq/MjXJIeq2kD0j6cOVV78Csf7X7Gbi7Tpo1Xo83ykn6AnAsMAlYALwb+F+yG+isTbT7Gbi7Tpo1Xpk7qU8G/hxYGhGnS9oX+F59w7L+NnPqxO3aIKD9zsDdddKsscokiK6I2Cpps6ThwFPA/nWOy/qZz8DNrLfKJIglkkYAl5HdNPcCsKiuUVld+AzczHqj1n0QFwNXR8TfpaJvS7oFGB4RyxoSnZk1VTvfO2M7rlYvpoeBf5W0RtJXJU2OiDW9SQ6SpklaJWm1pHMLlo+TdLukpZKWSZpesPwFSZ8r/5HMrD9U7p3p3NBF8PK9M/OWdjY7NGuQbhNERHwjIo4GjgF+B1wu6ZeSviDp4J52LGkIcDFZr6dJwKmSJlWtdj5wfURMBk4B/r1q+deBH5f+NGbWb2rdO2ODQ5nhvn8TEV9JX+Knkj0P4qES+z4SWB0Rj0bERuBa4MTq3QOVx27tCayrLJA0A/g1sKLEe5lZP2v3e2dsx/WYICTtLOl4Sd8nO5tfBZxUYt9jgMdz82tTWd5s4EOS1pLdY3FWes9XAf8A/FMPsZ0paYmkJevXry8RkpmV1d09Mu1y74ztuG4ThKR3Srqc7Iv9DLLnUP9pRJwSETf30/ufClwREWOB6cBVknYiSxz/FhEv1No4Ii6NiCkRMWXUqFH9FJKZge9et9rdXGeRPRPisxHxbB/23cn290uMTWV5HwOmAUTEIknDgJHAUcDJkr4KjAC2SvpjRHyrD3GYWR/43hmrNVjfjo7Wuhg4SNIEssRwCvDBqnUeA94OXCHpEGAYsD4i3lJZQdJs4AUnh/bgbpEDi++dGdzK3CjXJxGxWdIngYXAEODyiFgh6UJgSUTMBz4LXCbpM2QN1h+JiOh+r9bK2n1IcTPbXt0SBEBELCBrfM6XXZCbXgn8RQ/7mF2X4BKf8fafdh9S3My2V9cE0ep8xtu/3C3SbGAZ1AmiuzPec+Yu45p7HmtSVK+08onnmDR6eM8rNlm7DyluZtsr9cCggaq7M9uNW7Y2OJLaJo0ezomHt/4VjbtFmg0sg/oKorsz3jEjOrju40c3IaL25m6RZgPLoE4QA+EhOq3G3SLNBo5BnSB8xmtm1r1BnSDAZ7zWutwF25pt0CcIs1bkLtjWCgZ1LyazVuVnMVgrcIIwa0G+6dBagROEWQvysxisFThBmLUg33RorcCN1GYtyF2wrRU4QZi1KHfBtmZzFZOZmRVygjAzs0JOEGZmVsgJwszMCjlBmJlZIScIMzMr5ARhZmaFnCDMzKyQE4SZmRVygjAzs0JOEGZmVsgJwszMCjlBmJlZIScIMzMr5ARhZmaFnCDMzKyQHxjUR/OWdvppX2Y2oDlB9MG8pZ3MunE5XZu2ANC5oYtZNy4HcJIwswGjrlVMkqZJWiVptaRzC5aPk3S7pKWSlkmansrfKeleScvTz+PqGWdvzVm4altyqOjatIU5C1c1KSIzs/5XtysISUOAi4F3AmuBxZLmR8TK3GrnA9dHxCWSJgELgPHA08DxEbFO0muBhUDLnJqv29DVq3Izs3ZUzyuII4HVEfFoRGwErgVOrFongOFpek9gHUBELI2Idal8BdAhadc6xtor+43o6FW5mVk7qmeCGAM8nptfyyuvAmYDH5K0luzq4ayC/bwPuC8iXqpeIOlMSUskLVm/fn3/RF3CzKkT6Rg6ZLuyjqFDmDl1YsNiMDOrt2Z3cz0VuCIixgLTgaskbYtJ0qHAV4CPF20cEZdGxJSImDJq1KiGBAxZQ/SXTjqMMSM6EDBmRAdfOukwN1Cb2YBSz15MncD+ufmxqSzvY8A0gIhYJGkYMBJ4StJY4CbgwxHxSB3j7JMZk8c4IZjZgFbPK4jFwEGSJkjaBTgFmF+1zmPA2wEkHQIMA9ZLGgH8CDg3Iu6sY4xmZtaNuiWIiNgMfJKsB9JDZL2VVki6UNIJabXPAmdIegC4BvhIRETa7kDgAkn3p9er6xWrmZm9krLv4/Y3ZcqUWLJkSbPDMDNrK5LujYgpRcua3UhtZmYtykNt2KDmMbXMuucEYYOWx9Qyq81VTDZoeUwts9qcIGzQ8phaZrU5Qdig5TG1zGpzgrBBy2NqmdXmRmobtCoN0e7FZFbMCcIGNY+pZdY9VzGZmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFRoww31LWg/8ptlx9GAk8HSzgyjJsdaHY62fdoq3lWI9ICIKn9k8YBJEO5C0pLtx11uNY60Px1o/7RRvu8TqKiYzMyvkBGFmZoWcIBrr0mYH0AuOtT4ca/20U7xtEavbIMzMrJCvIMzMrJAThJmZFXKC6CeS9pd0u6SVklZI+lQqny2pU9L96TU9t80sSaslrZI0tQkxr5G0PMW1JJXtLelWSb9KP/dK5ZL0zRTvMklHNDDOibnjd7+k5yR9ulWOraTLJT0l6cFcWa+Po6TT0vq/knRaA2OdI+mXKZ6bJI1I5eMldeWO77dz27w+/e2sTp9HDYq1179zSdNS2WpJ5/Z3nDVivS4X5xpJ96fyph7XXokIv/rhBYwGjkjTewAPA5OA2cDnCtafBDwA7ApMAB4BhjQ45jXAyKqyrwLnpulzga+k6enAjwEBbwR+0aTjPAT4LXBAqxxb4K3AEcCDfT2OwN7Ao+nnXml6rwbF+i5g5zT9lVys4/PrVe3nnhS/0ud5d4Ni7dXvPL0eAV4D7JLWmdSIWKuWfw24oBWOa29evoLoJxHxRETcl6afBx4Caj1o4ETg2oh4KSJ+DawGjqx/pD06Efhumv4uMCNXfmVk7gZGSBrdhPjeDjwSEbXumm/osY2InwPPFMTQm+M4Fbg1Ip6JiGeBW4FpjYg1In4SEZvT7N3A2Fr7SPEOj4i7I/tWu5KXP19dY62hu9/5kcDqiHg0IjYC16Z1GxZrugr4AHBNrX006rj2hhNEHUgaD0wGfpGKPpku3y+vVDWQJY/Hc5utpXZCqYcAfiLpXklnprJ9I+KJNP1bYN803QrxApzC9v9orXpse3scWyFmgI+SnblWTJC0VNIdkt6SysaQxVfR6Fh78ztvheP6FuDJiPhVrqwVj+srOEH0M0mvAv4L+HREPAdcAvwpcDjwBNmlZqt4c0QcAbwb+HtJb80vTGcxLdMPWtIuwAnADamolY/tNq12HLsj6TxgM/D9VPQEMC4iJgNnA1dLGt6s+JK2+J1XOZXtT2pa8bgWcoLoR5KGkiWH70fEjQAR8WREbImIrcBlvFzV0Qnsn9t8bCprmIjoTD+fAm5KsT1ZqTpKP59Kqzc9XrJEdl9EPAmtfWzp/XFsasySPgK8B/irlNBI1TW/S9P3ktXlH5ziyldDNSzWPvzOm31cdwZOAq6rlLXice2OE0Q/SfWM/wE8FBFfz5Xn6+nfC1R6OcwHTpG0q6QJwEFkDVSNind3SXtUpskaKh9McVV60JwG3JyL98OpF84bgd/nqlAaZbszsVY9trkYenMcFwLvkrRXqjZ5VyqrO0nTgHOAEyLiD7nyUZKGpOnXkB3HR1O8z0l6Y/q7/3Du89U71t7+zhcDB0makK5AT0nrNso7gF9GxLaqo1Y8rt1qZgv5QHoBbyarRlgG3J9e04GrgOWpfD4wOrfNeWRnD6tocG8Fsl4dD6TXCuC8VL4PcBvwK+C/gb1TuYCLU7zLgSkNjnd34HfAnrmylji2ZEnrCWATWb3xx/pyHMnq/1en1+kNjHU1WT195e/222nd96W/jfuB+4Djc/uZQvbl/AjwLdKoDA2Itde/8/R/+HBadl6jjmsqvwL4RNW6TT2uvXl5qA0zMyvkKiYzMyvkBGFmZoWcIMzMrJAThJmZFXKCMDOzQk4Q1hYk/ZukT+fmF0r6Tm7+a5LOrrH9FZJOTtM/k/SKB8ZLGirpy8pGU71P0iJJ707L1kga2Ye4t71vN8svTiN6rqwa4fNkSQuURlbtT5JGS/phjeW7SPp5usnLBjEnCGsXdwJvApC0EzASODS3/E3AXTv4Hl8kG5X3tZENQTKDbGTeuomIv4+Iw8n66j8SEYen19yImB4RG+rwtmeT3YXcXUwbye7h+Ms6vLe1EScIaxd3AUen6UPJbiZ6Pt15vCtwCHCfpAskLZb0oKRLy46nL2k34AzgrIh4CbYN63B9wbpnp/0/WHVV8+E0iNwDkq4q2O6L6YpiSMmY1kgaqez5Ab9M2z4s6fuS3iHpznS1c2Raf3dlA9jdkwaC627U0vcBt6RtDk3r359iPyitMw/4qzJx2sDlS0hrCxGxTtJmSePIrhYWkY10eTTwe2B5RGyU9K2IuBAgfUm/B/hBibc4EHgssgEWuyXp9cDpwFFkd0X/QtIdwEbgfOBNEfG0pL2rtptDdjVyevTt7tQDgfeT3W29GPgg2d37JwD/SHa1cx7w04j4aKqaukfSf0fEi7k4JgDPVpIg8AngGxHx/TQURSV5PQi8oQ9x2gDiKwhrJ3eRJYdKgliUm78zrfM2Sb+QtBw4ju2rofrDm4GbIuLFiHgBuJFsOOfjgBsi4mmAiMg/G+DzZEOEfKKPyQHg1xGxPLJB6lYAt6V9LSd7AA1k4zedq+zJZT8DhgHjqs322KYAAAHASURBVPYzGlifm18E/KOkfwAOiIiuFP8WYKPSeF02ODlBWDuptEMcRnaGezfZFcSbgLskDQP+HTg5Ig4jq2cfVnLfq4Fxqs+wy4uB11dfVfTSS7nprbn5rbxcEyDgfbl2jHER8VDVfrrIHZOIuJrsKqQLWCDpuNy6uwJ/3IGYrc05QVg7uYusyuiZyIZ8fgYYQZYk7uLlL76nlT2Xo9veQ9UiG8X0P4BvpKqWyqib769a9X+AGZJ2UzYK7ntT2U+B90vaJ22bTwa3AF8GflTnM/KFwFmVdhdJkwvWeZiXrzgqo4k+GhHfJBs59HWpfB/g6YjYVMd4rcU5QVg7WU7We+nuqrLfR8TTqcfPZWRXFwvJztx743yy6peVyh4+/0NguzaJyB4rewXZUNK/AL4TEUsjYgVwEXCHpAeAr1dtd0OKbb6kjl7GVdYXgaHAMkkr0vx2UnvEI5IOTEUfAB5M1VKvJXvMJcDbgB/VKU5rEx7N1WyQkfRe4PURcX6NdW4Ezo2IhxsXmbUa92IyG2Qi4qZKVViRVMU2z8nBfAVhZmaF3AZhZmaFnCDMzKyQE4SZmRVygjAzs0JOEGZmVuj/A37tMhoK6ZryAAAAAElFTkSuQmCC\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "plt.title('Learning Curve')\n", + "plt.xlabel('Wall Clock Time (s)')\n", + "plt.ylabel('Validation Accuracy')\n", + "print(len(valid_loss_history))\n", + "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", + "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xudzM73mTjhI" + }, + "source": [ + "## 3. Model selection" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A3gC3u_E4cO1" + }, + "source": [ + "Given a dataset, which language model should you use for the fine tuning? It appears this is a simple question: just choose the best model according to the benchmarks such as [GLUE](https://gluebenchmark.com/leaderboard). However, we will see that under the resource constraints, the model selection is non trivial. \n", + "\n", + "In this example, we will tune the [spooky-author-identification](https://www.kaggle.com/competitions/spooky-author-identification/data?select=train.zip) dataset from kaggle. You can download the dataset from the website and upload it to Colab. We run FLAML for 30 mins using bert." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HjvdojhfTjhI", + "outputId": "c8848ff9-1ce3-4632-84aa-0a8199a7fce9" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[flaml.automl: 08-19 14:50:41] {2540} INFO - task = seq-classification\n", + "INFO:flaml.automl:task = seq-classification\n", + "[flaml.automl: 08-19 14:50:41] {2542} INFO - Data split method: stratified\n", + "INFO:flaml.automl:Data split method: stratified\n", + "[flaml.automl: 08-19 14:50:41] {2545} INFO - Evaluation method: holdout\n", + "INFO:flaml.automl:Evaluation method: holdout\n", + "[flaml.automl: 08-19 14:50:41] {2664} INFO - Minimizing error metric: 1-accuracy\n", + "INFO:flaml.automl:Minimizing error metric: 1-accuracy\n", + "[flaml.automl: 08-19 14:50:41] {2806} INFO - List of ML learners in AutoML Run: ['transformer']\n", + "INFO:flaml.automl:List of ML learners in AutoML Run: ['transformer']\n", + "[flaml.automl: 08-19 14:50:41] {3108} INFO - iteration 0, current learner transformer\n", + "INFO:flaml.automl:iteration 0, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'eval_loss': 0.8782516717910767, 'eval_automl_metric': 0.3650663942798774, 'eval_runtime': 63.6528, 'eval_samples_per_second': 76.902, 'eval_steps_per_second': 76.902, 'epoch': 0.3}\n", + "{'train_runtime': 136.3771, 'train_samples_per_second': 32.302, 'train_steps_per_second': 1.012, 'train_loss': 0.9700310748556386, 'epoch': 0.3}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_14-50-41/train_4ba0c0a8_18_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_14-50-41/checkpoint-138/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_14-50-41/train_4ba0c0a8_18_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_14-50-41/checkpoint-138/vocab.txt\n", + "loading file data/output/train_2022-08-19_14-50-41/train_4ba0c0a8_18_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_14-50-41/checkpoint-138/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_14-50-41/train_4ba0c0a8_18_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_14-50-41/checkpoint-138/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_14-50-41/train_4ba0c0a8_18_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_14-50-41/checkpoint-138/tokenizer_config.json\n", + "[flaml.automl: 08-19 14:54:42] {3242} INFO - Estimated sufficient time budget=2413758s. Estimated necessary time budget=2414s.\n", + "INFO:flaml.automl:Estimated sufficient time budget=2413758s. Estimated necessary time budget=2414s.\n", + "[flaml.automl: 08-19 14:54:42] {3294} INFO - at 241.5s,\testimator transformer's best error=0.3651,\tbest estimator transformer's best error=0.3651\n", + "INFO:flaml.automl: at 241.5s,\testimator transformer's best error=0.3651,\tbest estimator transformer's best error=0.3651\n", + "[flaml.automl: 08-19 14:54:42] {3108} INFO - iteration 1, current learner transformer\n", + "INFO:flaml.automl:iteration 1, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'eval_loss': 0.9422562122344971, 'eval_automl_metric': 0.4482124616956078, 'eval_runtime': 64.093, 'eval_samples_per_second': 76.373, 'eval_steps_per_second': 76.373, 'epoch': 0.3}\n", + "{'train_runtime': 142.1563, 'train_samples_per_second': 30.988, 'train_steps_per_second': 0.485, 'train_loss': 1.0089939504429914, 'epoch': 0.3}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_14-54-42/train_db826be0_19_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_14-54-42/checkpoint-69/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_14-54-42/train_db826be0_19_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_14-54-42/checkpoint-69/vocab.txt\n", + "loading file data/output/train_2022-08-19_14-54-42/train_db826be0_19_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_14-54-42/checkpoint-69/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_14-54-42/train_db826be0_19_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_14-54-42/checkpoint-69/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_14-54-42/train_db826be0_19_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_14-54-42/checkpoint-69/tokenizer_config.json\n", + "[flaml.automl: 08-19 14:58:51] {3294} INFO - at 490.1s,\testimator transformer's best error=0.3651,\tbest estimator transformer's best error=0.3651\n", + "INFO:flaml.automl: at 490.1s,\testimator transformer's best error=0.3651,\tbest estimator transformer's best error=0.3651\n", + "[flaml.automl: 08-19 14:58:51] {3108} INFO - iteration 2, current learner transformer\n", + "INFO:flaml.automl:iteration 2, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'eval_loss': 0.764643669128418, 'eval_automl_metric': 0.30684371807967314, 'eval_runtime': 64.3046, 'eval_samples_per_second': 76.122, 'eval_steps_per_second': 76.122, 'epoch': 0.3}\n", + "{'train_runtime': 139.6474, 'train_samples_per_second': 31.545, 'train_steps_per_second': 1.976, 'train_loss': 0.9045784438865773, 'epoch': 0.3}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_14-58-51/train_6fc6d930_20_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_14-58-51/checkpoint-276/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_14-58-51/train_6fc6d930_20_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_14-58-51/checkpoint-276/vocab.txt\n", + "loading file data/output/train_2022-08-19_14-58-51/train_6fc6d930_20_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_14-58-51/checkpoint-276/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_14-58-51/train_6fc6d930_20_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_14-58-51/checkpoint-276/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_14-58-51/train_6fc6d930_20_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_14-58-51/checkpoint-276/tokenizer_config.json\n", + "[flaml.automl: 08-19 15:02:56] {729} WARNING - checkpoint data/output/train_2022-08-19_14-50-41/train_4ba0c0a8_18_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_14-50-41/checkpoint-138 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-19_14-50-41/train_4ba0c0a8_18_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_14-50-41/checkpoint-138 not found\n", + "[flaml.automl: 08-19 15:02:56] {3294} INFO - at 735.2s,\testimator transformer's best error=0.3068,\tbest estimator transformer's best error=0.3068\n", + "INFO:flaml.automl: at 735.2s,\testimator transformer's best error=0.3068,\tbest estimator transformer's best error=0.3068\n", + "[flaml.automl: 08-19 15:02:56] {3108} INFO - iteration 3, current learner transformer\n", + "INFO:flaml.automl:iteration 3, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'eval_loss': 0.6895061731338501, 'eval_automl_metric': 0.26414708886619, 'eval_runtime': 64.1612, 'eval_samples_per_second': 76.292, 'eval_steps_per_second': 76.292, 'epoch': 0.3}\n", + "{'train_runtime': 137.9967, 'train_samples_per_second': 31.923, 'train_steps_per_second': 2.0, 'train_loss': 0.8616765340169271, 'epoch': 0.3}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_15-02-56/train_01e528ee_21_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-02-56/checkpoint-276/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_15-02-56/train_01e528ee_21_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-02-56/checkpoint-276/vocab.txt\n", + "loading file data/output/train_2022-08-19_15-02-56/train_01e528ee_21_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-02-56/checkpoint-276/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_15-02-56/train_01e528ee_21_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-02-56/checkpoint-276/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_15-02-56/train_01e528ee_21_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-02-56/checkpoint-276/tokenizer_config.json\n", + "[flaml.automl: 08-19 15:07:00] {729} WARNING - checkpoint data/output/train_2022-08-19_14-58-51/train_6fc6d930_20_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_14-58-51/checkpoint-276 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-19_14-58-51/train_6fc6d930_20_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_14-58-51/checkpoint-276 not found\n", + "[flaml.automl: 08-19 15:07:00] {3294} INFO - at 979.1s,\testimator transformer's best error=0.2641,\tbest estimator transformer's best error=0.2641\n", + "INFO:flaml.automl: at 979.1s,\testimator transformer's best error=0.2641,\tbest estimator transformer's best error=0.2641\n", + "[flaml.automl: 08-19 15:07:00] {3108} INFO - iteration 4, current learner transformer\n", + "INFO:flaml.automl:iteration 4, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'loss': 0.7586, 'learning_rate': 4.688468079515019e-06, 'epoch': 0.54}\n", + "{'eval_loss': 0.4876616895198822, 'eval_automl_metric': 0.18447395301327885, 'eval_runtime': 64.0236, 'eval_samples_per_second': 76.456, 'eval_steps_per_second': 76.456, 'epoch': 1.0}\n", + "{'train_runtime': 312.4704, 'train_samples_per_second': 46.993, 'train_steps_per_second': 2.938, 'train_loss': 0.6536963469062755, 'epoch': 1.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_15-07-00/train_933f400e_22_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-07-00/checkpoint-918/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_15-07-00/train_933f400e_22_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-07-00/checkpoint-918/vocab.txt\n", + "loading file data/output/train_2022-08-19_15-07-00/train_933f400e_22_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-07-00/checkpoint-918/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_15-07-00/train_933f400e_22_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-07-00/checkpoint-918/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_15-07-00/train_933f400e_22_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-07-00/checkpoint-918/tokenizer_config.json\n", + "[flaml.automl: 08-19 15:13:57] {729} WARNING - checkpoint data/output/train_2022-08-19_15-02-56/train_01e528ee_21_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-02-56/checkpoint-276 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-19_15-02-56/train_01e528ee_21_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-02-56/checkpoint-276 not found\n", + "[flaml.automl: 08-19 15:13:57] {3294} INFO - at 1396.3s,\testimator transformer's best error=0.1845,\tbest estimator transformer's best error=0.1845\n", + "INFO:flaml.automl: at 1396.3s,\testimator transformer's best error=0.1845,\tbest estimator transformer's best error=0.1845\n", + "[flaml.automl: 08-19 15:13:57] {3409} INFO - selected model: None\n", + "INFO:flaml.automl:selected model: None\n", + "[flaml.automl: 08-19 15:13:57] {2837} INFO - fit succeeded\n", + "INFO:flaml.automl:fit succeeded\n", + "[flaml.automl: 08-19 15:13:57] {2839} INFO - Time taken to find the best model: 1396.3099913597107\n", + "INFO:flaml.automl:Time taken to find the best model: 1396.3099913597107\n", + "[flaml.automl: 08-19 15:13:57] {2853} WARNING - Time taken to find the best model is 78% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n", + "WARNING:flaml.automl:Time taken to find the best model is 78% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" + ] + } + ], + "source": [ + "import flaml\n", + "from flaml import AutoML\n", + "import pandas as pd\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "df = pd.read_csv('spooky-author-identification.csv')\n", + "X, y = df.drop('author', axis=1), df['author']\n", + "\n", + "X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=123)\n", + "automl_model = AutoML()\n", + "\n", + "automl_settings = {\n", + " \"time_budget\": 1800, \n", + " \"task\": \"seq-classification\", \n", + " \"fit_kwargs_by_estimator\": {\n", + " \"transformer\": {\n", + " \"output_dir\": \"data/output/\", \n", + " \"model_path\": \"bert-base-uncased\", \n", + " }\n", + " },\n", + " \"metric\": \"accuracy\",\n", + " \"gpu_per_trial\": 1, \n", + " \"log_file_name\": \"spooky_bert.log\", \n", + " \"log_type\": \"all\", \n", + " \"use_ray\": False, # set whether to use Ray\n", + " \"n_concurrent_trials\": 1,\n", + " \"keep_search_state\": True, # keeping the search state\n", + "}\n", + "\n", + "from flaml import tune\n", + "custom_hp = {\n", + " \"transformer\": {\n", + " \"num_train_epochs\": {\n", + " \"domain\": tune.choice([0.3, 1, 2, 3, 4, 5]),\n", + " \"init_value\": 0.3, \n", + " \"low_cost_init_value\": 0.3,\n", + " },\n", + " }\n", + "}\n", + "\n", + "automl_model.fit(X_train=X_train, y_train=y_train,X_val=X_val, y_val=y_val, custom_hp=custom_hp, **automl_settings)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9jZiKSU75jjl" + }, + "source": [ + "The job ran for 23m and searched for 4 trials. This time is shorter than our budget 30m because FLAML early stops the last trial which will run for too long. If you want to run for longer time, set a larger time budget. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "xpA-rzYzTjhI", + "outputId": "bacf6804-5ae5-4cea-ee01-be4f35f5c90f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "the best loss for spooky author identification: 0.18447395301327885\n" + ] + } + ], + "source": [ + "print(\"the best loss for spooky author identification: {}\".format(automl_model.best_loss))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TzDjaBTA6ZaD" + }, + "source": [ + "Next, we set the model to roberta and run again. RoBERTa outperforms BERT by 15% on the [SuperGLUE](https://super.gluebenchmark.com/) benchmark, as well as [GLUE](https://gluebenchmark.com/), [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/), [RACE](https://www.cs.cmu.edu/~glai1/data/race/), etc. Does this mean we should always use RoBERTa and never use BERT? To answer this question, we run the same experiment again with RoBERTa:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6MTZCJz1TjhJ", + "outputId": "8adde438-ec14-44d2-f174-c549deb44729" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[flaml.automl: 08-19 15:21:09] {2540} INFO - task = seq-classification\n", + "INFO:flaml.automl:task = seq-classification\n", + "[flaml.automl: 08-19 15:21:09] {2542} INFO - Data split method: stratified\n", + "INFO:flaml.automl:Data split method: stratified\n", + "[flaml.automl: 08-19 15:21:09] {2545} INFO - Evaluation method: holdout\n", + "INFO:flaml.automl:Evaluation method: holdout\n", + "[flaml.automl: 08-19 15:21:09] {2664} INFO - Minimizing error metric: 1-accuracy\n", + "INFO:flaml.automl:Minimizing error metric: 1-accuracy\n", + "[flaml.automl: 08-19 15:21:09] {2806} INFO - List of ML learners in AutoML Run: ['transformer']\n", + "INFO:flaml.automl:List of ML learners in AutoML Run: ['transformer']\n", + "[flaml.automl: 08-19 15:21:09] {3108} INFO - iteration 0, current learner transformer\n", + "INFO:flaml.automl:iteration 0, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'eval_loss': 0.7796056866645813, 'eval_automl_metric': 0.3170582226762002, 'eval_runtime': 65.2086, 'eval_samples_per_second': 75.067, 'eval_steps_per_second': 75.067, 'epoch': 0.3}\n", + "{'train_runtime': 139.2884, 'train_samples_per_second': 31.626, 'train_steps_per_second': 0.991, 'train_loss': 0.9700887928838315, 'epoch': 0.3}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_15-21-09/train_8d9e8af4_24_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_15-21-09/checkpoint-138/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_15-21-09/train_8d9e8af4_24_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_15-21-09/checkpoint-138/vocab.json\n", + "loading file data/output/train_2022-08-19_15-21-09/train_8d9e8af4_24_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_15-21-09/checkpoint-138/merges.txt\n", + "loading file data/output/train_2022-08-19_15-21-09/train_8d9e8af4_24_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_15-21-09/checkpoint-138/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_15-21-09/train_8d9e8af4_24_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_15-21-09/checkpoint-138/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_15-21-09/train_8d9e8af4_24_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_15-21-09/checkpoint-138/tokenizer_config.json\n", + "[flaml.automl: 08-19 15:25:18] {3242} INFO - Estimated sufficient time budget=2487757s. Estimated necessary time budget=2488s.\n", + "INFO:flaml.automl:Estimated sufficient time budget=2487757s. Estimated necessary time budget=2488s.\n", + "[flaml.automl: 08-19 15:25:18] {3294} INFO - at 249.0s,\testimator transformer's best error=0.3171,\tbest estimator transformer's best error=0.3171\n", + "INFO:flaml.automl: at 249.0s,\testimator transformer's best error=0.3171,\tbest estimator transformer's best error=0.3171\n", + "[flaml.automl: 08-19 15:25:18] {3108} INFO - iteration 1, current learner transformer\n", + "INFO:flaml.automl:iteration 1, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'eval_loss': 1.0274657011032104, 'eval_automl_metric': 0.5805924412665986, 'eval_runtime': 65.4388, 'eval_samples_per_second': 74.803, 'eval_steps_per_second': 74.803, 'epoch': 0.3}\n", + "{'train_runtime': 141.3162, 'train_samples_per_second': 31.173, 'train_steps_per_second': 0.488, 'train_loss': 1.0675905752873076, 'epoch': 0.3}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_15-25-18/train_21eb5d04_25_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_15-25-18/checkpoint-69/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_15-25-18/train_21eb5d04_25_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_15-25-18/checkpoint-69/vocab.json\n", + "loading file data/output/train_2022-08-19_15-25-18/train_21eb5d04_25_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_15-25-18/checkpoint-69/merges.txt\n", + "loading file data/output/train_2022-08-19_15-25-18/train_21eb5d04_25_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_15-25-18/checkpoint-69/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_15-25-18/train_21eb5d04_25_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_15-25-18/checkpoint-69/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_15-25-18/train_21eb5d04_25_s=9223372036854775807,e=9.7119e-06,s=-1,s=0.3,e=64,d=14_2022-08-19_15-25-18/checkpoint-69/tokenizer_config.json\n", + "[flaml.automl: 08-19 15:29:30] {3294} INFO - at 500.5s,\testimator transformer's best error=0.3171,\tbest estimator transformer's best error=0.3171\n", + "INFO:flaml.automl: at 500.5s,\testimator transformer's best error=0.3171,\tbest estimator transformer's best error=0.3171\n", + "[flaml.automl: 08-19 15:29:30] {3108} INFO - iteration 2, current learner transformer\n", + "INFO:flaml.automl:iteration 2, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'eval_loss': 0.6534864902496338, 'eval_automl_metric': 0.27436159346271705, 'eval_runtime': 65.6703, 'eval_samples_per_second': 74.539, 'eval_steps_per_second': 74.539, 'epoch': 0.3}\n", + "{'train_runtime': 142.8116, 'train_samples_per_second': 30.846, 'train_steps_per_second': 1.933, 'train_loss': 0.8491502291914346, 'epoch': 0.3}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_15-29-30/train_b7eb3874_26_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_15-29-30/checkpoint-276/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_15-29-30/train_b7eb3874_26_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_15-29-30/checkpoint-276/vocab.json\n", + "loading file data/output/train_2022-08-19_15-29-30/train_b7eb3874_26_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_15-29-30/checkpoint-276/merges.txt\n", + "loading file data/output/train_2022-08-19_15-29-30/train_b7eb3874_26_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_15-29-30/checkpoint-276/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_15-29-30/train_b7eb3874_26_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_15-29-30/checkpoint-276/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_15-29-30/train_b7eb3874_26_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_15-29-30/checkpoint-276/tokenizer_config.json\n", + "[flaml.automl: 08-19 15:33:42] {729} WARNING - checkpoint data/output/train_2022-08-19_15-21-09/train_8d9e8af4_24_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_15-21-09/checkpoint-138 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-19_15-21-09/train_8d9e8af4_24_s=9223372036854775807,e=1e-05,s=-1,s=0.3,e=32,d=20_2022-08-19_15-21-09/checkpoint-138 not found\n", + "[flaml.automl: 08-19 15:33:42] {3294} INFO - at 753.0s,\testimator transformer's best error=0.2744,\tbest estimator transformer's best error=0.2744\n", + "INFO:flaml.automl: at 753.0s,\testimator transformer's best error=0.2744,\tbest estimator transformer's best error=0.2744\n", + "[flaml.automl: 08-19 15:33:42] {3108} INFO - iteration 3, current learner transformer\n", + "INFO:flaml.automl:iteration 3, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'eval_loss': 0.586589515209198, 'eval_automl_metric': 0.2402451481103166, 'eval_runtime': 65.9616, 'eval_samples_per_second': 74.21, 'eval_steps_per_second': 74.21, 'epoch': 0.3}\n", + "{'train_runtime': 140.9957, 'train_samples_per_second': 31.243, 'train_steps_per_second': 1.958, 'train_loss': 0.7886253025220789, 'epoch': 0.3}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_15-33-42/train_4e5c88d0_27_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-33-42/checkpoint-276/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_15-33-42/train_4e5c88d0_27_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-33-42/checkpoint-276/vocab.json\n", + "loading file data/output/train_2022-08-19_15-33-42/train_4e5c88d0_27_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-33-42/checkpoint-276/merges.txt\n", + "loading file data/output/train_2022-08-19_15-33-42/train_4e5c88d0_27_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-33-42/checkpoint-276/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_15-33-42/train_4e5c88d0_27_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-33-42/checkpoint-276/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_15-33-42/train_4e5c88d0_27_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-33-42/checkpoint-276/tokenizer_config.json\n", + "[flaml.automl: 08-19 15:37:54] {729} WARNING - checkpoint data/output/train_2022-08-19_15-29-30/train_b7eb3874_26_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_15-29-30/checkpoint-276 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-19_15-29-30/train_b7eb3874_26_s=9223372036854775807,e=1.0297e-05,s=-1,s=0.3,e=16,d=26_2022-08-19_15-29-30/checkpoint-276 not found\n", + "[flaml.automl: 08-19 15:37:54] {3294} INFO - at 1004.8s,\testimator transformer's best error=0.2402,\tbest estimator transformer's best error=0.2402\n", + "INFO:flaml.automl: at 1004.8s,\testimator transformer's best error=0.2402,\tbest estimator transformer's best error=0.2402\n", + "[flaml.automl: 08-19 15:37:54] {3108} INFO - iteration 4, current learner transformer\n", + "INFO:flaml.automl:iteration 4, current learner transformer\n", + "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " FutureWarning,\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'loss': 0.7223, 'learning_rate': 4.688468079515019e-06, 'epoch': 0.54}\n", + "{'eval_loss': 0.4903346300125122, 'eval_automl_metric': 0.1953013278855975, 'eval_runtime': 65.2412, 'eval_samples_per_second': 75.029, 'eval_steps_per_second': 75.029, 'epoch': 1.0}\n", + "{'train_runtime': 310.9644, 'train_samples_per_second': 47.221, 'train_steps_per_second': 2.952, 'train_loss': 0.624375353711363, 'epoch': 1.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "***** Running Prediction *****\n", + " Num examples = 4895\n", + " Batch size = 1\n", + "Didn't find file data/output/train_2022-08-19_15-37-54/train_e47787a2_28_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-37-54/checkpoint-918/added_tokens.json. We won't load it.\n", + "loading file data/output/train_2022-08-19_15-37-54/train_e47787a2_28_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-37-54/checkpoint-918/vocab.json\n", + "loading file data/output/train_2022-08-19_15-37-54/train_e47787a2_28_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-37-54/checkpoint-918/merges.txt\n", + "loading file data/output/train_2022-08-19_15-37-54/train_e47787a2_28_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-37-54/checkpoint-918/tokenizer.json\n", + "loading file None\n", + "loading file data/output/train_2022-08-19_15-37-54/train_e47787a2_28_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-37-54/checkpoint-918/special_tokens_map.json\n", + "loading file data/output/train_2022-08-19_15-37-54/train_e47787a2_28_s=9223372036854775807,e=1.0297e-05,s=-1,s=1,e=16,d=26_2022-08-19_15-37-54/checkpoint-918/tokenizer_config.json\n", + "[flaml.automl: 08-19 15:44:56] {729} WARNING - checkpoint data/output/train_2022-08-19_15-33-42/train_4e5c88d0_27_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-33-42/checkpoint-276 not found\n", + "WARNING:flaml.automl:checkpoint data/output/train_2022-08-19_15-33-42/train_4e5c88d0_27_s=9223372036854775807,e=1.48e-05,s=-1,s=0.3,e=16,d=25_2022-08-19_15-33-42/checkpoint-276 not found\n", + "[flaml.automl: 08-19 15:44:56] {3294} INFO - at 1426.8s,\testimator transformer's best error=0.1953,\tbest estimator transformer's best error=0.1953\n", + "INFO:flaml.automl: at 1426.8s,\testimator transformer's best error=0.1953,\tbest estimator transformer's best error=0.1953\n", + "[flaml.automl: 08-19 15:44:56] {3409} INFO - selected model: None\n", + "INFO:flaml.automl:selected model: None\n", + "[flaml.automl: 08-19 15:44:56] {2837} INFO - fit succeeded\n", + "INFO:flaml.automl:fit succeeded\n", + "[flaml.automl: 08-19 15:44:56] {2839} INFO - Time taken to find the best model: 1426.8331220149994\n", + "INFO:flaml.automl:Time taken to find the best model: 1426.8331220149994\n", + "[flaml.automl: 08-19 15:44:56] {2853} WARNING - Time taken to find the best model is 79% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n", + "WARNING:flaml.automl:Time taken to find the best model is 79% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" + ] + } + ], + "source": [ + "automl_settings[\"fit_kwargs_by_estimator\"][\"transformer\"][\"model_path\"] = \"roberta-base\"\n", + "automl_settings[\"log_file_name\"] = \"spooky_roberta.log\"\n", + "automl_model.fit(X_train=X_train, y_train=y_train,X_val=X_val, y_val=y_val, custom_hp=custom_hp, **automl_settings)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MknjX6ij7lpX" + }, + "source": [ + "We plot the performance of BERT and RoBERTa w.r.t. the wall clock time. We find that although RoBERTa frequently outperforms BERT on benchmark datasets, its performance on the spooky-author-identification dataset is worse than BERT using the same time budget. Therefore, model selection is a non trivial problem. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 300 + }, + "id": "IHqpFgG3TjhJ", + "outputId": "dcd3f094-1689-4ebf-d796-55c127ba048c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5\n", + "5\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeXUlEQVR4nO3df3RV5Z3v8fenKUrGi4BCFUkE7EJGRQU9MnjpDzsOQlsVnVoLY1vtL8bbYltnFlVu77IO/TFVutpO12LGOpbqmtUKXC7F1DqLWluvq61WQlWUaDSiown2NoJgrUEgfO8feyccDifkBE5ycnY+r7XOyt7PfvbJ87DDJzt77/M8igjMzCy73lbpBpiZWf9y0JuZZZyD3sws4xz0ZmYZ56A3M8u4t1e6AYXGjBkTEydOrHQzzMyqysaNG1+NiLHFtg26oJ84cSKNjY2VboaZWVWR9F89bfOlGzOzjHPQm5llnIPezCzjBt01+mL27NlDa2sru3btqnRTBqXhw4dTV1fHsGHDKt0UMxuEqiLoW1tbGTFiBBMnTkRSpZszqEQE27Zto7W1lUmTJlW6OWY2CFVF0O/atcsh3wNJHH/88bS3t1e6KWbWg3WPtbFsfTNbd3Rw0qhaFs+ZwmXTxw/Y96+KoAcc8ofgfxuzwWvdY20sWfskHXs6AWjb0cGStU8CDFjY+2asmVk/Wra+uTvku3Ts6WTZ+uYBa4ODvkQvvvgiU6dOPez9161bR1NTUxlbZGbVYOuOjj6V9wcH/QDYu3evg95siDppVG2fyvuDg74P9u7dy1VXXcVpp53GFVdcwZtvvsnGjRt573vfy7nnnsucOXN45ZVXALjgggv44he/SC6X45ZbbqGhoYHFixczbdo0nn/++Qr3xMwGyuI5U6gdVnNAWe2wGhbPmTJgbaiam7Fd/umnm2na+npZ3/P0k47lK5ec0Wu95uZmfvCDHzBr1iw++clPsnz5cn7yk59wzz33MHbsWFatWsWXv/xlVqxYAcDu3bu7x+157rnnuPjii7niiivK2nYzG9y6brh+ac0mdnfuY3yxp242rYYHlsLOVhhZBxfeBGddWbY2VF3QV1J9fT2zZs0C4KMf/Sjf+MY3eOqpp5g9ezYAnZ2djBs3rrv+Rz7ykYq008wGl8umj+fuR18CYNXfn3/gxk2r4aefhz3pNfudLyfrULawLynoJc0F/gWoAe6IiG8WbD8ZuAsYlda5MSLuS7ctAT4FdAKfj4j1R9LgUs68+0vhY4wjRozgjDPO4OGHHy5a/5hjjhmIZplZNXtg6f6Q77KnIykvU9D3eo1eUg2wHHg/cDqwQNLpBdX+F7A6IqYD84F/Tfc9PV0/A5gL/Gv6flXppZde6g71H//4x8ycOZP29vbusj179rB58+ai+44YMYI//elPA9ZWM6sSO1v7Vn4YSrkZOwNoiYgtEbEbWAnMK6gTwLHp8khga7o8D1gZEW9FxAtAS/p+VWnKlCksX76c0047jddee43rrruONWvWcMMNN3D22Wczbdo0fvvb3xbdd/78+Sxbtozp06f7ZqyZ7Teyrm/lh6GUSzfjgZfz1luBvyqoczPwc0nXAccAf5O37yMF+x70UTBJC4GFACeffHIp7R5wEydO5JlnnjmofNq0aTz00EMHlT/44IMHrM+aNcuPV5rZwS686cBr9ADDapPyMinX45ULgDsjog74APAfkkp+74i4PSJyEZEbO7boTFhmZtl01pVwyfeg5uhkfWR9sj7AT920AfV563VpWb5PkVyDJyIeljQcGFPivmZmQ9tZV8LGu5LlT/ys7G9fyln3BmCypEmSjiK5udpQUOcl4EIASacBw4H2tN58SUdLmgRMBh4tV+PNzKx3vZ7RR8ReSYuA9SSPTq6IiM2SlgKNEdEA/CPw75KuJ7kxe01EBLBZ0mqgCdgLfC4iOot/JzMz6w8lPUefPhN/X0HZTXnLTcCsHvb9OvD1I2ijmZkdAY91Y2aWcQ76MnrwwQe5+OKLj+g97rzzTrZu3dp7RTOzEjnoD0NEsG/fvrK/b2dnp4PezMrOQV+iF198kSlTpvDxj3+cqVOn8qlPfYqpU6dy5plnsmrVqu56r7/+Oh/84AeZMmUK1157bfcvhJ///Oecf/75nHPOOXz4wx/mjTfeAJIPYt1www2cc8453H333TQ2NnLVVVcxbdo0Ojo6WLp0Keeddx5Tp05l4cKFJPe4zcxKV32jV/7njfCHJ8v7nieeCe//Zq/VnnvuOe666y7a2tq47bbbeOKJJ3j11Vc577zzeM973gPAo48+SlNTExMmTGDu3LmsXbuWCy64gK997Wv84he/4JhjjuGWW27h29/+NjfdlNzPPv744/n9738PwB133MG3vvUtcrkcAIsWLequ97GPfYx7772XSy65pLz9N7NMq76gr6AJEyYwc+ZMrr/+ehYsWEBNTQ0nnHAC733ve9mwYQPHHnssM2bM4JRTTgFgwYIF/PrXv2b48OE0NTV1D3G8e/duzj9//1ClhxrO+Fe/+hW33norb775Jtu3b+eMM85w0JtZn1Rf0Jdw5t1fShl2uHAoY0lEBLNnz+buu+/u0/vu2rWLz372szQ2NlJfX8/NN9/Mrl27+t5wMxvSfI3+MLz73e9m1apVdHZ20t7ezkMPPcSMGcmgnI8++igvvPAC+/btY9WqVbzrXe9i5syZ/OY3v6GlpQWAP//5zzz77LNF3zt/OOOuUB8zZgxvvPEGa9asGYDemVnWVN8Z/SBw+eWX8/DDD3P22WcjiVtvvZUTTzyRZ555hvPOO49FixbR0tLC+973Pi6//HLe9ra3ceedd7JgwQLeeustAL72ta9x6qmnHvTe11xzDddeey21tbU8/PDDfOYzn2Hq1KmceOKJnHfeeQPdVTPLAA22pzhyuVx0zbPa5emnn+a0006rUIuqg/+NzAa3j3w/maDooKkEu/zwg8nXwxzUTNLGiMgV2+ZLN2ZmGeegN8uiTavhO1Ph5lHJ102rK90iq6CquUYfEQc90WKJwXb5zSps0+oDZyza+XKyDmWdzMKqR1UE/fDhw9m2bRvHH3+8w75ARLBt2zaGDx9e6abYYPHA0gOnpYNk/Z5F+ye3sAF307adycIPRxav8Icnkw9v9oOqCPq6ujpaW1tpb2+vdFMGpeHDh1NXV76JhK3K7WwtXt751sC2w/rmxDPhzCv65a2rIuiHDRvGpEmTKt0Ms+owsi65XHNQeX2/TFNnpVna9dTNJ3p46qYflXQzVtJcSc2SWiTdWGT7dyQ9nr6elbQjb1tn3rbCKQjNrNwuvAmG1R5YNqw2Kbchqdczekk1wHJgNtAKbJDUkM4qBUBEXJ9X/zpget5bdETEtPI12cwOqeuG6z2Lkss1I+uTkPeN2CGrlEs3M4CWiNgCIGklMI9kHthiFgBfKU/zzOywnHXl/huvvlwz5JVy6WY8kH/BrzUtO4ikCcAk4Jd5xcMlNUp6RNJlPey3MK3T6BuuZmblVe4PTM0H1kREZ17ZhPRjuX8HfFfSOwt3iojbIyIXEbmxY8eWuUlmZkNbKUHfBtTnrdelZcXMBw4Yizci2tKvW4AHOfD6vZmZ9bNSgn4DMFnSJElHkYT5QU/PSPpLYDTwcF7ZaElHp8tjgFn0fG3fzMz6Qa83YyNir6RFwHqgBlgREZslLQUaI6Ir9OcDK+PAz+OfBnxf0j6SXyrfzH9ax8wO37rH2li2vpmtOzo4aVQti+dM4bLpRW+f2RBX0gemIuI+4L6CspsK1m8ust9vgf75TK/ZELbusTaWrH2Sjj3J7bC2HR0sWZvMpeywt0JV8clYMzvQsvXN3SHfpWNPJ19as4m7H30J2D+2StcnMq2yml55ndPHHVuR7+1his2q0NYdHUXLd3fuG+CWWKlOH3cs86ZV5q8tn9GbVaGTRtXSViTsx4+q3T+DUTpKYiXGVrHBxWf0lj1DYNKNxXOmUDus5oCy2mE1LJ4zpUItssHMZ/SWLUNk0o2uG65fWrOJ3Z37GO+nbuwQHPSWLUNo0o3LgMnHJDdczzhhJDxO8urSjxNZWHXxpRvLFk+6sV8/TmRh1cVn9JYtQ2zSjUpOZmHVw2f0li2edMPsIA56y5azroRLvgc1RyfrI+uT9QzdiDXrK1+6sezxpBtmB/AZvZlZxjnozcwyzkFvZpZxDnozs4xz0JuZZVxJQS9prqRmSS2Sbiyy/TuSHk9fz0rakbftaknPpa+ry9l4MzPrXa+PV0qqAZYDs4FWYIOkhvwpASPi+rz615FOAC7pOOArQA4IYGO672tl7YWZmfWolDP6GUBLRGyJiN3ASmDeIeovAO5Ol+cA90fE9jTc7wfmHkmDzcysb0r5wNR4IH/wkFbgr4pVlDQBmAT88hD7HjSOqqSFwEKAk08+uYQmmSU8QbZZ78r9ydj5wJqI6Oy1Zp6IuB24HSCXy0WZ22QZdcgJsivZMLNBppSgbwPq89br0rJi5gOfK9j3goJ9Hyy9eWY9O9QE2V3jtGd9YuxKTjht1aOUa/QbgMmSJkk6iiTMGworSfpLYDSQ/z9rPXCRpNGSRgMXpWVmR8wTZFd2wmmrHr2e0UfEXkmLSAK6BlgREZslLQUaI6Ir9OcDKyMi8vbdLumrJL8sAJZGxPbydsGGqkNNkH3GCZ4Y26xLSdfoI+I+4L6CspsK1m/uYd8VwIrDbF/fbVqdTCe3szWZhOLCmzxEbUYtnjPlgGv0kDdB9uOH2NFsiMnWMMVDZGJoSxxygmwHvVm3bAX9EJoY2hI9TpDtibHNumVrrBtPDG1dPDG2WbdsndEPsYmhLeEJss0OLVtn9J4Y2szsINkK+q6JoUfWA/LE0GZmZO3SDSSh7mA3M+uWrTN6MzM7iIPezCzjHPRmZhnnoDczyzgHvZlZxjnozcwyzkFvZpZxDnozs4xz0JuZZVxJQS9prqRmSS2SbuyhzpWSmiRtlvTjvPJOSY+nr4OmIDQzs/7V6xAIkmqA5cBsoBXYIKkhIpry6kwGlgCzIuI1Se/Ie4uOiJhW5nabmVmJSjmjnwG0RMSWiNgNrATmFdT5DLA8Il4DiIg/lreZZmZ2uEoJ+vFA/iDvrWlZvlOBUyX9RtIjkubmbRsuqTEtv6zYN5C0MK3T2N7e3qcOmJnZoZVr9Mq3A5OBC4A64CFJZ0bEDmBCRLRJOgX4paQnI+L5/J0j4nbgdoBcLhdlapOZmVHaGX0bUJ+3XpeW5WsFGiJiT0S8ADxLEvxERFv6dQvwIDD9CNtsZmZ9UErQbwAmS5ok6ShgPlD49Mw6krN5JI0huZSzRdJoSUfnlc8CmjAzswHT66WbiNgraRGwHqgBVkTEZklLgcaIaEi3XSSpCegEFkfENkn/Hfi+pH0kv1S+mf+0Tjmte6yNZeub2bqjg5NG1bJ4zhQum154K8HMbOgp6Rp9RNwH3FdQdlPecgD/kL7y6/wWOPPIm3lo6x5rY8naJ+nY0wlA244Olqx9EsBhb2ZDXiamEly2vrk75Lt07OnkS2s2cfejL1WoVTZQml55ndPHHVvpZpgNWpkYAmHrjo6i5bs79w1wS6wSTh93LPOm+S83s55k4oz+pFG1tBUJ+/Gjaln19+dXoEVmZoNHJs7oF8+ZQu2wmgPKaofVsHjOlAq1yMxs8MjEGX3XDVc/dWNmdrBMBD0kYe9gNzM7WCYu3ZiZWc8c9GZmGeegNzPLOAe9mVnGOejNzDLOQW9mlnEOejOzjHPQm5llnIPezCzjHPRmZhlXUtBLmiupWVKLpBt7qHOlpCZJmyX9OK/8aknPpa+ry9VwMzMrTa9j3UiqAZYDs0kmAd8gqSF/SkBJk4ElwKyIeE3SO9Ly44CvADkggI3pvq+VvytmZlZMKWf0M4CWiNgSEbuBlcC8gjqfAZZ3BXhE/DEtnwPcHxHb0233A3PL03QzMytFKUE/Hng5b701Lct3KnCqpN9IekTS3D7si6SFkholNba3t5feejMz61W5bsa+HZgMXAAsAP5d0qhSd46I2yMiFxG5sWPHlqlJZmYGpQV9G1Cft16XluVrBRoiYk9EvAA8SxL8pexrZmb9qJSg3wBMljRJ0lHAfKChoM46krN5JI0huZSzBVgPXCRptKTRwEVpmZmZDZBen7qJiL2SFpEEdA2wIiI2S1oKNEZEA/sDvQnoBBZHxDYASV8l+WUBsDQitvdHR8zMrDhFRKXbcIBcLheNjY2VboaZWVWRtDEicsW2+ZOxZmYZ56A3M8s4B72ZWcY56M3MMs5Bb2aWcQ56M7OMc9CbmWWcg97MLOMc9GZmGeegNzPLOAe9mVnGOejNzDLOQW9mlnEOejOzjHPQm5llXK8Tj9jgt+6xNpatb2brjg5OGlXL4jlTuGz6QXOwm9kQVdIZvaS5kpoltUi6scj2ayS1S3o8fX06b1tnXnnhFIR2hNY91saStU/StqODANp2dLBk7ZOse8xT85pZotczekk1wHJgNskk4BskNUREU0HVVRGxqMhbdETEtCNvqhWzbH0zHXs6Dyjr2NPJsvXNPqs3M6C0M/oZQEtEbImI3cBKYF7/NstKtXVHR5/KzWzoKSXoxwMv5623pmWFPiRpk6Q1kurzyodLapT0iKTLin0DSQvTOo3t7e2lt944aVRtn8rNbOgp11M3PwUmRsRZwP3AXXnbJqQT1v4d8F1J7yzcOSJuj4hcROTGjh1bpiYNDYvnTKF2WM0BZbXDalg8Z0qFWmRmg00pQd8G5J+h16Vl3SJiW0S8la7eAZybt60t/boFeBCYfgTttQKXTR/PP//tmYwfVYuA8aNq+ee/PdPX582sWymPV24AJkuaRBLw80nOzrtJGhcRr6SrlwJPp+WjgTcj4i1JY4BZwK3larwlLps+3sFuZj3qNegjYq+kRcB6oAZYERGbJS0FGiOiAfi8pEuBvcB24Jp099OA70vaR/LXwzeLPK1jZmb9SBFR6TYcIJfLRWNjY6WbYWZWVSRtTO+HHsRDIJiZZZyD3sws4xz0ZmYZ56A3M8s4B72ZWcY56M3MMs5Bb2aWcQ56M7OMc9CbmWWcg97MLOMc9GZmGeegNzPLOAe9mVnGOejNzDLOQW9mlnEOejOzjCsp6CXNldQsqUXSjUW2XyOpXdLj6evTeduulvRc+rq6nI03M7Pe9TqVoKQaYDkwG2gFNkhqKDIl4KqIWFSw73HAV4AcEMDGdN/XytJ6MzPrVSln9DOAlojYEhG7gZXAvBLffw5wf0RsT8P9fmDu4TXVzMwORylBPx54OW+9NS0r9CFJmyStkVTfl30lLZTUKKmxvb29xKabmVkpynUz9qfAxIg4i+Ss/a6+7BwRt0dELiJyY8eOLVOTzMwMSgv6NqA+b70uLesWEdsi4q109Q7g3FL3NTOz/lVK0G8AJkuaJOkoYD7QkF9B0ri81UuBp9Pl9cBFkkZLGg1clJaZmdkA6fWpm4jYK2kRSUDXACsiYrOkpUBjRDQAn5d0KbAX2A5ck+67XdJXSX5ZACyNiO390A8zM+uBIqLSbThALpeLxsbGSjfDzKyqSNoYEbli2/zJWDOzjHPQm5llnIPezCzjHPRmZhnnoDczyzgHvZlZxjnozcwyzkFvZpZxDnozs4xz0JuZZZyD3sws4xz0ZmYZ56A3M8s4B72ZWcY56M3MMq6koJc0V1KzpBZJNx6i3ockhaRcuj5RUoekx9PXbeVquJmZlabXGaYk1QDLgdlAK7BBUkNENBXUGwF8AfhdwVs8HxHTytReMzPro1LO6GcALRGxJSJ2AyuBeUXqfRW4BdhVxvaZmdkRKiXoxwMv5623pmXdJJ0D1EfEz4rsP0nSY5L+r6R3H35TzczscPR66aY3kt4GfJt0QvACrwAnR8Q2SecC6ySdERGvF7zHQmAhwMknn3ykTTIzszylnNG3AfV563VpWZcRwFTgQUkvAjOBBkm5iHgrIrYBRMRG4Hng1MJvEBG3R0QuInJjx449vJ6YmVlRpQT9BmCypEmSjgLmAw1dGyNiZ0SMiYiJETEReAS4NCIaJY1Nb+Yi6RRgMrCl7L0wM7Me9XrpJiL2SloErAdqgBURsVnSUqAxIhoOsft7gKWS9gD7gGsjYns5Gm5mZqVRRFS6DQfI5XLR2NhY6WaYmVUVSRsjIldsmz8Za2aWcQ56M7OMc9CbmWWcg97MLOMc9GZmGeegNzPLOAe9mVnGOeizZtNq+M5UuHlU8nXT6kq3yMwq7IgHNbNBZNNq+OnnYU9Hsr7z5WQd4KwrK9cuM6son9FnyQNL94d8lz0dSbmZDVkO+izZ2dq3cjMbEhz0WTKyrm/lZjYkOOiz5MKbYFjtgWXDapNyMxuyHPRZctaVcMn3YGQ9oOTrJd/zjVizIc5P3WTNWVc62M3sAD6jNzPLuJKCXtJcSc2SWiTdeIh6H5IUknJ5ZUvS/ZolzSlHo83MrHS9XrpJ53xdDswGWoENkhoioqmg3gjgC8Dv8spOJ5lj9gzgJOAXkk6NiM7ydcHMzA6llDP6GUBLRGyJiN3ASmBekXpfBW4BduWVzQNWRsRbEfEC0JK+n5mZDZBSgn488HLeemta1k3SOUB9RPysr/uamVn/OuKnbiS9Dfg2cM0RvMdCYGG6+oak5oIqY4BXD/f9B5ks9QWy1R/3ZfDKUn/6qy8TetpQStC3AfV563VpWZcRwFTgQUkAJwINki4tYV8AIuJ24PaeGiCpsafZzatNlvoC2eqP+zJ4Zak/lehLKZduNgCTJU2SdBTJzdWGro0RsTMixkTExIiYCDwCXBoRjWm9+ZKOljQJmAw8WvZemJlZj3o9o4+IvZIWAeuBGmBFRGyWtBRojIiGQ+y7WdJqoAnYC3zOT9yYmQ2skq7RR8R9wH0FZUUHUImICwrWvw58/TDb16XHyzpVKEt9gWz1x30ZvLLUnwHviyJioL+nmZkNIA+BYGaWcQ56M7OMq3jQS6qX9CtJTZI2S/pCWn6cpPslPZd+HZ2WS9L30vFzNqUf1hpUJNVIekzSven6JEm/S9u8Kn16ifRppFVp+e8kTaxku4uRNErSGknPSHpa0vnVemwkXZ/+jD0l6W5Jw6vp2EhaIemPkp7KK+vzsZB0dVr/OUlXD6K+LEt/zjZJ+omkUXnbio6ZpRLH4epvxfqTt+0flYwBNiZdH/hjExEVfQHjgHPS5RHAs8DpwK3AjWn5jcAt6fIHgP8EBMwEflfpPhTp0z8APwbuTddXA/PT5duA/5Eufxa4LV2eD6yqdNuL9OUu4NPp8lHAqGo8NiSfyH4BqM07JtdU07EB3gOcAzyVV9anYwEcB2xJv45Ol0cPkr5cBLw9Xb4lry+nA08ARwOTgOdJngCsSZdPSX82nwBOHyzHJi2vJ3li8b+AMZU6NhX9we3hH+wekgHUmoFxadk4oDld/j6wIK9+d73B8CL5UNgDwF8D96YH89W8H+DzgfXp8nrg/HT57Wk9VboPeX0ZmYajCsqr7tiwfziO49J/63uBOdV2bICJBeHYp2MBLAC+n1d+QL1K9qVg2+XAj9LlJcCSvG3r02PVfbyK1RsM/QHWAGcDL7I/6Af82FT80k2+9M/j6SQjYJ4QEa+km/4AnJAuD/bxc74LfAnYl64fD+yIiL3pen57u/uSbt+Z1h8sJgHtwA/TS1F3SDqGKjw2EdEGfAt4CXiF5N96I9V7bLr09VgM2mNU4JMkZ71QpX2RNA9oi4gnCjYNeH8GTdBL+m/A/wG+GBGv52+L5NfboH8OVNLFwB8jYmOl21Imbyf5c/TfImI68GeSywPdqujYjCYZTXUSyZDZxwBzK9qoMquWY9EbSV8m+YDljyrdlsMl6S+A/wkMigmbB0XQSxpGEvI/ioi1afH/kzQu3T4O+GNaXtL4ORUyC7hU0oskwzn/NfAvwChJXR9Oy29vd1/S7SOBbQPZ4F60Aq0R0TXHwBqS4K/GY/M3wAsR0R4Re4C1JMerWo9Nl74ei8F8jJB0DXAxcFX6iwuqsy/vJDmpeCLNgzrg95JOpAL9qXjQSxLwA+DpiPh23qYGoOuu89Uk1+67yj+e3rmeCezM+9O1oiJiSUTURTLmz3zglxFxFfAr4Iq0WmFfuvp4RVp/0JyRRcQfgJclTUmLLiQZzqLqjg3JJZuZkv4i/Znr6ktVHps8fT0W64GLJI1O/8q5KC2rOElzSS57XhoRb+Zt6mnMrEOOw1VJEfFkRLwj9o8B1kry0MkfqMSxqdSNi7wbDu8i+XNzE/B4+voAyfXQB4DngF8Ax6X1RTLj1fPAk0Cu0n3ooV8XsP+pm1NIfjBbgP8NHJ2WD0/XW9Ltp1S63UX6MQ1oTI/POpKnAary2AD/BDwDPAX8B8lTHFVzbIC7Se4v7CEJjk8dzrEguf7dkr4+MYj60kJyjborB27Lq//ltC/NwPvzyj9A8qTe88CXB9OxKdj+Ivtvxg74sfEQCGZmGVfxSzdmZta/HPRmZhnnoDczyzgHvZlZxjnozcwyzkFvZpZxDnozs4z7/xQNBu15VctMAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from flaml.data import get_output_from_log\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "for each_file_name in ['bert', 'roberta']:\n", + " time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", + " get_output_from_log(filename='spooky_' + each_file_name + '.log', time_budget=3000)\n", + " print(len(valid_loss_history))\n", + " plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", + " plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", + "\n", + "plt.legend(['bert', 'roberta'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lT7IwNCoTjhJ" + }, + "source": [ + "## 4. Other Tasks" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Fzkr77iATjhJ" + }, + "source": [ + "Besides sequence classification, FLAML currently also supports four other tasks (more tasks are to be supported, which can be found on [FLAML's documentation website] (https://microsoft.github.io/FLAML/docs/Examples/AutoML-NLP)):\n", + "\n", + "- sequence regression: predicting a float number from the input sequence, e.g., predicting the rating of a hotel review based on the text content;\n", + "- token classification: predicting the label of each token in a sequence, e.g., named entity recognition;\n", + "- multiple choice: predicting the best second half of a sentence that comes next to the first part of a sentence based on common sensen reasoning. An example is seen below;\n", + "- (abstractive) summarization: generating the textual summarization of an input paragraph;\n", + "\n", + "Here we look into two tasks: multiple choice classification and text summarization. These tasks require significant computational resources, therefore instead of Colab, we run them using 4 NVIDIA V100 GPUs and Ray Tune on our server." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Y4VgUR5TTjhJ" + }, + "source": [ + "### 4.1 Multiple Choice Example" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OO8GqaH3TjhJ" + }, + "source": [ + "Multiple choice is a task of predicting the best second half of a sentence that follows the first half based on common sense reasoning. An example of multiple-choice classification problem is:\n", + "\n", + "On stage, a woman takes a seat at the piano. She\n", + "a) sits on a bench as her sister plays with the doll.\n", + "b) smiles with someone as the music plays.\n", + "c) is in the crowd, watching the dancers.\n", + "d) *nervously sets her fingers on the keys*." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 382, + "referenced_widgets": [ + "3588f07c45694ec4a484afaaa9e9c599", + "998b0ca5b37b47b88ea47327462c76fa", + "38bb77cefa2e4c17b8e9c419125d6c45", + "ded6921a6b8140b3bcd59d0e7bbd7900", + "ad994aff0bf94c2ea4ac9aa8d5c067e3", + "6be493d86857493190ee47a08c04ff40", + "46e340fe82414a58b283c78cdf953773", + "760feb714cc54846a52fc399703891d7", + "a21f2bb483e44c749ec41a2b1784ee4b", + "9cf2c0d8439a4f5a86b4769a27babb94", + "007a43463f5e4da3983f59dfeb793e64", + "2e00672f9d1f46cea3e5db651bca19a3", + "cc378c1990634f7da6ffba019fe38c59", + "642323b1bafa4d0fbeca1adff2426c02", + "a902290681e942cbae40024baaa2e9b7", + "3a014eef1b7d44698572bc5cada4cb8c", + "c1792bbceb854dc5880003f64e5623cb", + "b167b817426e4832b73a7a37b72115c1", + "5ea642008bf74641a021e17b7e3fd6e7", + "2d975d14c3f0434583e73ae97f580951", + "26f3abaf861a4c63986ff0691294d70c", + "1a701b0fa5a34fb9b64fb92b5c8e4306", + "7dc52faf4b3b4643b7d7019f1722c1d8", + "44cf4e612b5f482d8bf224413c1bc852", + "f4ec7bc190af4c9dbe6a5fc05fad4540", + "9dd6ab0e0cb940bebe25cba5492b2486", + "e556c049fbb24669a49b26c7f107e6a5", + "805c722b7dec4fc59ef64f8704e29424", + "0c3a0eb88b16493e9d0f62e3d5abf195", + "bfbcfceca0444337ac6c4033a7734fc1", + "5ad8132df42340c58f1375b1e52eb5bc", + "aa1c4b91b583440a8e6f79dd06cfd200", + "b37bd95afbe44cd196fc5ab2d52bccd0", + "b649c32bcc8446cf91c53604fc1dcaa6", + "b70e2d813be3473f90ca76e293251c0b", + "b66e17d6fd094f44bc10eade34fc5261", + "9509ad27a9b54e3e80f796d224f3e189", + "920c8dd736a4454f9469fb3fa0a9af90", + "8f5311cebd554f5ba645b8d33b0722a3", + "aad092fcb29d4045a342288aa9d6a329", + "811d914b52904fa0913adc9daa33695c", + "98f179b9be5044c79bc867f5261e2b47", + "18091d361aa44881a3db5d1951882082", + "2a1aa694683d4df9b509f5ce4d6d53b0", + "f74dfe0a3de64c3ea051e14fba9a04e4", + "37d4912ed8ee4c0c9f0a9187bad156fd", + "6284429508d849bd8259460913efc250", + "e1f77bef878c4b0bbfac867c5a9eea98", + "20624397998c4e188b419c6267affb65", + "3b6eaa3d64924ec581c412b04b9196fa", + "94a2d9480adc426abb4ade344ca8dd2f", + "d4830572efa244968881c31932ec5dff", + "21f848683b2648c08a7476658d382177", + "5a392cc22ee84433bac08ef8a6a3e0d4", + "98bcaff9e28547e3b1f9b0640d598f99", + "3b684b9f50ce48ff92b075d62619368b", + "a3e42e8e532c4628abaa6e154d667ed2", + "49290455306b47aaaf8153bed5e49742", + "d8468fc2f0b94b2b8dab75336a0d29a3", + "b09d990f98f0419e84f5939d3b48d381", + "4b92faf53c2b4f7986066af8026ffc3a", + "8497fe93c0d148a49f9a0a0c56961f36", + "d226d72577ea4cc299ed78c2fa99a486", + "82e0dbe1328a4c54807932984e0c4efb", + "eb395373be244e6e8815087d5d32a801", + "1ffd6e8c1f834dc48d66116b6089f7a2", + "155d7e95c2504507b83b12dc60f1edc1", + "feaf1f712b8e499db2558dc0fdd4261e", + "51392c27affa4fd3b4184cde01b7029d", + "5dd6914461ea456e9dc96ccf8c391c6e", + "069feb62f1ec4392b04ee1d80aa4b445", + "62bdec145ccb48faa4fe5f51d2879732", + "c96613989db447b5acfb35cfef553145", + "db589666b507409f9647930b1222b0a9", + "1fed6cdd71c4453b976e8651b7b34cae", + "f307d9be05d24940b57d9edc82be8976", + "20abb46ea9c948b8ba85a921aee8af6d", + "ca550fa3fe1147bd8285c2b7cadde206", + "263e99ca21124b79a26e1078b187273d", + "894de256daec49329f6404326eddaa39", + "7feeefa264da4af89ddc8ddf331b4f9f", + "4b2b38b8064040849b10c63b9f2ed8fd", + "a91dc7ddd7f641d9b60b59bbbde7bae6", + "96c6874b9bd045a5bc67596b2ab04df2", + "57aca5124cc14ed69da5a0b24a2c1052", + "8f2c2b10e21e42569ef5396e42c65e30", + "be21476b15a14c0084712a9d5aedc22f", + "362f58e6d05f4d0c865cbe6a956d677b", + "c274c717ac7e4fa2888e0d101c3fe1fb", + "9cbec0a1fe3247ed8a46290df56756fa", + "35151380719b41349a2113b0b893bd6f", + "1fb6335656b1444abe05aa94a7d13825", + "a6431dfb76084a838d63849fa362de35", + "26885bbae6c646e7bcc4a0459620c37a", + "a56b058d910244588c1454b02c8cda8c", + "efca4a3072e94170a1b851f9dec6164d", + "791101442898470f8524ebee4cb9459d", + "6a24dd061b2d40d4baca4036059fc94c", + "9fdc731eb58247cbafef9286b49c66f9", + "4fa9926221cb4d29bc0cc0c3d0bf93f3", + "4e956bd06d3a45eca66b192990416a62", + "e319f91c3ba841f99a9a1ca1c7b551f2", + "da31e023dddb4c25a035258c0e4ed0d7", + "574c7a42dad940379a96b9f0968d3be1", + "ab2c48f34b7f43c5a5fb37c80a7d47f3", + "834bdf06646a4d009648b6bc270c7624", + "2e0b939889d84c07a90d36a57065aac4", + "929cdd7c2f8e4902aac96a9a3afa5866", + "87639f90c2ab47db986419c03e165d7a", + "a18b9c22460940cfa53b657849b034bf" + ] + }, + "id": "hQ5fX0N3TjhJ", + "outputId": "e1701a84-daad-4e70-82ac-c2c14d718793" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "No config specified, defaulting to: swag/regular\n", + "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n", + "No config specified, defaulting to: swag/regular\n", + "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n", + "No config specified, defaulting to: swag/regular\n", + "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "73546\n" + ] + } + ], + "source": [ + "from datasets import load_dataset\n", + "\n", + "train_dataset = load_dataset(\"swag\", split=\"train\").to_pandas()\n", + "dev_dataset = load_dataset(\"swag\", split=\"validation\").to_pandas()\n", + "test_dataset = load_dataset(\"swag\", split=\"test\").to_pandas()\n", + "\n", + "custom_sent_keys = [\n", + " \"sent1\",\n", + " \"sent2\",\n", + " \"ending0\",\n", + " \"ending1\",\n", + " \"ending2\",\n", + " \"ending3\",\n", + " \"gold-source\",\n", + " \"video-id\",\n", + " \"startphrase\",\n", + " \"fold-ind\",\n", + " ] # specify the column names of the input sentences\n", + "label_key = \"label\" # specify the column name of the label\n", + "\n", + "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n", + "X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]\n", + "X_test = test_dataset[custom_sent_keys]\n", + "\n", + "print(len(X_train))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 53 + }, + "id": "19m2ZpRGTjhJ", + "outputId": "65a82458-dfd0-4e90-d0ac-fdbd231822f1" + }, + "outputs": [ + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'Members of the procession walk down the street holding small horn brass instruments.'" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_dataset.iloc[0][\"sent1\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "uvNeyzFsTjhJ", + "outputId": "b423df4f-a056-4abd-cece-ac653ea639e2" + }, + "outputs": [ + { + "data": { + "text/html": [ + "== Status ==
Current time: 2022-08-20 09:44:29 (running for 00:30:25.10)
Memory usage on this node: 23.7/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/4 CPUs, 0/4 GPUs, 0.0/252.27 GiB heap, 0.0/112.11 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: a6161fe9 with val_loss=0.2717684694591622 and parameters={'learning_rate': 1.0471607729914847e-05, 'num_train_epochs': 4, 'per_device_train_batch_size': 16, 'seed': 21, 'global_max_steps': 9223372036854775807, 'learner': 'transformer', 'FLAML_sample_size': 10000}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03
Number of trials: 12/1000000 (12 TERMINATED)

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m The following columns in the test set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: sent1, ending3, video-id, startphrase, fold-ind, ending0, ending1, ending2, gold-source, sent2. If sent1, ending3, video-id, startphrase, fold-ind, ending0, ending1, ending2, gold-source, sent2 are not expected by `BertForMultipleChoice.forward`, you can safely ignore this message.\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m ***** Running Prediction *****\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m Num examples = 20006\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m Batch size = 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m {'eval_loss': 0.6692697405815125, 'eval_automl_metric': 0.25667299810056987, 'eval_runtime': 158.3379, 'eval_samples_per_second': 126.35, 'eval_steps_per_second': 126.35, 'epoch': 0.93}\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m {'eval_loss': 0.8068006634712219, 'eval_automl_metric': 0.301259622113366, 'eval_runtime': 166.133, 'eval_samples_per_second': 120.422, 'eval_steps_per_second': 120.422, 'epoch': 1.02}\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m {'train_runtime': 444.8991, 'train_samples_per_second': 89.908, 'train_steps_per_second': 11.239, 'train_loss': 0.9226323432562794, 'epoch': 0.93}\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m {'train_runtime': 462.8118, 'train_samples_per_second': 108.035, 'train_steps_per_second': 6.752, 'train_loss': 1.0871613750307578, 'epoch': 1.02}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m The following columns in the test set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: ending2, ending1, sent1, gold-source, ending0, fold-ind, video-id, ending3, sent2, startphrase. If ending2, ending1, sent1, gold-source, ending0, fold-ind, video-id, ending3, sent2, startphrase are not expected by `BertForMultipleChoice.forward`, you can safely ignore this message.\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m ***** Running Prediction *****\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m Num examples = 20006\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m Batch size = 1\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m The following columns in the test set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: ending1, ending0, ending3, gold-source, video-id, fold-ind, startphrase, ending2, sent1, sent2. If ending1, ending0, ending3, gold-source, video-id, fold-ind, startphrase, ending2, sent1, sent2 are not expected by `BertForMultipleChoice.forward`, you can safely ignore this message.\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m ***** Running Prediction *****\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m Num examples = 20006\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m Batch size = 1\n", + "\u001b[2m\u001b[36m(train pid=9672)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_259ba87c_9_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_09-35-49/checkpoint-630/added_tokens.json. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=9672)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_259ba87c_9_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_09-35-49/checkpoint-630/vocab.txt\n", + "\u001b[2m\u001b[36m(train pid=9672)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_259ba87c_9_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_09-35-49/checkpoint-630/tokenizer.json\n", + "\u001b[2m\u001b[36m(train pid=9672)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=9672)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_259ba87c_9_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_09-35-49/checkpoint-630/special_tokens_map.json\n", + "\u001b[2m\u001b[36m(train pid=9672)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_259ba87c_9_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_09-35-49/checkpoint-630/tokenizer_config.json\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_31f5f7b3_10_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-11/checkpoint-612/added_tokens.json. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_31f5f7b3_10_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-11/checkpoint-612/vocab.txt\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_31f5f7b3_10_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-11/checkpoint-612/tokenizer.json\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_31f5f7b3_10_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-11/checkpoint-612/special_tokens_map.json\n", + "\u001b[2m\u001b[36m(train pid=9840)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_31f5f7b3_10_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-11/checkpoint-612/tokenizer_config.json\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_46c248b3_12_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-46/checkpoint-1167/added_tokens.json. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_46c248b3_12_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-46/checkpoint-1167/vocab.txt\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_46c248b3_12_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-46/checkpoint-1167/tokenizer.json\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_46c248b3_12_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-46/checkpoint-1167/special_tokens_map.json\n", + "\u001b[2m\u001b[36m(train pid=10183)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_46c248b3_12_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-46/checkpoint-1167/tokenizer_config.json\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_3f1c7a06_11_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-32/checkpoint-635/added_tokens.json. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_3f1c7a06_11_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-32/checkpoint-635/vocab.txt\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_3f1c7a06_11_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-32/checkpoint-635/tokenizer.json\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_3f1c7a06_11_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-32/checkpoint-635/special_tokens_map.json\n", + "\u001b[2m\u001b[36m(train pid=10015)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-14-03/train_3f1c7a06_11_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train__2022-08-20_09-36-32/checkpoint-635/tokenizer_config.json\n", + "2022-08-20 09:47:40,258\tINFO tune.py:747 -- Total run time: 2017.12 seconds (1805.20 seconds for the tuning loop).\n", + "[flaml.automl: 08-20 09:47:52] {3322} INFO - selected model: None\n", + "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'loss': 1.0426, 'learning_rate': 1.0186867471868435e-05, 'epoch': 0.11}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[flaml.automl: 08-20 09:51:13] {3465} INFO - retrain transformer for 200.9s\n", + "[flaml.automl: 08-20 09:51:13] {3472} INFO - retrained model: None\n", + "[flaml.automl: 08-20 09:51:13] {2749} INFO - fit succeeded\n", + "[flaml.automl: 08-20 09:51:13] {2750} INFO - Time taken to find the best model: 1323.6405737400055\n", + "[flaml.automl: 08-20 09:51:13] {2761} WARNING - Time taken to find the best model is 74% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'train_runtime': 128.202, 'train_samples_per_second': 2294.692, 'train_steps_per_second': 143.43, 'train_loss': 1.0108715354543423, 'epoch': 0.14}\n" + ] + } + ], + "source": [ + "''' import AutoML class from flaml package '''\n", + "from flaml import AutoML\n", + "automl = AutoML()\n", + "\n", + "import ray\n", + "\n", + "if ray.is_initialized() == False:\n", + " ray.init(num_gpus=4, num_cpus=4)\n", + "\n", + "automl_settings = {\n", + " \"time_budget\": 1800, # setting the time budget\n", + " \"task\": \"multichoice-classification\", # setting the task as multiplechoice-classification\n", + " \"fit_kwargs_by_estimator\": { # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base\n", + " \"transformer\": {\n", + " \"output_dir\": \"data/output/\", # setting the output directory\n", + " \"model_path\": \"bert-base-uncased\", # the batch size for validation (inference)\n", + " }\n", + " },\n", + " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n", + " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n", + " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n", + " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n", + " \"n_concurrent_trials\": 4\n", + "}\n", + "\n", + "'''The main flaml automl API'''\n", + "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 350 + }, + "id": "kh7ZJsIKTjhJ", + "outputId": "36fd683c-0792-4c26-ecc3-9e981b791b39" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 5.316409886511772e-06, 'num_train_epochs': 1, 'per_device_train_batch_size': 64, 'seed': 26, 'global_max_steps': 152, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 5.316409886511772e-06, 'num_train_epochs': 1, 'per_device_train_batch_size': 64, 'seed': 26, 'global_max_steps': 152, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 322, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 322, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.5662610420278344e-06, 'num_train_epochs': 1, 'per_device_train_batch_size': 64, 'seed': 6, 'global_max_steps': 151, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 322, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 5.461587558683657e-06, 'num_train_epochs': 1, 'per_device_train_batch_size': 64, 'seed': 25, 'global_max_steps': 157, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 322, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 5.163225512301641e-06, 'num_train_epochs': 1, 'per_device_train_batch_size': 64, 'seed': 20, 'global_max_steps': 152, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 3, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 322, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.0471607729914847e-05, 'num_train_epochs': 4, 'per_device_train_batch_size': 16, 'seed': 21, 'global_max_steps': 629, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0471607729914847e-05, 'num_train_epochs': 4, 'per_device_train_batch_size': 16, 'seed': 21, 'global_max_steps': 629, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.54963197430746e-06, 'num_train_epochs': 2, 'per_device_train_batch_size': 64, 'seed': 19, 'global_max_steps': 154, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0471607729914847e-05, 'num_train_epochs': 4, 'per_device_train_batch_size': 16, 'seed': 21, 'global_max_steps': 629, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 7.798079645313044e-06, 'num_train_epochs': 2, 'per_device_train_batch_size': 64, 'seed': 15, 'global_max_steps': 155, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0471607729914847e-05, 'num_train_epochs': 4, 'per_device_train_batch_size': 16, 'seed': 21, 'global_max_steps': 629, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "8\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlcElEQVR4nO3de5xdVX338c/XIYHBGiaBwSbDJfEhBEFsAiNeqBdQSPRRMiJi6EXACqUWbeXVSFIqUngo2LTl0aepPGARUW6SQogaHGlBtFwzmEguGggBIZMo4RJAGMiFX//Y64Sdk5nJ3smcmTMz3/frdV6z99prr/M7k8vvrLX2XlsRgZmZWVFvGOgAzMxscHHiMDOzUpw4zMysFCcOMzMrxYnDzMxKceIwM7NSnDjM+pCk90paOdBxmNWSE4cNGZIel/ShgYwhIn4WEZNq1b6kqZJ+KulFSesl3SXphFq9n1l3nDjMSpDUMIDvfRJwE3ANsB/wZuB84GM70ZYk+d+/7RT/xbEhT9IbJM2S9KikZyR9T9KY3PGbJP1G0vPp2/xhuWNXS/qGpIWSXgKOST2bv5H0UDrnRkl7pPofkLQmd36PddPxL0laJ2mtpM9KCkkHdfMZBPwLcFFEfDMino+I1yLirog4I9W5QNJ3c+eMT+3tlvZ/IuliSXcDLwMzJXVUvc8XJS1I27tL+idJT0j6raTLJTXu4h+HDQFOHDYcfB5oA94PjAOeA+bmjt8GTAT2BX4OXFt1/h8BFwNvAv47lZ0MTAMmAG8HTuvl/butK2kacA7wIeAg4AO9tDEJ2B+Y10udIv4UOJPss1wOTJI0MXf8j4Dr0valwMHA5BRfC1kPx4Y5Jw4bDs4CzouINRHxKnABcFLlm3hEXBURL+aO/YGkvXLn3xoRd6dv+K+ksq9HxNqIeBb4Ptl/rj3pqe7JwLciYnlEvJzeuyd7p5/rin3kHl2d3m9zRDwP3AqcApASyCHAgtTDORP4YkQ8GxEvAv8AzNjF97chwInDhoMDgVskbZC0AfglsAV4s6QGSZemYawXgMfTOfvkzn+ymzZ/k9t+Gfi9Xt6/p7rjqtru7n0qnkk/x/ZSp4jq97iOlDjIehvzUxJrBvYEHsz93n6Uym2Yc+Kw4eBJ4MMR0ZR77RERnWT/WU4nGy7aCxifzlHu/FotIb2ObJK7Yv9e6q4k+xyf6KXOS2T/2Vf8fjd1qj/L7UCzpMlkCaQyTPU00AUclvud7RURvSVIGyacOGyoGSFpj9xrN7Kx/IslHQggqVnS9FT/TcCrZN/o9yQbjukv3wNOl/RWSXsCX+6pYmTPPzgH+LKk0yWNSpP+fyjpilRtCfA+SQekobbZOwogIjaRXak1BxhDlkiIiNeAK4HLJO0LIKlF0tSd/bA2dDhx2FCzkOybcuV1AfA1YAHwY0kvAvcB70z1rwF+DXQCK9KxfhERtwFfB+4EVuXe+9Ue6s8DPgV8BlgL/Bb4P2TzFETE7cCNwEPAg8APCoZyHVmP66aI2JwrP7cSVxrG+0+ySXob5uQHOZnVB0lvBZYBu1f9B25WV9zjMBtAkj6e7pcYDXwV+L6ThtU7Jw6zgfXnwFPAo2RXev3FwIZjtmMeqjIzs1Lc4zAzs1J2G+gA+sM+++wT48ePH+gwzMwGlQcffPDpiNjups9hkTjGjx9PR0fHjiuamdlWkn7dXbmHqszMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMyslGFxVZWZ2WA0f3Enc9pXsnZDF+OaGpk5dRJtU1oGOiwnDjOzejR/cSezb15K16YtAHRu6GL2zUsBBjx5OHGYmdWhOe0rtyaNiq5NW/jSvIe4/oEnCrVx6LhRfOVjh/V5bJ7jMDOrQ2s3dHVbvnHLa/0cyfZq2uOQNI3sIToNwDcj4tKq45cBx6TdPYF9I6IpPcbyG8AoshVDL46IG9M5VwPvB55P550WEUtq+TnMzPrbuKZGOrtJHi1Njdz45+8egIheV7Meh6QGYC7wYeBQ4BRJh+brRMQXI2JyREwG/h9wczr0MvDpiDgMmAb8X0lNuVNnVs5z0jCzoWjm1Ek0jmjYpqxxRAMzpw78QxhrOVR1FLAqIlZHxEbgBmB6L/VPAa4HiIiHI+KRtL2W7HkF2y20ZWY2VLVNaeGSEw9nZEP233RLUyOXnHj4gE+MQ20TRwvwZG5/TSrbjqQDgQnAHd0cOwoYSfagm4qLJT0k6TJJu/fQ5pmSOiR1rF+/fmc/g5nZgGmb0sKUA5p454Qx3D3r2LpIGlA/k+MzgHkRsc0lBJLGAt8BTo+IyozQbOAQ4B3AGODc7hqMiCsiojUiWpub3VkxM+srtUwcncD+uf39Ull3ZpCGqSokjQJ+CJwXEfdVyiNiXWReBb5FNiRmZmb9pJaJYxEwUdIESSPJksOC6kqSDgFGA/fmykYCtwDXRMS8qvpj008BbcCyWn0AMzPbXs0ux42IzZLOBtrJLse9KiKWS7oQ6IiIShKZAdwQ2z78/GTgfcDekk5LZZXLbq+V1AwIWAKcVavPYGZm26vpfRwRsRBYWFV2ftX+Bd2c913guz20eWwfhmhmZiV5yZGS6nXRMTOz/uLEUUI9LzpmZtZfnDhK6ItFx8zMylix7gUOHTtqoMPYRr3cxzEo1POiY2Y2NB06dhTTJ9fXiIZ7HCXU86JjZmb9xT2OEup50TEzs/7iHkcJlQnwL817iI1bXqPFV1WZ2TDkxFFS25SWrRPhHp4ys+HIQ1VmZlaKE4eZmZXixGFmZqU4cZiZWSlOHGZmVooTh5mZleLEYWZmpThxmJlZKU4cZmZWihOHmZmV4sRhZmalOHGYmVkpNU0ckqZJWilplaRZ3Ry/TNKS9HpY0obcsVMlPZJep+bKj5S0NLX5dUmq5WcwM7Nt1Wx1XEkNwFzgOGANsEjSgohYUakTEV/M1f88MCVtjwG+ArQCATyYzn0O+AZwBnA/sBCYBtxWq89hZmbbqmWP4yhgVUSsjoiNwA3A9F7qnwJcn7anArdHxLMpWdwOTJM0FhgVEfdFRADXAG01+wRmZradWiaOFuDJ3P6aVLYdSQcCE4A7dnBuS9ou0uaZkjokdaxfv36nPoCZmW2vXibHZwDzImJLXzUYEVdERGtEtDY3N/dVs2Zmw14tE0cnsH9uf79U1p0ZvD5M1du5nWm7SJtmZlYDtUwci4CJkiZIGkmWHBZUV5J0CDAauDdX3A4cL2m0pNHA8UB7RKwDXpD0rnQ11aeBW2v4GczMrErNrqqKiM2SziZLAg3AVRGxXNKFQEdEVJLIDOCGNNldOfdZSReRJR+ACyPi2bT9OeBqoJHsaipfUWVm1o9qljgAImIh2SWz+bLzq/Yv6OHcq4CruinvAN7Wd1GamVkZ9TI5bmZmg4QTh5mZleLEYWZmpThxmJlZKU4cZmZWihOHmZmV4sRhZmalOHGYmVkpNb0B0MzM+sb8xZ3MaV/J2g1djGtqZObUSbRN6XZx8Jpz4jAzq3PzF3cy++aldG3KFhDv3NDF7JuXAgxI8vBQlZlZnZvTvnJr0qjo2rSFOe0rByQeJw4zszq3dkNXqfJa22HikLR3fwRiZmbdG9fUWKq81or0OO6TdJOkj6RnYJiZWT+aOXUSjSMatilrHNHAzKmTBiSeIonjYOAK4E+BRyT9g6SDaxuWmZlVtE1p4ZITD6elqREBLU2NXHLi4fV7VVV6wNLtwO2SjgG+C3xO0i+AWRFxb68NmJnZLmub0jJgiaLaDhNHmuP4E7Iex2+Bz5M9AnYycBMwoYbxmZlZnSlyH8e9wHeAtohYkyvvkHR5bcIyM7N6VSRxTMo/DzwvIr7ax/GYmVmdKzI5/mNJTZUdSaMltdcuJDMzq2dFEkdzRGyo7ETEc8C+RRqXNE3SSkmrJM3qoc7JklZIWi7pulR2jKQludcrktrSsaslPZY7NrlILGZm1jeKDFVtkXRARDwBIOlAoNuhqzxJDcBc4DhgDbBI0oKIWJGrMxGYDRwdEc9J2hcgIu4km3xH0hhgFfDjXPMzI2JegdjNzKyPFUkc5wH/LekuQMB7gTMLnHcUsCoiVgNIugGYDqzI1TkDmJt6MUTEU920cxJwW0S8XOA9zcysxnY4VBURPwKOAG4EbgCOjIgicxwtwJO5/TWpLO9g4GBJd0u6T9K0btqZAVxfVXaxpIckXSZp9wKxmJlZHym6yOEW4CngBeBQSe/ro/ffDZgIfAA4BbiyaiJ+LHA4kE9Us4FDgHcAY4Bzu2tY0pmSOiR1rF+/vo/CNTOzIoscfhb4Kdl/3n+ffl5QoO1OYP/c/n6pLG8NsCAiNkXEY8DDZImk4mTglojYVCmIiHWReRX4FtmQ2HYi4oqIaI2I1ubm5gLhmplZEUV6HH9F9u3+1xFxDDAF2FDgvEXAREkTJI0kG3JaUFVnPllvA0n7kA1drc4dP4WqYarUCyEtuNgGLCsQi5mZ9ZEik+OvRMQrkpC0e0T8StIOl2SMiM2SzibroTQAV0XEckkXAh0RsSAdO17SCrLhsJkR8QyApPFkPZa7qpq+VlIz2UT9EuCsQp/UzMz6RJHEsSbNO8wnW+jwOeDXRRqPiIXAwqqy83PbAZyTXtXnPs72k+lExLFF3tvMzGqjyOq4H0+bF0i6E9gL+FFNozIzs7rVa+JIN/Etj4hDACKietjIzMyGmV4nxyNiC7BS0gH9FI+ZmdW5InMco4Hlkh4AXqoURsQJNYvKzMzqVpHE8eWaR2FmZoNGkclxz2uYmdlWRR4d+yKvr4Y7EhgBvBQRo2oZmJmZ1aciPY43VbbT3drTgXfVMigzM6tfRRc5BLIb9iJiPjC1NuGYmVm9KzJUdWJu9w1AK/BKzSIyM7O6VuSqqo/ltjcDj5MNV5mZ2TBUZI7j9P4IxMzMBociz+P4dtXDlUZLuqqmUZmZ2U6bv7iToy+9gwmzfsjRl97B/MXVj0LaNUWGqt4eERsqOxHxnKQpfRqFmZn1ifmLO5l981K6Nm0BoHNDF7NvXgpA25TtFhzfKUWuqnqDpNGVHUljKJZwzMysn81pX7k1aVR0bdrCnPaVffYeRRLAPwP3Srop7X8SuLjPIjAzsz6zdkNXqfKdUWRy/BpJHUDlAUonRsSKPovAzMz6zLimRjq7SRLjmhr77D2KTI6/C3gyIv41Iv6V7ImA7+yzCMzMrM/MnDqJxhEN25Q1jmhg5tQdPvG7sCJzHN8Afpfb/10qMzOzOtM2pYVLTjyclqZGBLQ0NXLJiYf32cQ4FJvjUHo2OAAR8ZokT46bmdWptiktfZooqhXpcayW9AVJI9Lrr4DVNYvIzMzqWpHEcRbwHqATWAO8EzijSOOSpklaKWmVpFk91DlZ0gpJyyVdlyvfImlJei3IlU+QdH9q80ZJI4vEYmZmfaPIVVVPATMq+5IagY8CN/V4UlavAZgLHEeWcBZJWpC/IkvSRGA2cHS6sXDfXBNdETG5m6a/ClwWETdIuhz4MzznYmbWbwotqy6pQdJHJH0HeAz4VIHTjgJWRcTqiNgI3MD2iyOeAcyNiOdga5LqLQ6RXRY8LxV9G2gr8hnMzKxv9Jo4JL1f0v8nWxH3z8h6D2+JiJMKtN0CPJnbX5PK8g4GDpZ0t6T7JE3LHdtDUkcqb0tlewMbImJzL21WYj8znd+xfv36AuGamVkRPQ5VSVoDPEE2DPQ3EfGipMci4uU+fv+JwAeA/YCfSjo8rY11YER0SnoLcIekpcDzRRuOiCuAKwBaW1tjB9XNzKyg3noc84BxZMNSH5P0Rl5/9ngRncD+uf39UlneGmBBRGyKiMeAh8kSCRHRmX6uBn4CTAGeAZpylwN316aZmdVQj4kjIv4amEC2VtUHgJVAc7oK6vcKtL0ImJiughpJNsG+oKrO/NQ2kvYhG7panZZu3z1XfjSwIt1PcidQGSo7Fbi1QCxmZtZHep3jSM8YvzMiziRLIqeQTXA/vqOG0zzE2UA78EvgexGxXNKFkk5I1dqBZyStIEsIMyPiGeCtQIekX6TyS3NXY50LnCNpFdmcx7+X+sRmZrZLCt8BHhGbgB8AP0iX5BY5ZyGwsKrs/Nx2AOekV77OPcDhPbS5muyKLTMzGwCFLsetFhF9tz6vmZkNKjuVOMzMbPhy4jAzs1J2OMch6WBgJnBgvn5EHNvjSWZmNmQVmRy/CbgcuBLYsoO6Zma2A/MXdzKnfSVrN3QxrqmRmVMn1XQZ9L5WJHFsjggvImhm1gfmL+5k9s1L6dqUfQ/v3NDF7JuXAgya5FFkjuP7kj4naaykMZVXzSMzMxuC5rSv3Jo0Kro2bWFO+8oBiqi8Ij2OU9PPmbmyAN7S9+GYmQ1tazd0fzdDT+X1qMjzOCb0RyBmZsPBuKZGOrtJEuOaCt1XXRd2OFSVHhf7BUnz0utsSSP6Izgzs6Fm5tRJNI5o2KascUQDM6dOGqCIyisyVPUNYATwb2n/T1PZZ2sVlJnZUFWZAB/qV1W9IyL+ILd/R1p80MzMdkLblJZBlSiqFbmqaouk/1XZSQ9W8v0cZmbDVJEex0zgTkmrAZHdQX56TaMyM7O6VeSqqv+SNBGozNysjIhXaxuWmZnVq96eOX5sRNwh6cSqQwdJIiJurnFsZmZWh3rrcbwfuAP4WDfHAnDiMDMbhnpMHBHxlbR5YUQ8lj8myTcFmpkNU0Umx/8DOKKqbB5wZN+HMzwM9pUxzWx4622O4xDgMGCvqnmOUcAetQ5sqBoKK2Oa2fDW230ck4CPAk1k8xyV1xHAGUUalzRN0kpJqyTN6qHOyZJWSFou6bpUNlnSvansIUmfytW/WtJjkpak1+QisdSLobAyppkNb73NcdwK3Crp3RFxb9mGJTUAc4HjgDXAIkkLImJFrs5EYDZwdEQ8J2nfdOhl4NMR8YikccCDktojYkM6PjMi5pWNqR4MhZUxzWx4KzLHsVjSX5INW20dooqIz+zgvKOAVRGxGkDSDcB0YEWuzhnA3Ih4LrX5VPr5cO591kp6CmgGNhSIt64NhZUxzWx4K7LkyHeA3wemAncB+wEvFjivBXgyt78mleUdDBws6W5J90maVt2IpKOAkcCjueKL0xDWZZJ27+7NJZ0pqUNSx/r16wuE2z+GwsqYZja8FUkcB0XEl4GXIuLbwP8G3tlH778bMBH4AHAKcKWkpspBSWPJEtfpEfFaKp4NHAK8AxgDnNtdwxFxRUS0RkRrc3NzH4W769qmtHDJiYfT0tSIgJamRi458XBPjJvZoFFkqGpT+rlB0tuA3wD79lK/ohPYP7e/XyrLWwPcHxGbgMckPUyWSBZJGgX8EDgvIu6rnBAR69Lmq5K+BfxNgVjqymBfGdPMhrciPY4rJI0GvgwsIJuj+McC5y0CJkqaIGkkMCOdnzefrLeBpH3Ihq5Wp/q3ANdUT4KnXgiSBLQBywrEYmZmfaTIIoffTJt3UeI54xGxWdLZQDvQAFwVEcslXQh0RMSCdOx4SSvIlmqfGRHPSPoT4H3A3pJOS02eFhFLgGslNZOt1LsEOKtoTGX4Jj0zs+71dgPgOb2dGBH/sqPGI2IhsLCq7PzcdgDnpFe+zneB7/bQ5rE7et9d5Zv0zMx61luP403p5ySyiejKMNPHgAdqGdRA6+kmvS/Ne4jrH3iCFete4NCxowYoOjOzgdXbDYB/DyDpp8AREfFi2r+AbNJ6yOrpZryNW7ILuw4dO4rpk93zMLPhqchVVW8GNub2N6ayIaunm/Ramhq58c/fPQARmZnVjyJXVV0DPCDpgtTbuB+4upZBDTTfpGdm1rMiV1VdLOk24L2p6PSIWFzbsAZWZQLcV1WZmW2vt6uqRkXEC5LGAI+nV+XYmIh4tvbhDRzfpGdm1r3eehzXkS2r/iDZo2IrlPYL39NhZmZDR29XVX00/fRjYs3MbKvehqqqHxe7jYj4ed+HY2Zm9a63oap/7uVYADW/g9vMzOpPb0NVx/RnIGZmNjgUuQGQtJz6oWz7BMBrahWUmZnVrx0mDklfIVv6/FCyBQs/DPw32Y2BZmY2zBS5c/wk4IPAbyLidOAPgL1qGpWZmdWtIomjKz22dXN6Kt9TbPtkPzMzG0aKzHF0pOeAX0l2M+DvgHtrGZSZmdWv3u7jmAtcFxGfS0WXS/oRMCoiHuqX6MzMrO701uN4GPin9Izv7wHXD/XFDc3MbMd6nOOIiK9FxLuB9wPPAFdJ+pWkr0g6uN8iNDOzurLDyfGI+HVEfDUipgCnAG3AL2sdmJmZ1acdJg5Ju0n6mKRrgduAlcCJRRqXNE3SSkmrJM3qoc7JklZIWi7pulz5qZIeSa9Tc+VHSlqa2vy6JBWJxczM+kZvk+PHkfUwPgI8ANwAnBkRLxVpWFIDMBc4DlgDLJK0ICJW5OpMBGYDR0fEc5L2TeVjgK8ArWTrYj2Yzn0O+AZwBtmTCBcC08gSmpmZ9YPeehyzgXuAt0bECRFxXdGkkRwFrIqI1RGxkSzxTK+qcwYwNyUEIuKpVD4VuD0ink3HbgempYn6URFxX0QE2d3rbSViMjOzXdTbIoe7uvptC/Bkbn8N8M6qOgcDSLobaAAuiIgf9XBuS3qt6aZ8O5LOBM4EOOCAA3b6Q5iZ2baK3DleS7sBE8nWwjoFuDLdbLjLIuKKiGiNiNbm5ua+aNLMzKht4uhk26VJ9ktleWuABRGxKSIeI7t3ZGIv53am7d7aNDOzGqpl4lgETJQ0QdJIYAawoKrOfLLeBpL2IRu6Wg20A8dLGi1pNHA80B4R64AXJL0rXU31aeDWGn4GMzOrUuh5HDsjIjZLOpssCTQAV0XEckkXAh0RsYDXE8QKYAswMyKeAZB0EVnyAbgwIp5N258DrgYaya6m8hVVZmb9SNnFSUNba2trdHR0DHQYZmaDiqQHI6K1unygJ8fNzGyQceIwM7NSnDjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMysFCcOMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrBQnDjMzK6WmiUPSNEkrJa2SNKub46dJWi9pSXp9NpUfkytbIukVSW3p2NWSHssdm1zLz2BmZtvarVYNS2oA5gLHAWuARZIWRMSKqqo3RsTZ+YKIuBOYnNoZA6wCfpyrMjMi5tUqdjMz61ktexxHAasiYnVEbARuAKbvRDsnAbdFxMt9Gp2Zme2UWiaOFuDJ3P6aVFbtE5IekjRP0v7dHJ8BXF9VdnE65zJJu/dRvGZmVsBAT45/HxgfEW8Hbge+nT8oaSxwONCeK54NHAK8AxgDnNtdw5LOlNQhqWP9+vW1iN3MbFiqZeLoBPI9iP1S2VYR8UxEvJp2vwkcWdXGycAtEbEpd866yLwKfItsSGw7EXFFRLRGRGtzc/MufhQzM6uoZeJYBEyUNEHSSLIhpwX5CqlHUXEC8MuqNk6hapiqco4kAW3Asr4N28zMelOzq6oiYrOks8mGmRqAqyJiuaQLgY6IWAB8QdIJwGbgWeC0yvmSxpP1WO6qavpaSc2AgCXAWbX6DGZmtj1FxEDHUHOtra3R0dEx0GGYmQ0qkh6MiNbq8oGeHDczs0HGicPMzEpx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMysFCcOMzMrxYnDzMxKqdlaVcPB/MWdzGlfydoNXYxramTm1Em0TenukSNmZkOHE8dOmr+4k9k3L6Vr0xYAOjd0MfvmpQBOHmY2pHmoaifNaV+5NWlUdG3awpz2lQMUkZlZ/3Di2ElrN3SVKjczGyqcOHbSuKbGUuVmZkOFE8dOmjl1Eo0jGrYpaxzRwMypkwYoIjOz/uHJ8Z1UmQD3VVVmNtw4ceyCtiktThRmNux4qMrMzEpx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSlFEDHQMNSdpPfDrgY6jgH2Apwc6iAIcZ99ynH1vsMRa73EeGBHN1YXDInEMFpI6IqJ1oOPYEcfZtxxn3xsssQ6WOKt5qMrMzEpx4jAzs1KcOOrLFQMdQEGOs285zr43WGIdLHFuw3McZmZWinscZmZWihOHmZmV4sTRTyRNkrQk93pB0l9LGiPpdkmPpJ+jU31J+rqkVZIeknREP8b6RUnLJS2TdL2kPSRNkHR/iudGSSNT3d3T/qp0fHw/xvlXKcblkv46ldXF71PSVZKekrQsV1Y6NkmnpvqPSDq1n+L8ZPqdviaptar+7BTnSklTc+XTUtkqSbP6Kc45kn6Vfme3SGqq0zgvSjEukfRjSeNS+YD9ue+yiPCrn19AA/Ab4EDgH4FZqXwW8NW0/RHgNkDAu4D7+ym2FuAxoDHtfw84Lf2ckcouB/4ibX8OuDxtzwBu7Kc43wYsA/YkezzAfwIH1cvvE3gfcASwLFdWKjZgDLA6/Rydtkf3Q5xvBSYBPwFac+WHAr8AdgcmAI+mv8sNafstwMhU59B+iPN4YLe0/dXc77Pe4hyV2/5C7t/LgP257+rLPY6B8UHg0Yj4NTAd+HYq/zbQlranA9dE5j6gSdLYfopvN6BR0m5k/zGvA44F5vUQZyX+ecAHJakfYnwr2T+0lyNiM3AXcCJ18vuMiJ8Cz1YVl41tKnB7RDwbEc8BtwPTah1nRPwyIlZ2U306cENEvBoRjwGrgKPSa1VErI6IjcANqW6t4/xx+rMHuA/Yr07jfCG3+0agckXSgP257yonjoExA7g+bb85Ital7d8Ab07bLcCTuXPWpLKaiohO4J+AJ8gSxvPAg8CG3D/SfCxb40zHnwf2rnWcZL2N90raW9KeZN/e9qfOfp9VysZWDzHn1XOcnyH79k4v8QxYnJIulvQk8MfA+fUaZ1FOHP0szQ2cANxUfSyyfuqAXh+dxt2nk3Xxx5F9Q6qrbzuQfSsmG574MfAjYAmwparOgP8+e1LPsQ02ks4DNgPXDnQsPYmI8yJif7IYzx7oeHaVE0f/+zDw84j4bdr/bWXIJP18KpV3kn2DrtgvldXah4DHImJ9RGwCbgaOJutGVx41nI9la5zp+F7AM/0QJxHx7xFxZES8D3gOeJj6+33mlY2tHmLOq7s4JZ0GfBT445SM6SWeevh9Xgt8Im3Xc5y9cuLof6fw+jAVwAKgctXEqcCtufJPpysv3gU8nxvmqKUngHdJ2jPNVXwQWAHcCZzUQ5yV+E8C7sj9A64pSfumnweQzW9cR/39PvPKxtYOHC9pdOoJHp/KBsoCYIayK+kmABOBB4BFwERlV96NJBuKXVDrYCRNA74EnBARL9dxnBNzu9OBX+XiHAx/7tsb6Nn54fQiG/Z5BtgrV7Y38F/AI2RXBo1J5QLmkl0FspTc1S39EOffk/3lXgZ8h+zqlLeQ/eNbRTbMtnuqu0faX5WOv6Uf4/wZWVL7BfDBevp9kn05WAdsIhuj/rOdiY1s7H5Vep3eT3F+PG2/CvwWaM/VPy/FuRL4cK78I2Q9vkeB8/opzlVkcwFL0uvyOo3zP9K/pYeA7wMtA/3nvqsvLzliZmaleKjKzMxKceIwM7NSnDjMzKwUJw4zMyvFicPMzEpx4rBBT9JlSqvjpv12Sd/M7f+zpHN6Of9qSSel7Z9UrwibykdIujStVvpzSfdK+nA69rikfXYi7q3v28PxuWlF1RWSuvT6ysonSVqYXw22r0gaK+kHvRwfKemnuZtBbRhy4rCh4G7gPQCS3gDsAxyWO/4e4J5dfI+LgLHA2yLiCLIFCt+0i232KiL+MiImk9178GhETE6veRHxkYjYUIO3PQe4speYNpLdi/KpGry3DRJOHDYU3AO8O20fRnaz1YvpztvdyVbS/bmk8yUtUvYMjyuKruKbFlE8A/h8RLwKEBG/jYjvdVP3nNT+sqpe0KfTMxd+Iek73Zx3UeqBNBSM6XFJ+0gar+yZFFdLeljStZI+JOnu1Ds6KtV/o7JnRTwgabGknlaF/QTZ2l9IOizVX5Jir9wBPZ9ssT4bptzdtEEvItZK2pyWHnkPcC/ZaqLvJlutd2lEbJT0rxFxIUD6z/ujZHfy7shBwBOx7fLY25F0JHA68E6yu4Lvl3QXsBH4O+A9EfG0pDFV580h672cHjt3R+5BwCfJ7jZeBPwR8Idki2n+LVnv6Dyy5WA+k4a4HpD0nxHxUi6OCcBzleQInAV8LSKuTUt0VJLaMuAdOxGnDRHucdhQcQ9Z0qgkjntz+3enOscoe0rhUrLnixzWXUO74A+BWyLipYj4HdkCke9N73VTRDwNEBH55zV8mWwJmrN2MmlAtijl0oh4DVgO/FdqaykwPtU5HpglaQnZA5r2AA6oamcssD63fy/wt5LOBQ6MiK4U/xZgo6SaDtVZ/XLisKGiMs9xONk34vvIehzvAe6RtAfwb8BJEXE42Tj+HgXbXgUcIGlUn0ed9RCOrO6FlPRqbvu13P5rvD6qIOATuXmSAyJbmj6vi9zvJCKuI+u1dAELJR2bq7s78MouxGyDmBOHDRX3kA09PRsRW9K3+iay5HEPr/+H+LSk3+P1lX53KLKVV/8d+Jpef9Z6s6RPVlX9GdCmbGXhN5ItFvgz4A7gk5L2Tufmk8SPgEuBH9b4G3w78PnKvI6kKd3UeZjXeyhIeguwOiK+TraS79tT+d7A05Etu2/DkBOHDRVLya6muq+q7PmIeDpdgXQlWW+kneybfhl/RzaMs0LSMuAHwDZzHhHxc+BqslWC7we+GRGLI2I5cDFwl6RfAP9Sdd5NKbYFkhpLxlXURcAI4CFJy9P+NtJ8x6OSDkpFJwPL0vDW24BrUvkxwA9rFKcNAl4d18y2kvRx4MiI+Lte6twMzIqIh/svMqsnvqrKzLaKiFsqQ2rdSUN18500hjf3OMzMrBTPcZiZWSlOHGZmVooTh5mZleLEYWZmpThxmJlZKf8DKMTcjGm+IEYAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from flaml.data import get_output_from_log\n", + "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", + " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n", + "for config in config_history:\n", + " print(config)\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "plt.title('Learning Curve')\n", + "plt.xlabel('Wall Clock Time (s)')\n", + "plt.ylabel('Validation Accuracy')\n", + "print(len(valid_loss_history))\n", + "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", + "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "664qCdihTjhJ" + }, + "source": [ + "### 4.2 Text Summarization Example" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kmB4kaF_TjhJ" + }, + "source": [ + "The text summarization task summarizes a long text into a short sentence. For example:\n", + "\n", + "- Document: Army explosives experts were called out to deal with a suspect package at the offices on the Newtownards Road on Friday night. Roads were sealed off and traffic diverted as a controlled explosion was carried out. The premises, used by East Belfast MP Naomi Long, have been targeted a number of times. Most recently, petrol bomb attacks were carried out on the offices on consecutive nights in April and May. The attacks began following a Belfast City Council vote in December 2012 restricting the flying of the union flag at the City Hall. Condemning the latest hoax, Alliance MLA Chris Lyttle said: \"It is a serious incident for the local area, it causes serious disruption, it puts people's lives at risk, it can prevent emergency services reaching the area. \"Ultimately we need people with information to share that with the police in order for them to do their job and bring these people to justice.\n", + "\n", + "- Summary: A suspicious package left outside an Alliance Party office in east Belfast has been declared a hoax.\n", + "\n", + "In this example, we use FLAML to perform *abstractive summarization* using the t5-small language model, i.e., the summary is generated word-by-word. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "amlQnvcxTjhK", + "outputId": "5382a8f5-8c7a-4884-a8bf-8151c7f27624" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using custom data configuration default\n", + "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n", + "Using custom data configuration default\n", + "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "204045\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using custom data configuration default\n", + "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n" + ] + } + ], + "source": [ + "from datasets import load_dataset\n", + "\n", + "train_dataset = load_dataset(\"xsum\", split=\"train\").to_pandas()\n", + "print(len(train_dataset))\n", + "valid_dataset = load_dataset(\"xsum\", split=\"validation\").to_pandas()\n", + "test_dataset = load_dataset(\"xsum\", split=\"test\").to_pandas()\n", + "\n", + "custom_sent_keys = [\"document\"] # specify the column names of the input sentences\n", + "label_key = \"summary\" # specify the column name of the label \n", + "\n", + "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n", + "X_val, y_val = valid_dataset[custom_sent_keys], valid_dataset[label_key]\n", + "X_test = test_dataset[custom_sent_keys]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 626 + }, + "id": "aYq8XAtxTjhK", + "outputId": "267fffbb-e5a5-4f45-b8d1-4f718b298e01" + }, + "outputs": [ + { + "data": { + "text/html": [ + "== Status ==
Current time: 2022-08-20 10:53:00 (running for 01:00:10.16)
Memory usage on this node: 24.9/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/4 CPUs, 0/4 GPUs, 0.0/252.27 GiB heap, 0.0/112.11 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: 888b71b8 with val_loss=0.8562685839953479 and parameters={'learning_rate': 4.747405262702932e-05, 'num_train_epochs': 0.1, 'per_device_train_batch_size': 64, 'seed': 19, 'global_max_steps': 9223372036854775807, 'learner': 'transformer', 'FLAML_sample_size': 10000}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50
Number of trials: 8/1000000 (8 TERMINATED)

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m {'eval_loss': 3.629159688949585, 'eval_automl_metric': 0.8566706912648477, 'eval_runtime': 1292.7731, 'eval_samples_per_second': 8.766, 'eval_steps_per_second': 8.766, 'epoch': 0.1}\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m {'train_runtime': 1299.5697, 'train_samples_per_second': 0.769, 'train_steps_per_second': 0.012, 'train_loss': 3.952885627746582, 'epoch': 0.1}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m {'eval_loss': 3.3510377407073975, 'eval_automl_metric': 0.8490142159489932, 'eval_runtime': 1295.8577, 'eval_samples_per_second': 8.745, 'eval_steps_per_second': 8.745, 'epoch': 0.1}\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m {'train_runtime': 1303.1308, 'train_samples_per_second': 0.767, 'train_steps_per_second': 0.025, 'train_loss': 3.790097713470459, 'epoch': 0.1}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m ***** Running Prediction *****\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m Num examples = 11332\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m Batch size = 1\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m ***** Running Prediction *****\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m Num examples = 11332\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m Batch size = 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m {'eval_loss': 3.418060064315796, 'eval_automl_metric': 0.8528926893980235, 'eval_runtime': 1291.0528, 'eval_samples_per_second': 8.777, 'eval_steps_per_second': 8.777, 'epoch': 0.1}\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m {'train_runtime': 1297.7666, 'train_samples_per_second': 0.771, 'train_steps_per_second': 0.012, 'train_loss': 3.8431835174560547, 'epoch': 0.1}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m ***** Running Prediction *****\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m Num examples = 11332\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m Batch size = 1\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m {'eval_loss': 3.036466598510742, 'eval_automl_metric': 0.8168502922908516, 'eval_runtime': 1287.5045, 'eval_samples_per_second': 8.802, 'eval_steps_per_second': 8.802, 'epoch': 1.0}\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m {'train_runtime': 1341.8008, 'train_samples_per_second': 7.453, 'train_steps_per_second': 0.117, 'train_loss': 3.455144991540605, 'epoch': 1.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m ***** Running Prediction *****\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m Num examples = 11332\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m Batch size = 1\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_bba8754c_5_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-19/checkpoint-16/spiece.model. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_bba8754c_5_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-19/checkpoint-16/added_tokens.json. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_bba8754c_5_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-19/checkpoint-16/tokenizer.json\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_bba8754c_5_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-19/checkpoint-16/special_tokens_map.json\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_bba8754c_5_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-19/checkpoint-16/tokenizer_config.json\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001b[2m\u001b[36m(train pid=12776)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_c5039194_6_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-35/checkpoint-32/spiece.model. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_c5039194_6_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-35/checkpoint-32/added_tokens.json. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_c5039194_6_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-35/checkpoint-32/tokenizer.json\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_c5039194_6_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-35/checkpoint-32/special_tokens_map.json\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_c5039194_6_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-37-35/checkpoint-32/tokenizer_config.json\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001b[2m\u001b[36m(train pid=12943)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_de0b5a76_7_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_e_2022-08-20_10-38-15/checkpoint-16/spiece.model. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_de0b5a76_7_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_e_2022-08-20_10-38-15/checkpoint-16/added_tokens.json. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_de0b5a76_7_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_e_2022-08-20_10-38-15/checkpoint-16/tokenizer.json\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_de0b5a76_7_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_e_2022-08-20_10-38-15/checkpoint-16/special_tokens_map.json\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_de0b5a76_7_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_e_2022-08-20_10-38-15/checkpoint-16/tokenizer_config.json\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001b[2m\u001b[36m(train pid=13121)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_f69bd7dc_8_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-38-56/checkpoint-157/spiece.model. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_f69bd7dc_8_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-38-56/checkpoint-157/added_tokens.json. We won't load it.\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_f69bd7dc_8_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-38-56/checkpoint-157/tokenizer.json\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m loading file None\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_f69bd7dc_8_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-38-56/checkpoint-157/special_tokens_map.json\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-08-20_09-52-50/train_f69bd7dc_8_FLAML_sample_size=10000,global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_e_2022-08-20_10-38-56/checkpoint-157/tokenizer_config.json\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001b[2m\u001b[36m(train pid=13312)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", + "2022-08-20 11:25:23,543\tINFO tune.py:747 -- Total run time: 5553.09 seconds (3602.98 seconds for the tuning loop).\n", + "[flaml.automl: 08-20 11:25:27] {3322} INFO - selected model: None\n", + "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[flaml.automl: 08-20 11:37:29] {3465} INFO - retrain transformer for 721.8s\n", + "[flaml.automl: 08-20 11:37:29] {3472} INFO - retrained model: None\n", + "[flaml.automl: 08-20 11:37:29] {2749} INFO - fit succeeded\n", + "[flaml.automl: 08-20 11:37:29] {2750} INFO - Time taken to find the best model: 2666.945666074753\n", + "[flaml.automl: 08-20 11:37:29] {2761} WARNING - Time taken to find the best model is 74% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'train_runtime': 9.5593, 'train_samples_per_second': 2134.522, 'train_steps_per_second': 33.371, 'train_loss': 3.9266421794891357, 'epoch': 0.01}\n" + ] + } + ], + "source": [ + "''' import AutoML class from flaml package '''\n", + "from flaml import AutoML\n", + "automl = AutoML()\n", + "\n", + "import ray\n", + "\n", + "if ray.is_initialized() == False:\n", + " ray.init(num_gpus=4, num_cpus=4)\n", + "\n", + "automl_settings = {\n", + " \"time_budget\": 3600, # setting the time budget\n", + " \"task\": \"summarization\", # setting the task as summarization\n", + " \"fit_kwargs_by_estimator\": { # if model_path is not set, the default model is t5-small: https://huggingface.co/t5-small\n", + " \"transformer\": {\n", + " \"output_dir\": \"data/output/\", # setting the output directory\n", + " \"model_path\": \"t5-small\",\n", + " \"pad_to_max_length\": True,\n", + " }\n", + " },\n", + " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n", + " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n", + " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n", + " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n", + " \"metric\": \"rouge1\",\n", + " \"sample\": True, # sample: False # if the time is sufficient (e.g., longer than one trial's running time), you can set \n", + " \"n_concurrent_trials\": 4, \n", + "}\n", + "\n", + "from flaml import tune\n", + "custom_hp = {\n", + " \"transformer\": {\n", + " \"num_train_epochs\": {\n", + " \"domain\": tune.choice([0.1, 1, 2, 3, 4, 5]),\n", + " \"init_value\": 0.1, \n", + " \"low_cost_init_value\": 0.1,\n", + " },\n", + " }\n", + "}\n", + "\n", + "\n", + "'''The main flaml automl API'''\n", + "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, custom_hp=custom_hp, **automl_settings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xPy67MBFTjhK", + "outputId": "fe0ca67e-b129-4889-ee03-972620bc8421", + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 4.747405262702932e-05, 'num_train_epochs': 0.1, 'per_device_train_batch_size': 64, 'seed': 19, 'global_max_steps': 16, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 4.747405262702932e-05, 'num_train_epochs': 0.1, 'per_device_train_batch_size': 64, 'seed': 19, 'global_max_steps': 16, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.5662610420278344e-06, 'num_train_epochs': 0.1, 'per_device_train_batch_size': 64, 'seed': 6, 'global_max_steps': 16, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 4.747405262702932e-05, 'num_train_epochs': 0.1, 'per_device_train_batch_size': 64, 'seed': 19, 'global_max_steps': 16, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 5.316409886511772e-06, 'num_train_epochs': 1, 'per_device_train_batch_size': 64, 'seed': 26, 'global_max_steps': 157, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 4.747405262702932e-05, 'num_train_epochs': 0.1, 'per_device_train_batch_size': 64, 'seed': 19, 'global_max_steps': 16, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.999999999999999e-06, 'num_train_epochs': 0.1, 'per_device_train_batch_size': 32, 'seed': 20, 'global_max_steps': 32, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 4.747405262702932e-05, 'num_train_epochs': 0.1, 'per_device_train_batch_size': 64, 'seed': 19, 'global_max_steps': 16, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n", + "4\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAewklEQVR4nO3de5hcVZ3u8e9LbjSXEJSWITcSJUYTL4QUOFHgiKMnQUeIEJCLR6PPiJqDz1EfMiaiDoKeQTEqDIwSGWUcRQwMIioSvIKHm+mEkNAw0RAj6Q6jAYwQaMntd/7Yq6FSqerU7vROVXe/n+epJ7XXXnvXWqnufmvtvWttRQRmZmb12q/RDTAzs/7FwWFmZrk4OMzMLBcHh5mZ5eLgMDOzXBwcZmaWi4PDrA9JOkHSmka3w6xIDg4bMCStl/TmRrYhIn4dEZOL2r+kmZLulPS0pE2S7pB0SlGvZ1aNg8MsB0lDGvjac4AbgG8BY4HDgU8Db+/FviTJv//WK/7BsQFP0n6SFkh6RNITkpZIelHZ+hsk/bekv6RP81PL1l0r6auSbpX0DHBSGtlcIGlV2uZ7kvZP9d8oqaNs+5p10/p/lPSYpI2S/kFSSDqqSh8EfAm4JCKuiYi/RMTOiLgjIt6f6lwk6dtl20xI+xualn8l6XOS7gKeBeZLaqt4nY9KuiU9HyHpi5IelfRHSV+T1LKXb4cNAA4OGww+DMwG/gcwGvgzcFXZ+p8Ak4CXACuA71Rsfw7wOeBg4P+lsjOBWcBE4DXA3B5ev2pdSbOAjwFvBo4C3tjDPiYD44Abe6hTj/8FnEfWl68BkyVNKlt/DnBden4p8HLg6NS+MWQjHBvkHBw2GHwQuDAiOiLiOeAiYE73J/GI+EZEPF227rWSDinb/gcRcVf6hP/XVHZFRGyMiCeBH5L9ca2lVt0zgW9GRHtEPJteu5YXp38fq6/LNV2bXm97RPwF+AFwNkAKkFcAt6QRznnARyPiyYh4Gvi/wFl7+fo2ADg4bDA4Evi+pM2SNgMPAzuAwyUNkXRpOoz1FLA+bXNY2fYbquzzv8uePwsc1MPr16o7umLf1V6n2xPp3yN6qFOPyte4jhQcZKONm1OItQIHAMvL/t9uS+U2yDk4bDDYAJwcEaPKHvtHRCfZH8tTyQ4XHQJMSNuobPuippB+jOwkd7dxPdRdQ9aP03uo8wzZH/tuf1OlTmVffgq0SjqaLEC6D1M9DnQBU8v+zw6JiJ4C0gYJB4cNNMMk7V/2GEp2LP9zko4EkNQq6dRU/2DgObJP9AeQHY7ZV5YA75X0SkkHAJ+qVTGy+x98DPiUpPdKGplO+h8vaXGqthI4UdL4dKht4Z4aEBHbyK7Uugx4EVmQEBE7ga8DX5b0EgBJYyTN7G1nbeBwcNhAcyvZJ+Xux0XA5cAtwO2SngbuBV6X6n8L+APQCTyU1u0TEfET4Argl8Dastd+rkb9G4F3Au8DNgJ/BD5Ldp6CiPgp8D1gFbAc+FGdTbmObMR1Q0RsLyv/eHe70mG8n5GdpLdBTr6Rk1lzkPRK4EFgRMUfcLOm4hGHWQNJekf6vsShwOeBHzo0rNkVGhySZklaI2mtpAVV1p8oaYWk7elbsZXrR0rqkHRllXW3SHqwqLab7SMfAP4EPEJ2pdeHGtscsz0bWtSO09QMVwFvATqAZZJuiYiHyqo9SvZlqAtq7OYS4M4q+z4N2NKnDTZrgIiY1eg2mOVV5IjjOGBtRKyLiK3A9WSXPT4vItZHxCpgZ+XGkqaTzcVze0X5QWRXl3y2qIabmVlthY04yKYnKP+yUQcvXMnSozT52iLgXWRXe5S7JK17tt6GHHbYYTFhwoR6q5uZGbB8+fLHI2K3L30WGRx7Yx5wa0R0ZDMfZNKXlF4WER+VNKGnHUg6j2zKBMaPH09bW1tP1c3MrIKkP1QrLzI4Otn1m7BjU1k9ZgAnSJpHNj3DcElbyK63L0laT9b2l0j6VUS8sXIHEbEYWAxQKpV8zbGZWR8pMjiWAZMkTSQLjLPIpnfYo4g4t/u5pLlAKSK6r8r6aiqfAPyoWmiYmVlxCjs5nq5FPx9YSjap3JKIaJd0sdIdyyQdm+5dcAZwtaT2otpjZmZ9Y1B8c7xUKoXPcZiZ5SNpeUSUKsv9zXEzM8ulWa+qarib7+/ksqVr2Li5i9GjWpg/czKzp41pdLPMzBrOwVHFzfd3svCm1XRt2wFA5+YuFt60GsDhYf2GP/xYUXyOo4o3XPoLOjd37VY+fMh+TBs/qg9bZlaMx7c8x+8ff4adZb/e+wkmHnYghx00onENs31qyuiR/NPbp/Z6e5/jyGFjldAA2Lpjt5lRzJrShie7dgkNgJ2RlZvtLR+qqmL0qJaqI44xo1r43gdmNKBFZvlMXPDjquXbduz0z7DtNY84qpg/czItw4bsUtYybAjzZ/rmZ9Y/jB7VkqvcLA8HRxWzp43hn097NWNGtSCykcY/n/Zqn1i0fsMffqxIPlRVw+xpYxwU1m91/+z6qiorgoPDbIDyhx8rig9VmZlZLg4OMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLoUGh6RZktZIWitpQZX1J0paIWm7pDlV1o+U1CHpyrKy2yQ9IKld0tckDanczszMilNYcKQ/6FcBJwNTgLMlTamo9igwF7iuxm4uAe6sKDszIl4LvApoBc7oqzabmdmeFTniOA5YGxHrImIrcD1wanmFiFgfEauAnZUbS5oOHA7cXrHNU+npUGA4EAW03czMaigyOMYAG8qWO1LZHknaD1gEXFBj/VLgT8DTwI016pwnqU1S26ZNm/K028zMetCsJ8fnAbdGREe1lRExEzgCGAG8qUadxRFRiohSa2trcS01Mxtkhha4705gXNny2FRWjxnACZLmAQcBwyVtiYjnT7BHxF8l/YDs8NdP+6jNZma2B0UGxzJgkqSJZIFxFnBOPRtGxLndzyXNBUoRsUDSQcDBEfGYpKHA24Bf93nLzcyspsIOVUXEduB8YCnwMLAkItolXSzpFABJx0rqILsy6mpJ7XvY7YHALZJWASvJznN8rag+mJnZ7hQx8C9KKpVK0dbW1uhmmJn1K5KWR0SpsrxZT46bmVmTcnCYmVkuDg4zM8vFwWFmZrk4OMzMLBcHh5mZ5eLgMDOzXBwcZmaWi4PDzMxycXCYmVkuDg4zM8vFwWFmZrk4OMzMLBcHh5mZ5eLgMDOzXBwcZmaWi4PDzMxycXCYmVkuDg4zM8vFwWFmZrk4OMzMLBcHh5mZ5eLgMDOzXBwcZmaWi4PDzMxycXCYmVkuDg4zM8ul0OCQNEvSGklrJS2osv5ESSskbZc0p8r6kZI6JF2Zlg+Q9GNJ/yWpXdKlRbbfzMx2V1hwSBoCXAWcDEwBzpY0paLao8Bc4Loau7kEuLOi7IsR8QpgGvAGSSf3WaPNzGyPihxxHAesjYh1EbEVuB44tbxCRKyPiFXAzsqNJU0HDgduL6v/bET8Mj3fCqwAxhbXBTMzq1RkcIwBNpQtd6SyPZK0H7AIuKCHOqOAtwM/r7H+PEltkto2bdpUb5vNzGwPmvXk+Dzg1ojoqLZS0lDgu8AVEbGuWp2IWBwRpYgotba2FthUM7PBZWiB++4ExpUtj01l9ZgBnCBpHnAQMFzSlojoPsG+GPhdRHylrxprZmb1KTI4lgGTJE0kC4yzgHPq2TAizu1+LmkuUOoODUmfBQ4B/qGvG2xmZntW2KGqiNgOnA8sBR4GlkREu6SLJZ0CIOlYSR3AGcDVktp72qekscCFZFdprZC0UpIDxMxsH1JENLoNhSuVStHW1tboZpiZ9SuSlkdEqbK8WU+Om5lZk3JwmJlZLg4OMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5ODjMzCyXXgWHpJ/0dUPMzKx/qHkjJ0nH1FoFHF1Ia8zMrOn1dAfAZcAdZEFRaVQhrTEzs6bXU3A8DHwgIn5XuULShuKaZGZmzayncxwX9bD+w33fFDMz6w9qjjgi4sYe1t1cSGvMzKzp+XJcMzPLxcFhZma5ODjMzCyXPQaHpAMkfUrS19PyJEl/X3zTzMysGdUz4vgm8BwwIy13Ap8trEVmZtbU6gmOl0XEF4BtABHxLNW/FGhmZoNAPcGxVVILEACSXkY2AjEzs0Gop2+Od/sn4DZgnKTvAG8A5hbZKDMza157HHFExE+B08jC4rtAKSJ+Vc/OJc2StEbSWkkLqqw/UdIKSdslzamyfqSkDklXlpV9TtIGSVvqaYOZmfWteq6qOgY4EngM2AiMl/QyST2OViQNAa4CTgamAGdLmlJR7VGyQLquxm4uAe6sKPshcNye2m1mZsWo51DVvwLHAKvIToq/CmgHDpH0oYi4vcZ2xwFrI2IdgKTrgVOBh7orRMT6tG5n5caSpgOHkx0mK5Vtc29aX0fTzcysr9VzcnwjMC0iShExHZgGrAPeAnyhh+3GAOWz6Haksj2StB+wCLignvo19nGepDZJbZs2bertbszMrEI9wfHyiGjvXoiIh4BXdI8kCjIPuDUiOnq7g4hYnMKu1Nra2odNMzMb3Oo5VNUu6avA9Wn5ncBDkkaQvttRQycwrmx5bCqrxwzgBEnzgIOA4ZK2RMRuJ9jNzGzfqic45pKNAD6Slu8iO4S0DTiph+2WAZMkTSQLjLOAc+ppVESc2/1c0lyyK7kcGmZmTaCey3G7ImJRRLwjPb4YEc9GxM6IqHlJbERsB84HlpLdTXBJRLRLuljSKQCSjpXUAZwBXC2pvdb+ukn6QtrmgHSp7kX1ddXMzPqCIqLnCtLvSd8aLxcRLy2qUX2tVCpFW1tbo5thZtavSFoeEaXK8noOVZVvtD/Z6OBFfdUwMzPrX+o5VPVE2aMzIr4CvK34ppmZWTPa44gjfXO8235kI5B6RipmZjYA1RMAi8qebwfWA2cW0hozM2t6ewyOiOjpklszMxtk6pnk8BBJX+qevkPSIkmH7IvGmZlZ86lnypFvAE+THZ46E3iK7HayZmY2CNVzjuNlEXF62fJnJK0sqD1mZtbk6hlxdEk6vntB0huAruKaZGZmzayeEceHgH9P5zUEPAm8p9BWmZlZ06rnqqqVwGsljUxFz5BNWLiqwHaZmVmTqnmoKt3ve6GkKyW9hewE+buBtfh7HGZmg1ZPI47/AP4M3AO8H7iQ7FDVO9IoxMzMBqGeguOlEfFqAEnXAI8B4yPir/ukZWZm1pR6Co7n7+4XETskdTg0zMya3833d3LZ0jVs3NzF6FEtzJ85mdnTxvTZ/nsKjtdKeio9F9CSlgVERIysvamZmTXCzfd3svCm1XRt2wFA5+YuFt60GqDPwqPmyfGIGBIRI9Pj4IgYWvbcoWFm1oQuW7rm+dDo1rVtB5ctXdNnr1HPFwDNzKyf2Li5+veza5X3hoPDzGwAGT2qJVd5bzg4zMwGkPkzJ9MybMguZS3DhjB/5uQ+ew3fyc/MbADpPgHeqKuqzMysH5o9bUyfBkUlH6oyM7NcHBxmZpaLg8PMzHJxcJiZWS4ODjMzy6XQ4JA0S9IaSWslLaiy/kRJKyRtlzSnyvqRkjokXVlWNl3S6rTPKySpyD6YmdmuCgsOSUOAq4CTgSnA2ZKmVFR7FJgLXFdjN5cAd1aUfZXs/iCT0mNWHzXZzMzqUOSI4zhgbUSsi4itwPXAqeUVImJ9RKwCdlZuLGk6cDhwe1nZEcDIiLg3IgL4FjC7uC6YmVmlIoNjDLChbLkjle2RpP2ARcAFVfbZUc8+JZ0nqU1S26ZNm+putJmZ9axZT47PA26NiI491qwhIhZHRCkiSq2trX3YNDOzwa3IKUc6gXFly2NTWT1mACdImgccBAyXtAW4PO2nN/s0M7M+UGRwLAMmSZpI9sf9LOCcejaMiHO7n0uaC5QiYkFafkrS3wL3Ae8G/qWP221mZj0o7FBVRGwHzgeWAg8DSyKiXdLFkk4BkHSspA7gDOBqSe117HoecA2wFngE+EkhHTAzs6qUXZw0sJVKpWhra2t0M8zM+hVJyyOiVFnerCfHzcysSTk4zMwsFweHmZnl4uAwM7NcHBxmZpaLg8PMzHJxcJiZWS4ODjMzy8XBYWZmuTg4zMwsFweHmZnl4uAwM7NcHBxmZpaLg8PMzHJxcJiZWS4ODjMzy8XBYWZmuTg4zMwsFweHmZnl4uAwM7NcHBxmZpaLg8PMzHJxcJiZWS4ODjMzy8XBYWZmuTg4zMwsFweHmZnlUmhwSJolaY2ktZIWVFl/oqQVkrZLmlNWfmQqXympXdIHy9a9U9KqVP75IttvZma7Kyw4JA0BrgJOBqYAZ0uaUlHtUWAucF1F+WPAjIg4GngdsEDSaEkvBi4D/i4ipgJ/I+nviuqDmZntrsgRx3HA2ohYFxFbgeuBU8srRMT6iFgF7Kwo3xoRz6XFEWXtfCnwu4jYlJZ/BpxeVAfMzGx3RQbHGGBD2XJHKquLpHGSVqV9fD4iNgJrgcmSJkgaCswGxtXY/jxJbZLaNm3aVK2KmZn1QtOeHI+IDRHxGuAo4D2SDo+IPwMfAr4H/BpYD+yosf3iiChFRKm1tXVfNdvMbMArMjg62XU0MDaV5ZJGGg8CJ6TlH0bE6yJiBrAG+G0ftNXMzOpUZHAsAyZJmihpOHAWcEs9G0oaK6klPT8UOJ4sJJD0krLyecA1BbTdzMxqKCw4ImI7cD6wFHgYWBIR7ZIulnQKgKRjJXUAZwBXS2pPm78SuE/SA8AdwBcjYnVad7mkh4C7gEsjwiMOM7N9SBHR6DYUrlQqRVtbW6ObYWbWr0haHhGlyvKmPTluZmbNycFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5ODjMzCyXoY1ugBXj5vs7uWzpGjZu7mL0qBbmz5zM7GljGt0sMxsAHBwD0M33d7LwptV0bdsBQOfmLhbetBrA4WFme82Hqgagy5aueT40unVt28FlS9c0qEVmNpA4OAagjZu7cpWbmeXh4BiARo9qyVVuZpaHg2MAmj9zMi3DhuxS1jJsCPNnTm5Qi8xsIPHJ8QGo+wS4r6oysyIUGhySZgGXA0OAayLi0or1JwJfAV4DnBURN6byI4Hvk42IhgH/EhFfS+vOBj4BBLAReFdEPF5kP/qj2dPGOCjMrBCFHaqSNAS4CjgZmAKcLWlKRbVHgbnAdRXljwEzIuJo4HXAAkmjJQ0lC6KTIuI1wCrg/KL6YGZmuyvyHMdxwNqIWBcRW4HrgVPLK0TE+ohYBeysKN8aEc+lxRFl7VR6HChJwEiyUYeZme0jRQbHGGBD2XJHKquLpHGSVqV9fD4iNkbENuBDwGqywJgC/FuN7c+T1CapbdOmTb3tg5mZVWjaq6oiYkM6HHUU8B5Jh0saRhYc04DRZIeqFtbYfnFElCKi1Nraus/abWY20BUZHJ3AuLLlsaksl4jYCDwInAAcncoeiYgAlgCv3+uWmplZ3Yq8qmoZMEnSRLLAOAs4p54NJY0FnoiILkmHAscDXwaeAKZIao2ITcBbgIf3tL/ly5c/LukPvexHf3AYMFivLHPfB6fB3HfYd/0/slphYcEREdslnQ8sJbsc9xsR0S7pYqAtIm6RdCzZZbeHAm+X9JmImAq8ElgkKchOhn8xIlYDSPoMcKekbcAfyK7K2lNbBvSxKkltEVFqdDsawX133wejRvdf2REf688a/UPUSO67+z4YNbr/TXty3MzMmpODY2BY3OgGNJD7PjgN5r5Dg/vvQ1VmZpaLRxxmZpaLg8PMzHJxcDShNN3KLyU9JKld0v8pW/dhSf+Vyr+QyoZJ+ndJqyU9LGlhWf1ZktZIWitpQSP6k0etvkv6nqSV6bFe0sqybRam/q2RNLOsfED3XdJbJC1P7/tySW8q29f0VL5W0hVpbrem1pv3Pq0fL2mLpAvKygb0e5/WvUbSPan+akn7p/Li3/uI8KPJHsARwDHp+cHAb8nm5ToJ+BkwIq17Sfr3HOD69PwAYD0wgez7M48ALwWGAw8AUxrdv970vaLOIuDT6fmU1K8RwMTU3yGDpO/TgNHp+auAzrJ6vwH+lux7UD8BTm50//q6/2VlNwI3ABek5cHw3g8lm3LptWn5xcCQffXee8TRhCLisYhYkZ4/Tfbt+DFk83RdGmnm4Ij4U/cmZDMGDwVagK3AU9QxQ3Gz6aHvAKRPT2cC301Fp5KF5nMR8XtgLVm/B3zfI+L+yKbkAWgHWiSNkHQEMDIi7o3sL8m3gNn7rie904v3Hkmzgd+T9b/bgH/vgf8JrIqIB9I2T0TEjn313js4mpykCWSfLO8DXg6cIOk+SXekb95D9onrGbL7mDxK9k37J9nLGYobraLv3U4A/hgRv0vLtfo4GPpe7nRgRfpQMYasv936Vd+hvv5LOgj4OPCZis0Hw3v/ciAkLZW0QtI/pvJ98t771rFNLP1i/CfwkYh4Ko0oXkQ2DD0WWCLppWSfsHaQzRh8KPBrST9rULP7RGXfy1adTdknzoEob98lTQU+T/YptN/L0f+LgC9HxJZ+cAqnLjn6PpRsDr9jgWeBn0taDvxlX7TTwdGklE0h/5/AdyLiplTcAdyUhqC/kbSTbLKzc4DbIrtfyZ8k3QWUyD517fUMxftajb6TgvM0YHpZ9Z5mYR7ofe+eEPT7wLsj4pFU3EnW3279ou+Qu/+vA+You0hkFLBT0l+B5Qz8974DuDPSbbMl3QocA3ybffHeN/qkkB9VT5SJ7NjkVyrKPwhcnJ6/nCwYRDZc/2YqPxB4iOw+7kOBdWQnjbtPEk5tdP960/e0bhZwR0XZVHY9Ob6O7OToYOj7qNSv06rUrzxB+tZG96+v+1+x/iJeODk+GN77Q4EVZBfDDCW7aOZt++q9b/h/mB9Vf4iOJzvhvQpYmR5vTb8E3ya7P8kK4E2p/kFkV5W0p9CYX7avt5JdofEIcGGj+9bbvqd11wIfrLLNhal/ayi7gmSg9x34JNm5rZVlj+4r7Urp5+QR4ErSLBHN/OjNe1+27fPBMRje+1T+rvQ7/yDwhbLywt97TzliZma5+KoqMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHNbvSfqypI+ULS+VdE3Z8iJJH+th+2slzUnPfyVpt3s5K5uB+FJJv0tTPNwj6eS0br2kw3rR7udft8b6q9KsqA9J6iqbJXWOpFsljcr7mnW06QhJP+ph/XBJd6Yvpdkg5eCwgeAu4PUAkvYj+zb91LL1rwfu3svXuIRsBtNXRcQxZBPHHbyX++xRRPzviDia7DsJj0TE0elxY0S8NSI2F/CyHwO+3kObtgI/B95ZwGtbP+HgsIHgbmBGej6V7MtPT0s6VNII4JXACkmflrRM0oOSFtd7nwJJBwDvBz4cL8xM/MeIWFKl7sfS/h+sGAW9W9IqSQ9I+o8q212SRiBD6mzTekmHSZqg7P4s10r6raTvSHqzpLvS6Oi4VP9ASd+Q9BtJ90uqNVvs6cBtaZupqf7K1PZJqc7NwLn1tNMGJg83rd+LiI2StksaTza6uIdsRtAZZJO+rY6IrZKujIiLAdIf778HfljHSxwFPBq7Tjq3G0nTgfeSzaEk4D5Jd5BNc/9J4PUR8bikF1VsdxnZ6OW90btv5B4FnAG8D1hGNnfZ8cApwCfIRkcXAr+IiPelQ1y/kfSziHimrB0TgT93hyPZFDeXR8R3JA0nm8oFsmDunpnZBiGPOGyguJssNLqD456y5btSnZOUTUm/GngTux7O6gvHA9+PiGciYgtwE9l02G8Cbog0IV1kU953+xRwSER8sJehAfD7iFgdETvJpqD4edrXarIbekE2c+4CZXeQ+xWwPzC+Yj9HAJvKlu8BPiHp48CREdGV2r8D2Cqp0EN11rwcHDZQdJ/neDXZJ+J7yUYcrwfuVnZbzX8F5kTEq8mO4+9f577XAuMljezzVmcjhOmVo5Ccnit7vrNseScvHFUQcHrZeZLxEfFwxX66KPs/iYjryEYtXcCtKrs1Ldmkkn/dizZbP+bgsIHibrJDT09GxI70qX4UWXjczQt/EB9P9zyoeTVTpYh4Fvg34PJ0yAZJrZLOqKj6a2C2pAMkHQi8I5X9AjhD0ovTtuUhcRtwKfDjgj/BLwU+3H1eR9K0KnV+ywsjFJTd62VdRFwB/IBsxmVSPx6PbBp/G4QcHDZQrCa7mureirK/RMTj6Qqkr5ONRpaSfdLP45Nkh3EekvQg8COy2/M+L7Jbf15LNq31fcA1kd3etR34HHCHpAeAL1Vsd0Nq2y2SWnK2q16XAMOAVZLa0/Iu0vmORyQdlYrOBB5Mh7deRTbtN8BJwI8Laqf1A54d18yeJ+kdwPSI+GQPdW4CFkTEb/ddy6yZ+KoqM3teRHy/+5BaNelQ3c0OjcHNIw4zM8vF5zjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcvn/bO0s6LCmZj4AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "from flaml.data import get_output_from_log\n", + "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", + " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n", + "for config in config_history:\n", + " print(config)\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "plt.title('Learning Curve')\n", + "plt.xlabel('Wall Clock Time (s)')\n", + "plt.ylabel('Rouge 1')\n", + "print(len(valid_loss_history))\n", + "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", + "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AzID7DyALObP" + }, + "outputs": [], + "source": [ + "" ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" } - ], - "source": [ - "\n", - "from flaml.data import get_output_from_log\n", - "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", - " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n", - "for config in config_history:\n", - " print(config)\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "plt.title('Learning Curve')\n", - "plt.xlabel('Wall Clock Time (s)')\n", - "plt.ylabel('Rouge 1')\n", - "print(len(valid_loss_history))\n", - "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", - "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", - "plt.show()" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "e9d36fc5b7c3dd4177ff1b60184dd696c0acc18150a44682abca4d769811bd46" + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "Copy of automl_nlp.ipynb", + "provenance": [], + "include_colab_link": true + }, + "gpuClass": "standard", + "interpreter": { + "hash": "e9d36fc5b7c3dd4177ff1b60184dd696c0acc18150a44682abca4d769811bd46" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "007a43463f5e4da3983f59dfeb793e64": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "069feb62f1ec4392b04ee1d80aa4b445": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0c3a0eb88b16493e9d0f62e3d5abf195": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "155d7e95c2504507b83b12dc60f1edc1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_feaf1f712b8e499db2558dc0fdd4261e", + "IPY_MODEL_51392c27affa4fd3b4184cde01b7029d", + "IPY_MODEL_5dd6914461ea456e9dc96ccf8c391c6e" + ], + "layout": "IPY_MODEL_069feb62f1ec4392b04ee1d80aa4b445" + } + }, + "18091d361aa44881a3db5d1951882082": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1a701b0fa5a34fb9b64fb92b5c8e4306": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1fb6335656b1444abe05aa94a7d13825": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6a24dd061b2d40d4baca4036059fc94c", + "placeholder": "​", + "style": "IPY_MODEL_9fdc731eb58247cbafef9286b49c66f9", + "value": " 19566/20006 [00:02<00:00, 9081.23 examples/s]" + } + }, + "1fed6cdd71c4453b976e8651b7b34cae": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1ffd6e8c1f834dc48d66116b6089f7a2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "20624397998c4e188b419c6267affb65": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "20abb46ea9c948b8ba85a921aee8af6d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21f848683b2648c08a7476658d382177": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "263e99ca21124b79a26e1078b187273d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a91dc7ddd7f641d9b60b59bbbde7bae6", + "placeholder": "​", + "style": "IPY_MODEL_96c6874b9bd045a5bc67596b2ab04df2", + "value": "Generating train split: 100%" + } + }, + "26885bbae6c646e7bcc4a0459620c37a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26f3abaf861a4c63986ff0691294d70c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2a1aa694683d4df9b509f5ce4d6d53b0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2d975d14c3f0434583e73ae97f580951": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2e00672f9d1f46cea3e5db651bca19a3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_cc378c1990634f7da6ffba019fe38c59", + "IPY_MODEL_642323b1bafa4d0fbeca1adff2426c02", + "IPY_MODEL_a902290681e942cbae40024baaa2e9b7" + ], + "layout": "IPY_MODEL_3a014eef1b7d44698572bc5cada4cb8c" + } + }, + "2e0b939889d84c07a90d36a57065aac4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "35151380719b41349a2113b0b893bd6f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_efca4a3072e94170a1b851f9dec6164d", + "max": 20006, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_791101442898470f8524ebee4cb9459d", + "value": 20006 + } + }, + "3588f07c45694ec4a484afaaa9e9c599": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_998b0ca5b37b47b88ea47327462c76fa", + "IPY_MODEL_38bb77cefa2e4c17b8e9c419125d6c45", + "IPY_MODEL_ded6921a6b8140b3bcd59d0e7bbd7900" + ], + "layout": "IPY_MODEL_ad994aff0bf94c2ea4ac9aa8d5c067e3" + } + }, + "362f58e6d05f4d0c865cbe6a956d677b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "37d4912ed8ee4c0c9f0a9187bad156fd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3b6eaa3d64924ec581c412b04b9196fa", + "placeholder": "​", + "style": "IPY_MODEL_94a2d9480adc426abb4ade344ca8dd2f", + "value": "Downloading data: " + } + }, + "38bb77cefa2e4c17b8e9c419125d6c45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_760feb714cc54846a52fc399703891d7", + "max": 2348, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a21f2bb483e44c749ec41a2b1784ee4b", + "value": 2348 + } + }, + "3a014eef1b7d44698572bc5cada4cb8c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3b684b9f50ce48ff92b075d62619368b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a3e42e8e532c4628abaa6e154d667ed2", + "IPY_MODEL_49290455306b47aaaf8153bed5e49742", + "IPY_MODEL_d8468fc2f0b94b2b8dab75336a0d29a3" + ], + "layout": "IPY_MODEL_b09d990f98f0419e84f5939d3b48d381" + } + }, + "3b6eaa3d64924ec581c412b04b9196fa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "44cf4e612b5f482d8bf224413c1bc852": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_805c722b7dec4fc59ef64f8704e29424", + "placeholder": "​", + "style": "IPY_MODEL_0c3a0eb88b16493e9d0f62e3d5abf195", + "value": "Downloading data files: 100%" + } + }, + "46e340fe82414a58b283c78cdf953773": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "49290455306b47aaaf8153bed5e49742": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d226d72577ea4cc299ed78c2fa99a486", + "max": 2214653, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_82e0dbe1328a4c54807932984e0c4efb", + "value": 2214653 + } + }, + "4b2b38b8064040849b10c63b9f2ed8fd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b92faf53c2b4f7986066af8026ffc3a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4e956bd06d3a45eca66b192990416a62": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ab2c48f34b7f43c5a5fb37c80a7d47f3", + "placeholder": "​", + "style": "IPY_MODEL_834bdf06646a4d009648b6bc270c7624", + "value": "Generating test split: 98%" + } + }, + "4fa9926221cb4d29bc0cc0c3d0bf93f3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4e956bd06d3a45eca66b192990416a62", + "IPY_MODEL_e319f91c3ba841f99a9a1ca1c7b551f2", + "IPY_MODEL_da31e023dddb4c25a035258c0e4ed0d7" + ], + "layout": "IPY_MODEL_574c7a42dad940379a96b9f0968d3be1" + } + }, + "51392c27affa4fd3b4184cde01b7029d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db589666b507409f9647930b1222b0a9", + "max": 3, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1fed6cdd71c4453b976e8651b7b34cae", + "value": 3 + } + }, + "574c7a42dad940379a96b9f0968d3be1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "57aca5124cc14ed69da5a0b24a2c1052": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5a392cc22ee84433bac08ef8a6a3e0d4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5ad8132df42340c58f1375b1e52eb5bc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "5dd6914461ea456e9dc96ccf8c391c6e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f307d9be05d24940b57d9edc82be8976", + "placeholder": "​", + "style": "IPY_MODEL_20abb46ea9c948b8ba85a921aee8af6d", + "value": " 3/3 [00:00<00:00, 85.61it/s]" + } + }, + "5ea642008bf74641a021e17b7e3fd6e7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6284429508d849bd8259460913efc250": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d4830572efa244968881c31932ec5dff", + "max": 2238601, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_21f848683b2648c08a7476658d382177", + "value": 2238601 + } + }, + "62bdec145ccb48faa4fe5f51d2879732": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "642323b1bafa4d0fbeca1adff2426c02": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5ea642008bf74641a021e17b7e3fd6e7", + "max": 1775, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2d975d14c3f0434583e73ae97f580951", + "value": 1775 + } + }, + "6a24dd061b2d40d4baca4036059fc94c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6be493d86857493190ee47a08c04ff40": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "760feb714cc54846a52fc399703891d7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "791101442898470f8524ebee4cb9459d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7dc52faf4b3b4643b7d7019f1722c1d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_44cf4e612b5f482d8bf224413c1bc852", + "IPY_MODEL_f4ec7bc190af4c9dbe6a5fc05fad4540", + "IPY_MODEL_9dd6ab0e0cb940bebe25cba5492b2486" + ], + "layout": "IPY_MODEL_e556c049fbb24669a49b26c7f107e6a5" + } + }, + "7feeefa264da4af89ddc8ddf331b4f9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_be21476b15a14c0084712a9d5aedc22f", + "placeholder": "​", + "style": "IPY_MODEL_362f58e6d05f4d0c865cbe6a956d677b", + "value": " 73335/73546 [00:15<00:00, 4205.15 examples/s]" + } + }, + "805c722b7dec4fc59ef64f8704e29424": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "811d914b52904fa0913adc9daa33695c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "82e0dbe1328a4c54807932984e0c4efb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "834bdf06646a4d009648b6bc270c7624": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8497fe93c0d148a49f9a0a0c56961f36": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "87639f90c2ab47db986419c03e165d7a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "894de256daec49329f6404326eddaa39": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_57aca5124cc14ed69da5a0b24a2c1052", + "max": 73546, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_8f2c2b10e21e42569ef5396e42c65e30", + "value": 73546 + } + }, + "8f2c2b10e21e42569ef5396e42c65e30": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "8f5311cebd554f5ba645b8d33b0722a3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "920c8dd736a4454f9469fb3fa0a9af90": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "929cdd7c2f8e4902aac96a9a3afa5866": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "94a2d9480adc426abb4ade344ca8dd2f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9509ad27a9b54e3e80f796d224f3e189": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_18091d361aa44881a3db5d1951882082", + "placeholder": "​", + "style": "IPY_MODEL_2a1aa694683d4df9b509f5ce4d6d53b0", + "value": " 28.2M/? [00:00<00:00, 56.9MB/s]" + } + }, + "96c6874b9bd045a5bc67596b2ab04df2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "98bcaff9e28547e3b1f9b0640d598f99": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "98f179b9be5044c79bc867f5261e2b47": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "998b0ca5b37b47b88ea47327462c76fa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6be493d86857493190ee47a08c04ff40", + "placeholder": "​", + "style": "IPY_MODEL_46e340fe82414a58b283c78cdf953773", + "value": "Downloading builder script: " + } + }, + "9cbec0a1fe3247ed8a46290df56756fa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_26885bbae6c646e7bcc4a0459620c37a", + "placeholder": "​", + "style": "IPY_MODEL_a56b058d910244588c1454b02c8cda8c", + "value": "Generating validation split: 98%" + } + }, + "9cf2c0d8439a4f5a86b4769a27babb94": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9dd6ab0e0cb940bebe25cba5492b2486": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_aa1c4b91b583440a8e6f79dd06cfd200", + "placeholder": "​", + "style": "IPY_MODEL_b37bd95afbe44cd196fc5ab2d52bccd0", + "value": " 3/3 [00:13<00:00, 4.35s/it]" + } + }, + "9fdc731eb58247cbafef9286b49c66f9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a18b9c22460940cfa53b657849b034bf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a21f2bb483e44c749ec41a2b1784ee4b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a3e42e8e532c4628abaa6e154d667ed2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b92faf53c2b4f7986066af8026ffc3a", + "placeholder": "​", + "style": "IPY_MODEL_8497fe93c0d148a49f9a0a0c56961f36", + "value": "Downloading data: " + } + }, + "a56b058d910244588c1454b02c8cda8c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a6431dfb76084a838d63849fa362de35": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a902290681e942cbae40024baaa2e9b7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_26f3abaf861a4c63986ff0691294d70c", + "placeholder": "​", + "style": "IPY_MODEL_1a701b0fa5a34fb9b64fb92b5c8e4306", + "value": " 7.10k/? [00:00<00:00, 221kB/s]" + } + }, + "a91dc7ddd7f641d9b60b59bbbde7bae6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aa1c4b91b583440a8e6f79dd06cfd200": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aad092fcb29d4045a342288aa9d6a329": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ab2c48f34b7f43c5a5fb37c80a7d47f3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad994aff0bf94c2ea4ac9aa8d5c067e3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b09d990f98f0419e84f5939d3b48d381": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b167b817426e4832b73a7a37b72115c1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b37bd95afbe44cd196fc5ab2d52bccd0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b649c32bcc8446cf91c53604fc1dcaa6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b70e2d813be3473f90ca76e293251c0b", + "IPY_MODEL_b66e17d6fd094f44bc10eade34fc5261", + "IPY_MODEL_9509ad27a9b54e3e80f796d224f3e189" + ], + "layout": "IPY_MODEL_920c8dd736a4454f9469fb3fa0a9af90" + } + }, + "b66e17d6fd094f44bc10eade34fc5261": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_811d914b52904fa0913adc9daa33695c", + "max": 6710578, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_98f179b9be5044c79bc867f5261e2b47", + "value": 6710578 + } + }, + "b70e2d813be3473f90ca76e293251c0b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8f5311cebd554f5ba645b8d33b0722a3", + "placeholder": "​", + "style": "IPY_MODEL_aad092fcb29d4045a342288aa9d6a329", + "value": "Downloading data: " + } + }, + "be21476b15a14c0084712a9d5aedc22f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bfbcfceca0444337ac6c4033a7734fc1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c1792bbceb854dc5880003f64e5623cb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c274c717ac7e4fa2888e0d101c3fe1fb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9cbec0a1fe3247ed8a46290df56756fa", + "IPY_MODEL_35151380719b41349a2113b0b893bd6f", + "IPY_MODEL_1fb6335656b1444abe05aa94a7d13825" + ], + "layout": "IPY_MODEL_a6431dfb76084a838d63849fa362de35" + } + }, + "c96613989db447b5acfb35cfef553145": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ca550fa3fe1147bd8285c2b7cadde206": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_263e99ca21124b79a26e1078b187273d", + "IPY_MODEL_894de256daec49329f6404326eddaa39", + "IPY_MODEL_7feeefa264da4af89ddc8ddf331b4f9f" + ], + "layout": "IPY_MODEL_4b2b38b8064040849b10c63b9f2ed8fd" + } + }, + "cc378c1990634f7da6ffba019fe38c59": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c1792bbceb854dc5880003f64e5623cb", + "placeholder": "​", + "style": "IPY_MODEL_b167b817426e4832b73a7a37b72115c1", + "value": "Downloading metadata: " + } + }, + "d226d72577ea4cc299ed78c2fa99a486": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d4830572efa244968881c31932ec5dff": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8468fc2f0b94b2b8dab75336a0d29a3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_eb395373be244e6e8815087d5d32a801", + "placeholder": "​", + "style": "IPY_MODEL_1ffd6e8c1f834dc48d66116b6089f7a2", + "value": " 7.82M/? [00:00<00:00, 38.6MB/s]" + } + }, + "da31e023dddb4c25a035258c0e4ed0d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_87639f90c2ab47db986419c03e165d7a", + "placeholder": "​", + "style": "IPY_MODEL_a18b9c22460940cfa53b657849b034bf", + "value": " 19617/20005 [00:02<00:00, 9178.75 examples/s]" + } + }, + "db589666b507409f9647930b1222b0a9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ded6921a6b8140b3bcd59d0e7bbd7900": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9cf2c0d8439a4f5a86b4769a27babb94", + "placeholder": "​", + "style": "IPY_MODEL_007a43463f5e4da3983f59dfeb793e64", + "value": " 7.97k/? [00:00<00:00, 244kB/s]" + } + }, + "e1f77bef878c4b0bbfac867c5a9eea98": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5a392cc22ee84433bac08ef8a6a3e0d4", + "placeholder": "​", + "style": "IPY_MODEL_98bcaff9e28547e3b1f9b0640d598f99", + "value": " 7.89M/? [00:00<00:00, 41.2MB/s]" + } + }, + "e319f91c3ba841f99a9a1ca1c7b551f2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e0b939889d84c07a90d36a57065aac4", + "max": 20005, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_929cdd7c2f8e4902aac96a9a3afa5866", + "value": 20005 + } + }, + "e556c049fbb24669a49b26c7f107e6a5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "eb395373be244e6e8815087d5d32a801": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "efca4a3072e94170a1b851f9dec6164d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f307d9be05d24940b57d9edc82be8976": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f4ec7bc190af4c9dbe6a5fc05fad4540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bfbcfceca0444337ac6c4033a7734fc1", + "max": 3, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_5ad8132df42340c58f1375b1e52eb5bc", + "value": 3 + } + }, + "f74dfe0a3de64c3ea051e14fba9a04e4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_37d4912ed8ee4c0c9f0a9187bad156fd", + "IPY_MODEL_6284429508d849bd8259460913efc250", + "IPY_MODEL_e1f77bef878c4b0bbfac867c5a9eea98" + ], + "layout": "IPY_MODEL_20624397998c4e188b419c6267affb65" + } + }, + "feaf1f712b8e499db2558dc0fdd4261e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_62bdec145ccb48faa4fe5f51d2879732", + "placeholder": "​", + "style": "IPY_MODEL_c96613989db447b5acfb35cfef553145", + "value": "Extracting data files: 100%" + } + }, + "16fba9eb9e4542bc9d34eca00d71cc14": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c81f11a99b9d4d1d95533f24bea1d5ac", + "IPY_MODEL_e4ce5cf6ea174583a14675a75d31992d", + "IPY_MODEL_0c8473019e434db0ae34d58b69605a69" + ], + "layout": "IPY_MODEL_814d3f2b7212461ca51f8635b5106783", + "tabbable": null, + "tooltip": null + } + }, + "c81f11a99b9d4d1d95533f24bea1d5ac": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_129488cadbb9477ca593ae106ee8e9f7", + "placeholder": "​", + "style": "IPY_MODEL_c7aaa1fbd10942649c90044c0c901d99", + "tabbable": null, + "tooltip": null, + "value": "Downloading data: 100%" + } + }, + "e4ce5cf6ea174583a14675a75d31992d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_6fe78cfd377b4c10a626588f46a569cf", + "max": 7439277, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2aa0f33b8d3a4bc7bedf3b66e06b62f0", + "tabbable": null, + "tooltip": null, + "value": 7439277 + } + }, + "0c8473019e434db0ae34d58b69605a69": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_1794a34790b647b3a8a845c55e5e0744", + "placeholder": "​", + "style": "IPY_MODEL_13d813116e1846a3a0e42a5e8423f80e", + "tabbable": null, + "tooltip": null, + "value": " 7.44M/7.44M [00:00<00:00, 13.9MB/s]" + } + }, + "814d3f2b7212461ca51f8635b5106783": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "129488cadbb9477ca593ae106ee8e9f7": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c7aaa1fbd10942649c90044c0c901d99": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "6fe78cfd377b4c10a626588f46a569cf": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2aa0f33b8d3a4bc7bedf3b66e06b62f0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1794a34790b647b3a8a845c55e5e0744": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "13d813116e1846a3a0e42a5e8423f80e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "2aa02213244048fead33ea157c17837b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_dfee126a02ef4934a0f654a101dadc1b", + "IPY_MODEL_8e58d795d528405f8d1c48bfc2afe399", + "IPY_MODEL_e23212ae504e493a85e4b2524f0217e1" + ], + "layout": "IPY_MODEL_b84af540de4741ceb206456d2f05fa4b", + "tabbable": null, + "tooltip": null + } + }, + "dfee126a02ef4934a0f654a101dadc1b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_9ba620aa4e51456c9b3b2469c2c887c3", + "placeholder": "​", + "style": "IPY_MODEL_cc8f1bc7322542828205777903530f1e", + "tabbable": null, + "tooltip": null, + "value": "Generating train split: 98%" + } + }, + "8e58d795d528405f8d1c48bfc2afe399": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_c5f0ba4cda014a63a99ecc989f72f731", + "max": 67349, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_44803ba97fd54ceab2afe0555e21dfe8", + "tabbable": null, + "tooltip": null, + "value": 67349 + } + }, + "e23212ae504e493a85e4b2524f0217e1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f4ca1b7b5868446da945574f4db4373b", + "placeholder": "​", + "style": "IPY_MODEL_f7070757d4784c8099ef7dd9bd280ed3", + "tabbable": null, + "tooltip": null, + "value": " 65772/67349 [00:03<00:00, 18334.90 examples/s]" + } + }, + "b84af540de4741ceb206456d2f05fa4b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9ba620aa4e51456c9b3b2469c2c887c3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cc8f1bc7322542828205777903530f1e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "c5f0ba4cda014a63a99ecc989f72f731": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "44803ba97fd54ceab2afe0555e21dfe8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "f4ca1b7b5868446da945574f4db4373b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f7070757d4784c8099ef7dd9bd280ed3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "6a8be136bafe40eea3430dda4063e6db": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f28c1b9dee064db99c389beb98306f86", + "IPY_MODEL_b2a5550cd6474de7a46eab6a973305e0", + "IPY_MODEL_a4e9b7b28055406c9569e585296850c6" + ], + "layout": "IPY_MODEL_ac534b02efb34100b53999031767e8a3", + "tabbable": null, + "tooltip": null + } + }, + "f28c1b9dee064db99c389beb98306f86": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_5f9f744825e44fdbae9c126837e40efe", + "placeholder": "​", + "style": "IPY_MODEL_c3f7a2bb90b44a21a43939f78914f9b8", + "tabbable": null, + "tooltip": null, + "value": "Generating validation split: 92%" + } + }, + "b2a5550cd6474de7a46eab6a973305e0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_4ec28fbc9433413f8355e0c976839a94", + "max": 872, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_fab424da92b541cfac6b3bb05ee4e17b", + "tabbable": null, + "tooltip": null, + "value": 872 + } + }, + "a4e9b7b28055406c9569e585296850c6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_8cfa8c0a28f549c19649ec9b390aa528", + "placeholder": "​", + "style": "IPY_MODEL_470b17af8c8442e49757dc4e385d16f0", + "tabbable": null, + "tooltip": null, + "value": " 798/872 [00:00<00:00, 7979.17 examples/s]" + } + }, + "ac534b02efb34100b53999031767e8a3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f9f744825e44fdbae9c126837e40efe": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c3f7a2bb90b44a21a43939f78914f9b8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "4ec28fbc9433413f8355e0c976839a94": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fab424da92b541cfac6b3bb05ee4e17b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "8cfa8c0a28f549c19649ec9b390aa528": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "470b17af8c8442e49757dc4e385d16f0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "2c31d2eb9ae44ffbb0f02ad1b1e7937a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_cd5ec1bd9cf54dbfbfd577a096a9e588", + "IPY_MODEL_d1ba6870db484696879f0d6e5d3a9d70", + "IPY_MODEL_e7405147ca374cc4998ca947be069652" + ], + "layout": "IPY_MODEL_fb72bbb82aec4184a8e0a510177433cf", + "tabbable": null, + "tooltip": null + } + }, + "cd5ec1bd9cf54dbfbfd577a096a9e588": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f37c66b863854588b7a8891720372dc6", + "placeholder": "​", + "style": "IPY_MODEL_f48c7243c49a43acac4d1ba3a6fe674f", + "tabbable": null, + "tooltip": null, + "value": "Generating test split: 50%" + } + }, + "d1ba6870db484696879f0d6e5d3a9d70": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_95ad5a87c66f405599a710b8a5fa0a9d", + "max": 1821, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_de103ee4780843db8502ebe64f5d2b28", + "tabbable": null, + "tooltip": null, + "value": 1821 + } + }, + "e7405147ca374cc4998ca947be069652": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "2.0.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_428db79c8cd74257ad09539518a21835", + "placeholder": "​", + "style": "IPY_MODEL_3c95b6d54b294fe2a958056c463ce541", + "tabbable": null, + "tooltip": null, + "value": " 915/1821 [00:00<00:00, 9145.67 examples/s]" + } + }, + "fb72bbb82aec4184a8e0a510177433cf": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f37c66b863854588b7a8891720372dc6": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f48c7243c49a43acac4d1ba3a6fe674f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "95ad5a87c66f405599a710b8a5fa0a9d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de103ee4780843db8502ebe64f5d2b28": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "428db79c8cd74257ad09539518a21835": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3c95b6d54b294fe2a958056c463ce541": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLStyleModel", + "model_module_version": "2.0.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + } + } + } }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "nbformat": 4, + "nbformat_minor": 0 } From 2314cc5a7ef93d51bb999ba8c2e4b2a33f167019 Mon Sep 17 00:00:00 2001 From: Xueqing Liu Date: Mon, 22 Aug 2022 13:36:50 -0400 Subject: [PATCH 18/20] "intermediate_results" TypeError: argument of type 'NoneType' is not iterable (#695) * fix mlflow bug * bump version --- flaml/automl.py | 7 +++++-- flaml/version.py | 2 +- test/automl/test_notebook_example.py | 13 +++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/flaml/automl.py b/flaml/automl.py index 381a61a0a..66468237a 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -3069,7 +3069,9 @@ class AutoML(BaseEstimator): if mlflow is not None and mlflow.active_run(): with mlflow.start_run(nested=True): mlflow.log_metric("iter_counter", self._track_iter) - if "intermediate_results" in search_state.metric_for_logging: + if (search_state.metric_for_logging is not None) and ( + "intermediate_results" in search_state.metric_for_logging + ): for each_entry in search_state.metric_for_logging[ "intermediate_results" ]: @@ -3079,7 +3081,8 @@ class AutoML(BaseEstimator): "iter_counter", self._iter_per_learner[estimator] ) del search_state.metric_for_logging["intermediate_results"] - mlflow.log_metrics(search_state.metric_for_logging) + if search_state.metric_for_logging: + mlflow.log_metrics(search_state.metric_for_logging) mlflow.log_metric("trial_time", search_state.trial_time) mlflow.log_metric("wall_clock_time", self._state.time_from_start) mlflow.log_metric("validation_loss", search_state.val_loss) diff --git a/flaml/version.py b/flaml/version.py index 9eb1ebec5..bd538f76e 100644 --- a/flaml/version.py +++ b/flaml/version.py @@ -1 +1 @@ -__version__ = "1.0.11" +__version__ = "1.0.12" diff --git a/test/automl/test_notebook_example.py b/test/automl/test_notebook_example.py index 53b94d4d6..5f2382454 100644 --- a/test/automl/test_notebook_example.py +++ b/test/automl/test_notebook_example.py @@ -154,6 +154,19 @@ def test_mlflow(): pass # subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "mlflow"]) + from sklearn.datasets import load_iris + + with mlflow.start_run(): + automl = AutoML() + automl_settings = { + "time_budget": 2, # in seconds + "metric": "accuracy", + "task": "classification", + "log_file_name": "iris.log", + } + X_train, y_train = load_iris(return_X_y=True) + automl.fit(X_train=X_train, y_train=y_train, **automl_settings) + if __name__ == "__main__": test_automl(600) From e5c8a16fabc74098751598950911d7b319f0573b Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Mon, 22 Aug 2022 10:52:41 -0700 Subject: [PATCH 19/20] Update Research.md --- website/docs/Research.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/Research.md b/website/docs/Research.md index c5546c732..360aa0c52 100644 --- a/website/docs/Research.md +++ b/website/docs/Research.md @@ -20,4 +20,4 @@ For technical details, please check our research publications. * [Fair AutoML](https://arxiv.org/abs/2111.06495). Qingyun Wu, Chi Wang. ArXiv preprint arXiv:2111.06495 (2021). * [Mining Robust Default Configurations for Resource-constrained AutoML](https://arxiv.org/abs/2202.09927). Moe Kayali, Chi Wang. ArXiv preprint arXiv:2202.09927 (2022). -Many researchers and engineers have contributed to the technology development. In alphabetical order: Vijay Aski, Sebastien Bubeck, Surajit Chaudhuri, Kevin Chen, Yi Wei Chen, Nadiia Chepurko, Ofer Dekel, Alex Deng, Anshuman Dutt, Nicolo Fusi, Jianfeng Gao, Johannes Gehrke, Niklas Gustafsson, Silu Huang, Moe Kayali, Dongwoo Kim, Christian Konig, John Langford, Menghao Li, Mingqin Li, Susan Xueqing Liu, Zhe Liu, Naveen Gaur, Paul Mineiro, Vivek Narasayya, Jake Radzikowski, Marco Rossi, Amin Saied, Neil Tenenholtz, Olga Vrousgou, Chi Wang, Yue Wang, Markus Weimer, Qingyun Wu, Qiufeng Yin, Haozhe Zhang, Minjia Zhang, XiaoYun Zhang, Eric Zhu. +Many researchers and engineers have contributed to the technology development. In alphabetical order: Vijay Aski, Sebastien Bubeck, Surajit Chaudhuri, Kevin Chen, Yi Wei Chen, Nadiia Chepurko, Ofer Dekel, Alex Deng, Anshuman Dutt, Nicolo Fusi, Jianfeng Gao, Johannes Gehrke, Niklas Gustafsson, Silu Huang, Moe Kayali, Dongwoo Kim, Christian Konig, John Langford, Menghao Li, Mingqin Li, Susan Xueqing Liu, Zhe Liu, Naveen Gaur, Paul Mineiro, Vivek Narasayya, Jake Radzikowski, Marco Rossi, Amin Saied, Neil Tenenholtz, Olga Vrousgou, Chi Wang, Yue Wang, Markus Weimer, Qingyun Wu, Qiufeng Yin, Haozhe Zhang, Minjia Zhang, XiaoYun Zhang, Eric Zhu, Rui Zhuang. From 96f068859555ec1a1f9af389e647993ec7f2a213 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Wed, 24 Aug 2022 02:58:18 +0000 Subject: [PATCH 20/20] fix --- flaml/ml.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flaml/ml.py b/flaml/ml.py index 1d50512fb..e521c3239 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -536,6 +536,8 @@ def evaluate_model_CV( log_training_metric=log_training_metric, fit_kwargs=fit_kwargs, ) + if isinstance(metric_i, dict) and "intermediate_results" in metric_i.keys(): + del metric_i["intermediate_results"] if weight is not None: fit_kwargs["sample_weight"] = weight total_fold_num += 1