Merge branch 'microsoft:main' into main

test/nlp/test_autohf_summarization.py

@@ -4,7 +4,10 @@ import requests
 from utils import get_toy_data_summarization, get_automl_settings
 
 
-@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on mac os")
+@pytest.mark.skipif(
+    sys.platform == "darwin" or sys.version < "3.7",
+    reason="do not run on mac os or py < 3.7",
+)
 def test_summarization():
     # TODO: manual test for how effective postprocess_seq2seq_prediction_label is
     from flaml import AutoML

test/nlp/test_default.py

@@ -63,7 +63,6 @@ def test_starting_point_not_in_search_space():
         }
     }
     automl_settings["starting_points"] = "data:test/nlp/default/"
-    del automl_settings["fit_kwargs_by_estimator"][this_estimator_name]["model_path"]
 
     automl.fit(X_train, y_train, **automl_settings)
     assert len(automl._search_states[this_estimator_name].init_config) == len(
@@ -90,10 +89,13 @@ def test_points_to_evaluate():
     automl = AutoML()
     automl_settings = get_automl_settings(estimator_name="transformer_ms")
 
-    automl_settings["estimator_list"] = ["transformer_ms"]
-    automl_settings["starting_points"] = "data"
+    automl_settings["starting_points"] = "data:test/nlp/default/"
 
-    del automl_settings["fit_kwargs_by_estimator"]["transformer_ms"]["model_path"]
+    automl_settings["custom_hp"] = {
+        "transformer_ms": {
+            "model_path": {"domain": "google/electra-small-discriminator"}
+        }
+    }
 
     automl.fit(X_train, y_train, **automl_settings)
 
@@ -109,8 +111,6 @@ def test_zero_shot_nomodel():
 
     automl_settings = get_automl_settings(estimator_name)
 
-    del automl_settings["fit_kwargs_by_estimator"][estimator_name]["model_path"]
-
     (
         hyperparams,
         estimator_class,
@@ -146,10 +146,6 @@ def test_build_error_portfolio(path="./test/nlp/default", strategy="greedy"):
     location = "test/nlp/default"
     X_train, y_train, X_val, y_val, X_test = get_toy_data_seqclassification()
 
-    automl_settings = get_automl_settings(estimator_name)
-
-    del automl_settings["fit_kwargs_by_estimator"][estimator_name]["model_path"]
-
     try:
         (
             hyperparams,

test/nlp/utils.py

@@ -1514,6 +1514,15 @@ def get_automl_settings(estimator_name="transformer"):
         "use_ray": False,
     }
 
+    if estimator_name.endswith("ms"):
+        automl_settings["fit_kwargs_by_estimator"] = {
+            estimator_name: {
+                "output_dir": "test/data/output/",
+                "ckpt_per_epoch": 1,
+                "fp16": False,
+            }
+        }
+    else:
         automl_settings["fit_kwargs_by_estimator"] = {
             estimator_name: {
                 "model_path": "google/electra-small-discriminator",