From 2da5e1fd10ae8ee4d1cb0c939ecf3d34cd430ad8 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 26 Apr 2021 16:44:36 -0700
Subject: [PATCH] feat(ingest): setup scaffolding for tox testing (#2451)

---
 metadata-ingestion/pyproject.toml |  2 +-
 metadata-ingestion/setup.cfg      |  1 +
 metadata-ingestion/setup.py       | 22 +++++++++++++++-------
 metadata-ingestion/tox.ini        | 20 ++++++++++++++++++++
 4 files changed, 37 insertions(+), 8 deletions(-)
 create mode 100644 metadata-ingestion/tox.ini

diff --git a/metadata-ingestion/pyproject.toml b/metadata-ingestion/pyproject.toml
index ae12a75daa..bed2115f57 100644
--- a/metadata-ingestion/pyproject.toml
+++ b/metadata-ingestion/pyproject.toml
@@ -1,6 +1,6 @@
 [build-system]
 build-backend = "setuptools.build_meta"
-requires = ["setuptools", "wheel", "pip"]
+requires = ["setuptools>=54.0.0", "wheel", "pip>=21.0.0"]
 
 [tool.black]
 extend-exclude = '''
diff --git a/metadata-ingestion/setup.cfg b/metadata-ingestion/setup.cfg
index 200c6a05a1..9e726ccae4 100644
--- a/metadata-ingestion/setup.cfg
+++ b/metadata-ingestion/setup.cfg
@@ -11,6 +11,7 @@ exclude =
     .git,
     src/datahub/metadata,
     venv,
+    .tox,
     __pycache__
 per-file-ignores =
     # imported but unused
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 30ce814f43..1b5408f34f 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -78,7 +78,7 @@ plugins: Dict[str, Set[str]] = {
     "glue": {"boto3"},
 }
 
-dev_requirements = {
+base_dev_requirements = {
     *base_requirements,
     *framework_common,
     "black>=19.10b0",
@@ -89,17 +89,13 @@ dev_requirements = {
     "pytest>=6.2.2",
     "pytest-cov>=2.8.1",
     "pytest-docker",
+    "tox",
     "sqlalchemy-stubs",
     "deepdiff",
     "requests-mock",
     "freezegun",
     "build",
     "twine",
-    # Also add the plugins which are used for tests.
-    "apache-airflow==1.10.15",
-    "apache-airflow-backport-providers-snowflake",  # Used in the example DAGs.
-    # "apache-airflow>=2.0.2",
-    # "apache-airflow-providers-snowflake",
     *list(
         dependency
         for plugin in [
@@ -111,12 +107,23 @@ dev_requirements = {
             "glue",
             "datahub-kafka",
             "datahub-rest",
-            "airflow",
+            # airflow is added below
         ]
         for dependency in plugins[plugin]
     ),
 }
 
+dev_requirements = {
+    *base_dev_requirements,
+    "apache-airflow==1.10.15",
+    "apache-airflow-backport-providers-snowflake",  # Used in the example DAGs.
+}
+dev_requirements_airflow_2 = {
+    *base_dev_requirements,
+    "apache-airflow>=2.0.2",
+    "apache-airflow-providers-snowflake",
+}
+
 
 setuptools.setup(
     # Package metadata.
@@ -207,5 +214,6 @@ setuptools.setup(
         },
         "all": list(framework_common.union(*plugins.values())),
         "dev": list(dev_requirements),
+        "dev-airflow2": list(dev_requirements_airflow_2),
     },
 )
diff --git a/metadata-ingestion/tox.ini b/metadata-ingestion/tox.ini
new file mode 100644
index 0000000000..b674c2f313
--- /dev/null
+++ b/metadata-ingestion/tox.ini
@@ -0,0 +1,20 @@
+# tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = py3-airflow{2,1}
+
+[testenv]
+extras = dev
+usedevelop = true
+commands =
+    # black --check .
+    # isort --check-only .
+    # flake8 --count --statistics .
+    # mypy .
+    pytest -m 'not slow' --cov-fail-under=70
+
+[testenv:py3-airflow2]
+extras = dev-airflow2