feat(ingest): setup scaffolding for tox testing (#2451)

metadata-ingestion/pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 build-backend = "setuptools.build_meta"
-requires = ["setuptools", "wheel", "pip"]
+requires = ["setuptools>=54.0.0", "wheel", "pip>=21.0.0"]
 
 [tool.black]
 extend-exclude = '''

metadata-ingestion/setup.cfg

@@ -11,6 +11,7 @@ exclude =
     .git,
     src/datahub/metadata,
     venv,
+    .tox,
     __pycache__
 per-file-ignores =
     # imported but unused

metadata-ingestion/setup.py

@@ -78,7 +78,7 @@ plugins: Dict[str, Set[str]] = {
     "glue": {"boto3"},
 }
 
-dev_requirements = {
+base_dev_requirements = {
     *base_requirements,
     *framework_common,
     "black>=19.10b0",
@@ -89,17 +89,13 @@ dev_requirements = {
     "pytest>=6.2.2",
     "pytest-cov>=2.8.1",
     "pytest-docker",
+    "tox",
     "sqlalchemy-stubs",
     "deepdiff",
     "requests-mock",
     "freezegun",
     "build",
     "twine",
-    # Also add the plugins which are used for tests.
-    "apache-airflow==1.10.15",
-    "apache-airflow-backport-providers-snowflake",  # Used in the example DAGs.
-    # "apache-airflow>=2.0.2",
-    # "apache-airflow-providers-snowflake",
     *list(
         dependency
         for plugin in [
@@ -111,12 +107,23 @@ dev_requirements = {
             "glue",
             "datahub-kafka",
             "datahub-rest",
-            "airflow",
+            # airflow is added below
         ]
         for dependency in plugins[plugin]
     ),
 }
 
+dev_requirements = {
+    *base_dev_requirements,
+    "apache-airflow==1.10.15",
+    "apache-airflow-backport-providers-snowflake",  # Used in the example DAGs.
+}
+dev_requirements_airflow_2 = {
+    *base_dev_requirements,
+    "apache-airflow>=2.0.2",
+    "apache-airflow-providers-snowflake",
+}
+
 
 setuptools.setup(
     # Package metadata.
@@ -207,5 +214,6 @@ setuptools.setup(
         },
         "all": list(framework_common.union(*plugins.values())),
         "dev": list(dev_requirements),
+        "dev-airflow2": list(dev_requirements_airflow_2),
     },
 )

metadata-ingestion/tox.ini

@@ -0,0 +1,20 @@
+# tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = py3-airflow{2,1}
+
+[testenv]
+extras = dev
+usedevelop = true
+commands =
+    # black --check .
+    # isort --check-only .
+    # flake8 --count --statistics .
+    # mypy .
+    pytest -m 'not slow' --cov-fail-under=70
+
+[testenv:py3-airflow2]
+extras = dev-airflow2