yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-07 23:13:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): setup scaffolding for tox testing (#2451)

Browse Source
This commit is contained in:
Harshal Sheth 2021-04-26 16:44:36 -07:00 committed by GitHub
parent e075acaec9
commit 2da5e1fd10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 37 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/pyproject.toml
View File

@ -1,6 +1,6 @@
[build-system] [build-system]
build-backend = "setuptools.build_meta" build-backend = "setuptools.build_meta"
requires = ["setuptools", "wheel", "pip"] requires = ["setuptools>=54.0.0", "wheel", "pip>=21.0.0"]
[tool.black] [tool.black]
extend-exclude = ''' extend-exclude = '''

1
metadata-ingestion/setup.cfg
View File

@ -11,6 +11,7 @@ exclude =
.git, .git,
src/datahub/metadata, src/datahub/metadata,
venv, venv,
.tox,
__pycache__ __pycache__
per-file-ignores = per-file-ignores =
# imported but unused # imported but unused

22
metadata-ingestion/setup.py
View File

@ -78,7 +78,7 @@ plugins: Dict[str, Set[str]] = {
"glue": {"boto3"}, "glue": {"boto3"},
} }
dev_requirements = { base_dev_requirements = {
*base_requirements, *base_requirements,
*framework_common, *framework_common,
"black>=19.10b0", "black>=19.10b0",
@ -89,17 +89,13 @@ dev_requirements = {
"pytest>=6.2.2", "pytest>=6.2.2",
"pytest-cov>=2.8.1", "pytest-cov>=2.8.1",
"pytest-docker", "pytest-docker",
"tox",
"sqlalchemy-stubs", "sqlalchemy-stubs",
"deepdiff", "deepdiff",
"requests-mock", "requests-mock",
"freezegun", "freezegun",
"build", "build",
"twine", "twine",
# Also add the plugins which are used for tests.
"apache-airflow==1.10.15",
"apache-airflow-backport-providers-snowflake", # Used in the example DAGs.
# "apache-airflow>=2.0.2",
# "apache-airflow-providers-snowflake",
*list( *list(
dependency dependency
for plugin in [ for plugin in [
@ -111,12 +107,23 @@ dev_requirements = {
"glue", "glue",
"datahub-kafka", "datahub-kafka",
"datahub-rest", "datahub-rest",
"airflow", # airflow is added below
] ]
for dependency in plugins[plugin] for dependency in plugins[plugin]
), ),
} }
dev_requirements = {
*base_dev_requirements,
"apache-airflow==1.10.15",
"apache-airflow-backport-providers-snowflake", # Used in the example DAGs.
}
dev_requirements_airflow_2 = {
*base_dev_requirements,
"apache-airflow>=2.0.2",
"apache-airflow-providers-snowflake",
}
setuptools.setup( setuptools.setup(
# Package metadata. # Package metadata.
@ -207,5 +214,6 @@ setuptools.setup(
}, },
"all": list(framework_common.union(*plugins.values())), "all": list(framework_common.union(*plugins.values())),
"dev": list(dev_requirements), "dev": list(dev_requirements),
"dev-airflow2": list(dev_requirements_airflow_2),
}, },
) )

20
metadata-ingestion/tox.ini Normal file
View File

@ -0,0 +1,20 @@
# tox (https://tox.readthedocs.io/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.
[tox]
envlist = py3-airflow{2,1}
[testenv]
extras = dev
usedevelop = true
commands =
# black --check .
# isort --check-only .
# flake8 --count --statistics .
# mypy .
pytest -m 'not slow' --cov-fail-under=70
[testenv:py3-airflow2]
extras = dev-airflow2