feat(ci): pin tox requirements to speed up ci runs, remove airflow-1 suite until we can pin it (#4055)

This commit is contained in:
Swaroop Jagadish 2022-02-04 07:32:38 -08:00 committed by GitHub
parent 0e5811fd68
commit aab622793d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 720 additions and 10 deletions

View File

@ -39,7 +39,7 @@ framework_common = {
"docker",
"expandvars>=0.6.5",
"avro-gen3==0.7.1",
"avro>=1.10.2",
"avro>=1.10.2,<1.11",
"python-dateutil>=2.8.0",
"stackprinter",
"tabulate",

View File

@ -4,12 +4,12 @@
# and then run "tox" from this directory.
[tox]
envlist = py3-quick,py3-full,py3-airflow1
envlist = py3-quick,{py36,py39}-full,py3-airflow1
[gh-actions]
python =
3.6: py3-full, py3-airflow1
3.9: py3-full, py3-airflow1
3.6: py36-full
3.9: py39-full
# Providing optional features that add dependencies from setup.py as deps here
# allows tox to recreate testenv when new dependencies are added to setup.py.
@ -19,11 +19,14 @@ python =
[testenv]
deps =
.[dev]
-rtox_requirements/dev_requirements.txt
commands =
pytest --cov={envsitepackagesdir}/datahub --cov={envsitepackagesdir}/datahub_provider \
py3-quick,py3-airflow1: -m 'not integration and not slow_integration' --junit-xml=junit.quick.xml \
py3-full: --cov-fail-under 65 --junit-xml=junit.full.xml \
py3-quick: -m 'not integration and not slow_integration' --junit-xml=junit.quick.xml \
py36-full: --cov-fail-under 68 --junit-xml=junit.full.xml \
--continue-on-collection-errors \
-vv \
py39-full: --cov-fail-under 70 --junit-xml=junit.full.xml \
--continue-on-collection-errors \
-vv
@ -34,7 +37,10 @@ setenv =
deps =
.[dev-airflow1]
[testenv:py3-full]
[testenv:py39-full]
deps =
.[dev]
.[integration-tests]
-rtox_requirements/py39-full_requirements.txt
[testenv:py36-full]
deps =
-rtox_requirements/py36-full_requirements.txt

View File

@ -0,0 +1,75 @@
#
# This file is autogenerated by pip-compile with python 3.9
# To update, run:
#
# pip-compile
#
avro==1.10.2
# via
# acryl-datahub (setup.py)
# avro-gen3
avro-gen3==0.7.1
# via acryl-datahub (setup.py)
certifi==2021.10.8
# via requests
charset-normalizer==2.0.11
# via requests
click==8.0.3
# via
# acryl-datahub (setup.py)
# click-default-group
click-default-group==1.2.2
# via acryl-datahub (setup.py)
docker==5.0.3
# via acryl-datahub (setup.py)
entrypoints==0.4
# via acryl-datahub (setup.py)
expandvars==0.7.0
# via acryl-datahub (setup.py)
idna==3.3
# via requests
mypy-extensions==0.4.3
# via
# acryl-datahub (setup.py)
# typing-inspect
progressbar2==4.0.0
# via acryl-datahub (setup.py)
pydantic==1.9.0
# via acryl-datahub (setup.py)
python-dateutil==2.8.2
# via acryl-datahub (setup.py)
python-utils==3.1.0
# via progressbar2
pytz==2021.3
# via avro-gen3
pytz-deprecation-shim==0.1.0.post0
# via tzlocal
pyyaml==6.0
# via acryl-datahub (setup.py)
requests==2.27.1
# via docker
six==1.16.0
# via
# avro-gen3
# python-dateutil
stackprinter==0.2.5
# via acryl-datahub (setup.py)
tabulate==0.8.9
# via acryl-datahub (setup.py)
toml==0.10.2
# via acryl-datahub (setup.py)
typing-extensions==3.10.0.2
# via
# acryl-datahub (setup.py)
# pydantic
# typing-inspect
typing-inspect==0.7.1
# via acryl-datahub (setup.py)
tzdata==2021.5
# via pytz-deprecation-shim
tzlocal==4.1
# via avro-gen3
urllib3==1.26.8
# via requests
websocket-client==1.2.3
# via docker

View File

@ -0,0 +1,317 @@
acryl-PyHive==0.6.11
aenum==3.1.8
aiohttp==3.8.1
aiosignal==1.2.0
alembic==1.7.6
altair==4.1.0
anyio==3.5.0
apache-airflow==2.2.3
apache-airflow-providers-ftp==2.0.1
apache-airflow-providers-http==2.0.2
apache-airflow-providers-imap==2.1.0
apache-airflow-providers-snowflake==2.4.0
apache-airflow-providers-sqlite==2.0.1
apispec==3.3.2
appnope==0.1.2
argcomplete==1.12.3
argon2-cffi==21.3.0
argon2-cffi-bindings==21.2.0
asn1crypto==1.4.0
async-generator==1.10
async-timeout==4.0.2
asynctest==0.13.0
attrs==20.3.0
avro==1.10.2
avro-gen3==0.7.1
Babel==2.9.1
backcall==0.2.0
backports.zoneinfo==0.2.1
bcrypt==3.2.0
black==22.1.0
bleach==4.1.0
blinker==1.4
boto3==1.20.48
boto3-stubs==1.20.48
botocore==1.23.48
botocore-stubs==1.23.48
build==0.7.0
cached-property==1.5.2
cachetools==4.2.4
cattrs==1.0.0
certifi==2021.10.8
cffi==1.15.0
charset-normalizer==2.0.11
click==8.0.3
click-default-group==1.2.2
clickclick==20.10.2
colorama==0.4.4
colorlog==5.0.1
commonmark==0.9.1
confluent-kafka==1.8.2
contextvars==2.4
coverage==6.2
croniter==1.0.15
cryptography==3.4.8
cx-Oracle==8.3.0
dataclasses==0.8
decorator==4.4.2
deepdiff==5.7.0
defusedxml==0.7.1
dill==0.3.4
distlib==0.3.4
distro==1.6.0
dnspython==2.2.0
docker==5.0.3
docker-compose==1.29.2
dockerpty==0.4.1
docopt==0.6.2
docutils==0.16
ecdsa==0.17.0
elasticsearch==7.17.0
email-validator==1.1.3
entrypoints==0.4
expandvars==0.7.0
fastavro==1.4.7
filelock==3.4.1
flake8==4.0.1
flake8-tidy-imports==4.5.0
Flask==1.1.2
Flask-AppBuilder==3.4.4
Flask-Babel==2.0.0
Flask-Caching==1.10.1
Flask-Cors==3.0.10
Flask-JWT-Extended==3.25.1
Flask-Login==0.4.1
Flask-OpenID==1.3.0
Flask-SQLAlchemy==2.5.1
Flask-WTF==0.14.3
flatdict==4.0.1
freezegun==1.1.0
frozenlist==1.2.0
future==0.18.2
GeoAlchemy2==0.10.2
google-api-core==2.5.0
google-auth==2.6.0
google-cloud-appengine-logging==1.1.0
google-cloud-audit-log==0.2.0
google-cloud-bigquery==2.32.0
google-cloud-core==2.2.2
google-cloud-logging==3.0.0
google-crc32c==1.3.0
google-resumable-media==2.1.0
googleapis-common-protos==1.54.0
graphviz==0.19.1
great-expectations==0.14.5
greenlet==1.1.2
grpc-google-iam-v1==0.12.3
grpcio==1.43.0
grpcio-status==1.43.0
gunicorn==20.1.0
h11==0.12.0
httpcore==0.13.7
httpx==0.19.0
idna==3.3
idna-ssl==1.1.0
immutables==0.16
importlib-metadata==4.2.0
importlib-resources==5.4.0
inflection==0.5.1
iniconfig==1.1.1
ipykernel==5.5.6
ipython==7.16.3
ipython-genutils==0.2.0
ipywidgets==7.6.5
iso8601==1.0.2
isodate==0.6.1
isort==5.10.1
itsdangerous==1.1.0
jedi==0.17.2
Jinja2==3.0.3
jmespath==0.10.0
JPype1==1.3.0
jsonpatch==1.32
jsonpickle==2.1.0
jsonpointer==2.2
jsonschema==3.2.0
jupyter-client==7.1.2
jupyter-core==4.9.1
jupyterlab-pygments==0.1.2
jupyterlab-widgets==1.0.2
keyring==23.4.1
lazy-object-proxy==1.7.1
lockfile==0.12.2
looker-sdk==21.6.0
Mako==1.1.6
Markdown==3.3.4
MarkupSafe==2.0.1
marshmallow==3.14.1
marshmallow-enum==1.5.1
marshmallow-oneofschema==3.0.1
marshmallow-sqlalchemy==0.26.1
mccabe==0.6.1
mistune==0.8.4
more-itertools==8.12.0
multidict==5.2.0
mypy==0.910
mypy-boto3-glue==1.20.47
mypy-boto3-s3==1.20.46.post1
mypy-boto3-sagemaker==1.20.46.post1
mypy-extensions==0.4.3
nbclient==0.5.9
nbconvert==6.0.7
nbformat==5.1.3
nest-asyncio==1.5.4
networkx==2.5.1
notebook==6.4.8
numpy==1.19.5
okta==1.7.0
openapi-schema-validator==0.1.6
openapi-spec-validator==0.3.3
ordered-set==4.0.2
oscrypto==1.2.1
packaging==21.3
pandas==1.1.5
pandocfilters==1.5.0
paramiko==2.9.2
parse==1.19.0
parso==0.7.1
pathspec==0.9.0
pendulum==2.1.2
pep517==0.12.0
pep562==1.1
pexpect==4.8.0
pickleshare==0.7.5
pkginfo==1.8.2
platformdirs==2.4.0
pluggy==1.0.0
prison==0.2.1
progressbar2==3.55.0
prometheus-client==0.13.1
prompt-toolkit==3.0.26
proto-plus==1.19.9
protobuf==3.19.4
psutil==5.9.0
psycopg2-binary==2.9.3
ptyprocess==0.7.0
pure-sasl==0.6.2
py==1.11.0
py4j==0.10.9
pyasn1==0.4.8
pyasn1-modules==0.2.8
pybigquery==0.10.2
pycodestyle==2.8.0
pycparser==2.21
pycryptodome==3.14.0
pycryptodomex==3.14.0
pydantic==1.9.0
pydash==5.1.0
pydeequ==1.0.1
pydot==1.4.2
pydruid==0.6.2
pyflakes==2.4.0
Pygments==2.11.2
PyJWT==1.7.1
pymongo==4.0.1
PyMySQL==1.0.2
PyNaCl==1.5.0
pyOpenSSL==20.0.1
pyparsing==2.4.7
pyrsistent==0.16.1
pyspark==3.0.3
pytest==6.2.5
pytest-cov==3.0.0
pytest-docker==0.10.3
python-daemon==2.3.0
python-dateutil==2.8.2
python-dotenv==0.19.2
python-jose==3.3.0
python-ldap==3.4.0
python-nvd3==0.15.0
python-slugify==4.0.1
python-tds==1.11.0
python-utils==3.1.0
python3-openid==3.2.0
pytz==2021.3
pytz-deprecation-shim==0.1.0.post0
pytzdata==2020.1
PyYAML==5.4.1
pyzmq==22.3.0
readme-renderer==32.0
redash-toolbelt==0.1.9
requests==2.27.1
requests-mock==1.9.3
requests-toolbelt==0.9.1
rfc3986==1.5.0
rich==11.1.0
rsa==4.8
ruamel.yaml==0.17.17
ruamel.yaml.clib==0.2.6
s3transfer==0.5.1
sasl3==0.2.11
scipy==1.5.4
Send2Trash==1.8.0
setproctitle==1.2.2
six==1.16.0
sniffio==1.2.0
snowflake-connector-python==2.7.1
snowflake-sqlalchemy==1.2.4
sql-metadata==2.3.0
SQLAlchemy==1.3.24
SQLAlchemy-JSONField==1.0.0
sqlalchemy-pytds==0.3.4
sqlalchemy-redshift==0.8.9
sqlalchemy-stubs==0.4
SQLAlchemy-Utils==0.38.2
sqllineage==1.3.3
sqlparse==0.4.2
stackprinter==0.2.5
swagger-ui-bundle==0.0.9
tabulate==0.8.9
tenacity==8.0.1
termcolor==1.1.0
terminado==0.12.1
testpath==0.5.0
text-unidecode==1.3
texttable==1.6.4
thrift==0.15.0
thrift-sasl==0.4.3
toml==0.10.2
tomli==1.2.3
toolz==0.11.2
tornado==6.1
tox==3.24.5
tqdm==4.62.3
traitlets==4.3.3
trino==0.309.0
twine==3.8.0
typed-ast==1.4.3
types-cachetools==4.2.9
types-click==0.1.12
types-dataclasses==0.6.4
types-freezegun==1.1.6
types-pkg-resources==0.1.3
types-PyMySQL==1.0.13
types-python-dateutil==2.8.9
types-PyYAML==6.0.4
types-requests==2.27.8
types-six==1.16.10
types-tabulate==0.8.5
types-toml==0.10.3
types-urllib3==1.26.9
typing==3.7.4.3
typing-extensions==3.10.0.2
typing-inspect==0.7.1
tzdata==2021.5
tzlocal==4.1
unicodecsv==0.14.1
urllib3==1.26.8
virtualenv==20.13.0
wcwidth==0.2.5
webencodings==0.5.1
websocket-client==0.59.0
Werkzeug==1.0.1
widgetsnbextension==3.5.2
WTForms==2.3.3
xmltodict==0.12.0
yarl==1.7.2
zipp==3.6.0

View File

@ -0,0 +1,312 @@
acryl-PyHive==0.6.11
aenum==3.1.8
aiohttp==3.8.1
aiosignal==1.2.0
alembic==1.7.5
altair==4.2.0
anyio==3.5.0
apache-airflow==2.2.3
apache-airflow-providers-ftp==2.0.1
apache-airflow-providers-http==2.0.2
apache-airflow-providers-imap==2.1.0
apache-airflow-providers-snowflake==2.4.0
apache-airflow-providers-sqlite==2.0.1
apispec==3.3.2
appnope==0.1.2
argcomplete==1.12.3
argon2-cffi==21.3.0
argon2-cffi-bindings==21.2.0
asn1crypto==1.4.0
asttokens==2.0.5
async-timeout==4.0.2
attrs==20.3.0
avro==1.10.2
avro-gen3==0.7.1
Babel==2.9.1
backcall==0.2.0
bcrypt==3.2.0
black==22.1.0
bleach==4.1.0
blinker==1.4
boto3==1.20.46
boto3-stubs==1.20.46.post1
botocore==1.23.46
botocore-stubs==1.23.46.post1
build==0.7.0
cachetools==5.0.0
cattrs==1.1.2
certifi==2021.10.8
cffi==1.15.0
charset-normalizer==2.0.11
click==8.0.3
click-default-group==1.2.2
clickclick==20.10.2
colorama==0.4.4
colorlog==5.0.1
commonmark==0.9.1
confluent-kafka==1.8.2
coverage==6.3
croniter==1.0.15
cryptography==3.4.8
cx-Oracle==8.3.0
debugpy==1.5.1
decorator==5.1.1
deepdiff==5.7.0
defusedxml==0.7.1
dill==0.3.4
distlib==0.3.4
distro==1.6.0
dnspython==2.2.0
docker==5.0.3
docker-compose==1.29.2
dockerpty==0.4.1
docopt==0.6.2
docutils==0.16
ecdsa==0.17.0
elasticsearch==7.16.3
email-validator==1.1.3
entrypoints==0.3
executing==0.8.2
expandvars==0.7.0
fastavro==1.4.9
filelock==3.4.2
flake8==4.0.1
flake8-tidy-imports==4.6.0
Flask==1.1.2
Flask-AppBuilder==3.4.4
Flask-Babel==2.0.0
Flask-Caching==1.10.1
Flask-Cors==3.0.10
Flask-JWT-Extended==3.25.1
Flask-Login==0.4.1
Flask-OpenID==1.3.0
Flask-SQLAlchemy==2.5.1
Flask-WTF==0.14.3
flatdict==4.0.1
freezegun==1.1.0
frozenlist==1.3.0
future==0.18.2
GeoAlchemy2==0.10.2
google-api-core==2.4.0
google-auth==2.6.0
google-cloud-appengine-logging==1.1.0
google-cloud-audit-log==0.2.0
google-cloud-bigquery==2.32.0
google-cloud-core==2.2.2
google-cloud-logging==2.7.0
google-crc32c==1.3.0
google-resumable-media==2.1.0
googleapis-common-protos==1.54.0
graphviz==0.19.1
great-expectations==0.14.4
greenlet==1.1.2
grpc-google-iam-v1==0.12.3
grpcio==1.43.0
grpcio-status==1.43.0
gunicorn==20.1.0
h11==0.12.0
httpcore==0.13.7
httpx==0.19.0
idna==3.3
importlib-metadata==4.10.1
inflection==0.5.1
iniconfig==1.1.1
ipykernel==6.7.0
ipython==8.0.1
ipython-genutils==0.2.0
ipywidgets==7.6.5
iso8601==1.0.2
isort==5.10.1
itsdangerous==1.1.0
jedi==0.18.1
Jinja2==3.0.3
jmespath==0.10.0
JPype1==1.3.0
jsonpatch==1.32
jsonpickle==2.1.0
jsonpointer==2.2
jsonschema==3.2.0
jupyter-client==7.1.2
jupyter-core==4.9.1
jupyterlab-pygments==0.1.2
jupyterlab-widgets==1.0.2
keyring==23.5.0
lazy-object-proxy==1.7.1
lkml==1.1.2
lockfile==0.12.2
looker-sdk==21.6.0
Mako==1.1.6
Markdown==3.3.6
MarkupSafe==2.0.1
marshmallow==3.14.1
marshmallow-enum==1.5.1
marshmallow-oneofschema==3.0.1
marshmallow-sqlalchemy==0.26.1
matplotlib-inline==0.1.3
mccabe==0.6.1
mistune==0.8.4
more-itertools==8.12.0
multidict==6.0.2
mypy==0.910
mypy-boto3-glue==1.20.46.post1
mypy-boto3-s3==1.20.46.post1
mypy-boto3-sagemaker==1.20.46.post1
mypy-extensions==0.4.3
nbclient==0.5.10
nbconvert==6.4.1
nbformat==5.1.3
nest-asyncio==1.5.4
networkx==2.6.3
notebook==6.4.8
numpy==1.22.1
okta==1.7.0
openapi-schema-validator==0.2.3
openapi-spec-validator==0.4.0
ordered-set==4.0.2
oscrypto==1.2.1
packaging==21.3
pandas==1.4.0
pandocfilters==1.5.0
paramiko==2.9.2
parse==1.19.0
parso==0.8.3
pathspec==0.9.0
pendulum==2.1.2
pep517==0.12.0
pexpect==4.8.0
pickleshare==0.7.5
pip-tools==6.4.0
pkginfo==1.8.2
platformdirs==2.4.1
pluggy==1.0.0
prison==0.2.1
progressbar2==4.0.0
prometheus-client==0.13.1
prompt-toolkit==3.0.26
proto-plus==1.19.9
protobuf==3.19.4
psutil==5.9.0
psycopg2-binary==2.9.3
ptyprocess==0.7.0
pure-eval==0.2.2
pure-sasl==0.6.2
py==1.11.0
py4j==0.10.9
pyasn1==0.4.8
pyasn1-modules==0.2.8
pybigquery==0.10.2
pycodestyle==2.8.0
pycparser==2.21
pycryptodome==3.14.0
pycryptodomex==3.14.0
pydantic==1.9.0
pydash==5.1.0
pydeequ==1.0.1
pydot==1.4.2
pydruid==0.6.2
pyflakes==2.4.0
Pygments==2.11.2
PyJWT==1.7.1
pymongo==4.0.1
PyMySQL==1.0.2
PyNaCl==1.5.0
pyOpenSSL==20.0.1
pyparsing==2.4.7
pyrsistent==0.18.1
pyspark==3.0.3
pytest==6.2.5
pytest-cov==3.0.0
pytest-docker==0.10.3
python-daemon==2.3.0
python-dateutil==2.8.2
python-dotenv==0.19.2
python-jose==3.3.0
python-ldap==3.4.0
python-nvd3==0.15.0
python-slugify==4.0.1
python-tds==1.11.0
python-utils==3.1.0
python3-openid==3.2.0
pytz==2021.3
pytz-deprecation-shim==0.1.0.post0
pytzdata==2020.1
PyYAML==5.4.1
pyzmq==22.3.0
readme-renderer==32.0
redash-toolbelt==0.1.9
requests==2.27.1
requests-mock==1.9.3
requests-toolbelt==0.9.1
rfc3986==1.5.0
rich==11.1.0
rsa==4.8
ruamel.yaml==0.17.17
ruamel.yaml.clib==0.2.6
s3transfer==0.5.0
sasl3==0.2.11
scipy==1.7.3
Send2Trash==1.8.0
setproctitle==1.2.2
six==1.16.0
sniffio==1.2.0
snowflake-connector-python==2.7.1
snowflake-sqlalchemy==1.2.4
sql-metadata==2.2.2
SQLAlchemy==1.3.24
SQLAlchemy-JSONField==1.0.0
sqlalchemy-pytds==0.3.4
sqlalchemy-redshift==0.8.9
sqlalchemy-stubs==0.4
SQLAlchemy-Utils==0.38.2
sqllineage==1.3.3
sqlparse==0.4.2
stack-data==0.1.4
stackprinter==0.2.5
swagger-ui-bundle==0.0.9
tabulate==0.8.9
tenacity==8.0.1
termcolor==1.1.0
terminado==0.13.1
testpath==0.5.0
text-unidecode==1.3
texttable==1.6.4
thrift==0.15.0
thrift-sasl==0.4.3
toml==0.10.2
tomli==2.0.0
toolz==0.11.2
tornado==6.1
tox==3.24.5
tqdm==4.62.3
traitlets==5.1.1
trino==0.309.0
twine==3.7.1
types-cachetools==4.2.9
types-click==0.1.12
types-dataclasses==0.6.4
types-freezegun==1.1.6
types-pkg-resources==0.1.3
types-PyMySQL==1.0.13
types-python-dateutil==2.8.9
types-PyYAML==6.0.4
types-requests==2.27.8
types-six==1.16.10
types-tabulate==0.8.5
types-toml==0.10.3
types-urllib3==1.26.8
typing-extensions==3.10.0.2
typing-inspect==0.7.1
tzdata==2021.5
tzlocal==4.1
unicodecsv==0.14.1
urllib3==1.26.8
virtualenv==20.13.0
wcwidth==0.2.5
webencodings==0.5.1
websocket-client==0.59.0
Werkzeug==1.0.1
widgetsnbextension==3.5.2
WTForms==2.3.3
xmltodict==0.12.0
yarl==1.7.2
zipp==3.7.0