From a087a9fbb93d0a8bc81d825c6ad6f5bbbee860b1 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Thu, 8 Dec 2022 09:59:19 +0530 Subject: [PATCH] chore(ci): update base ingestion image requirements file (#6687) --- .../datahub-ingestion/base-requirements.txt | 402 ++++++++++++------ 1 file changed, 264 insertions(+), 138 deletions(-) diff --git a/docker/datahub-ingestion/base-requirements.txt b/docker/datahub-ingestion/base-requirements.txt index b752978699..7deb14483c 100644 --- a/docker/datahub-ingestion/base-requirements.txt +++ b/docker/datahub-ingestion/base-requirements.txt @@ -1,82 +1,113 @@ acryl-datahub -absl-py==1.1.0 +acryl-datahub-classify==0.0.4 acryl-iceberg-legacy==0.0.4 acryl-PyHive==0.6.13 aenum==3.1.11 -aiohttp==3.8.1 +aiohttp==3.8.3 aiosignal==1.2.0 -alembic==1.8.0 +alembic==1.8.1 altair==4.2.0 anyio==3.6.1 apache-airflow==2.2.5 -apache-airflow-providers-ftp==3.0.0 -apache-airflow-providers-http==3.0.0 +apache-airflow-providers-common-sql==1.2.0 +apache-airflow-providers-ftp==3.1.0 +apache-airflow-providers-http==4.0.0 apache-airflow-providers-imap==3.0.0 -apache-airflow-providers-sqlite==3.0.0 +apache-airflow-providers-snowflake==3.3.0 +apache-airflow-providers-sqlite==3.2.1 apispec==3.3.2 +appdirs==1.4.4 +appnope==0.1.3 argcomplete==2.0.0 argon2-cffi==21.3.0 argon2-cffi-bindings==21.2.0 -asgiref==3.5.2 asn1crypto==1.5.1 -asttokens==2.0.5 +asttokens==2.0.8 async-timeout==4.0.2 attrs==20.3.0 +Authlib==1.1.0 avro==1.10.2 -avro-gen3==0.7.4 -azure-core==1.24.1 +avro-gen3==0.7.7 +azure-core==1.25.1 azure-identity==1.10.0 -azure-storage-blob==12.12.0 -azure-storage-file-datalake==12.7.0 +azure-storage-blob==12.13.1 +azure-storage-file-datalake==12.8.0 Babel==2.10.3 backcall==0.2.0 -backports.zoneinfo==0.2.1; python_version < "3.9" +backoff==2.2.1 +bcrypt==4.0.0 beautifulsoup4==4.11.1 -bleach==5.0.0 -blinker==1.4 -boto3==1.24.14 -botocore==1.27.14 +black==22.8.0 +bleach==5.0.1 +blinker==1.5 +blis==0.7.9 +boto3==1.24.87 +boto3-stubs==1.24.87 +botocore==1.27.87 +botocore-stubs==1.27.87 +bowler==0.9.0 bracex==2.3.post1 +build==0.8.0 cached-property==1.5.2 -cachelib==0.8.0 +cachelib==0.9.0 cachetools==5.2.0 +catalogue==2.0.8 cattrs==1.10.0 -certifi==2022.6.15 -cffi==1.15.0 -chardet==4.0.0 -charset-normalizer==2.0.12 +certifi==2022.9.24 +cffi==1.15.1 +chardet==5.0.0 +charset-normalizer==2.1.1 click==8.1.3 click-default-group==1.2.2 +click-spinner==0.1.10 clickclick==20.10.2 clickhouse-driver==0.2.4 clickhouse-sqlalchemy==0.1.8 +cloudpickle==2.2.0 colorama==0.4.5 -colorlog==6.6.0 +colorlog==6.7.0 commonmark==0.9.1 -confluent-kafka==1.8.2 -connexion==2.13.1 -croniter==1.3.5 +confection==0.0.3 +confluent-kafka==1.9.2 +connexion==2.14.1 +coverage==6.5.0 +croniter==1.3.7 cryptography==36.0.2 cx-Oracle==8.3.0 -debugpy==1.6.0 +cymem==2.0.7 +dask==2022.1.1 +databricks-cli==0.17.3 +databricks-dbapi==0.6.0 +debugpy==1.6.3 decorator==5.1.1 +deepdiff==5.8.1 defusedxml==0.7.1 +deltalake==0.6.3 Deprecated==1.2.13 dill==0.3.5.1 +distlib==0.3.6 +distro==1.7.0 dnspython==2.2.1 -docker==5.0.3 +docker==6.0.0 +docker-compose==1.29.2 +dockerpty==0.4.1 +docopt==0.6.2 docutils==0.16 -ecdsa==0.17.0 +ecdsa==0.18.0 elasticsearch==7.13.4 -email-validator==1.2.1 +email-validator==1.3.0 entrypoints==0.4 et-xmlfile==1.1.0 -executing==0.8.3 +executing==1.1.0 expandvars==0.9.0 -fastapi==0.78.0 -fastavro==1.5.1 -fastjsonschema==2.15.3 -feast==0.18.0 +fastapi==0.85.0 +fastavro==1.6.1 +fastjsonschema==2.16.2 +feast==0.26.0 +filelock==3.8.0 +fissix==21.11.13 +flake8==5.0.4 +flake8-tidy-imports==4.8.0 Flask==1.1.2 Flask-AppBuilder==3.4.5 Flask-Babel==2.0.0 @@ -89,226 +120,321 @@ Flask-Session==0.4.0 Flask-SQLAlchemy==2.5.1 Flask-WTF==0.14.3 flatdict==4.0.1 -frozenlist==1.3.0 +freezegun==1.2.2 +frozenlist==1.3.1 +fsspec==2022.11.0 future==0.18.2 GeoAlchemy2==0.11.1 +gitdb==4.0.9 +GitPython==3.1.27 google-api-core==2.8.0 -google-auth==2.8.0 -google-cloud-appengine-logging==1.1.2 +google-api-python-client==2.64.0 +google-auth==2.12.0 +google-auth-httplib2==0.1.0 +google-cloud-appengine-logging==1.1.3 google-cloud-audit-log==0.2.0 -google-cloud-bigquery==3.2.0 -google-cloud-bigquery-storage==2.13.2 -google-cloud-core==2.3.1 +google-cloud-bigquery==2.34.4 +google-cloud-bigquery-storage==2.14.1 +google-cloud-core==2.3.2 google-cloud-logging==3.1.1 -google-crc32c==1.3.0 -google-resumable-media==2.3.3 +google-crc32c==1.5.0 +google-resumable-media==2.4.0 googleapis-common-protos==1.52.0 -graphviz==0.20 -great-expectations==0.15.2 -greenlet==1.1.2 +gql==3.4.0 +graphql-core==3.2.3 +graphviz==0.20.1 +great-expectations==0.15.22 +greenlet==1.1.3 grpc-google-iam-v1==0.12.3 -grpcio==1.44.0 -grpcio-reflection==1.44.0 +grpcio==1.50.0 +grpcio-reflection==1.48.2 grpcio-status==1.44.0 grpcio-tools==1.44.0 gunicorn==20.1.0 h11==0.12.0 hmsclient==0.1.1 httpcore==0.15.0 -httptools==0.4.0 +httplib2==0.20.4 +httptools==0.5.0 httpx==0.23.0 humanfriendly==10.0 -idna==3.3 +idna==3.4 ijson==3.1.4 -importlib-metadata==4.11.4 -importlib-resources==5.8.0 +importlib-metadata==5.0.0 inflection==0.5.1 -ipykernel==6.15.0 -ipython==8.4.0 +iniconfig==1.1.1 +ipaddress==1.0.23 +ipykernel==6.16.0 +ipython==8.5.0 ipython-genutils==0.2.0 -iso8601==1.0.2 +ipywidgets==8.0.2 +iso3166==2.1.1 +iso8601==1.1.0 isodate==0.6.1 +isort==5.10.1 itsdangerous==1.1.0 +jaraco.classes==3.2.3 jedi==0.18.1 Jinja2==3.0.3 jmespath==1.0.1 -JPype1==1.4.0 -jsonlines==3.0.0 +JPype1==1.4.1 +jsonlines==3.1.0 jsonpatch==1.32 +jsonpickle==2.2.0 jsonpointer==2.3 jsonschema==3.2.0 -jupyter-client==7.3.4 -jupyter-core==4.10.0 +jupyter-core==4.11.1 +jupyter_client==7.3.5 jupyterlab-pygments==0.2.2 +jupyterlab-widgets==3.0.3 +keyring==23.9.3 +langcodes==3.3.0 lazy-object-proxy==1.7.1 linear-tsv==1.1.0 -lkml==1.2.0 +lkml==1.3.0b5 +locket==1.0.0 lockfile==0.12.2 looker-sdk==22.2.1 -Mako==1.2.0 -Markdown==3.3.7 +lxml==4.9.1 +makefun==1.15.0 +Mako==1.2.3 +Markdown==3.4.1 MarkupSafe==2.0.1 -marshmallow==3.16.0 +marshmallow==3.18.0 marshmallow-enum==1.5.1 marshmallow-oneofschema==3.0.1 marshmallow-sqlalchemy==0.26.1 -matplotlib-inline==0.1.3 -mistune==0.8.4 -mixpanel==4.9.0 +matplotlib-inline==0.1.6 +mccabe==0.7.0 +mistune==2.0.4 +mixpanel==4.10.0 mmh3==3.0.0 -more-itertools==8.13.0 -moto==3.1.14 +more-itertools==8.14.0 +moreorless==0.4.0 +moto==4.0.6 msal==1.16.0 msal-extensions==1.0.0 msrest==0.7.1 multidict==6.0.2 +murmurhash==1.0.9 +mypy==0.991 +mypy-boto3-glue==1.24.87 +mypy-boto3-s3==1.24.76 +mypy-boto3-sagemaker==1.24.84 +mypy-boto3-sts==1.24.36.post1 mypy-extensions==0.4.3 +mysql-connector-python==8.0.30 nbclient==0.6.3 -nbconvert==6.5.1 -nbformat==5.4.0 -nest-asyncio==1.5.5 -networkx==2.8.4 +nbconvert==7.1.0 +nbformat==5.6.1 +nest-asyncio==1.5.6 +networkx==2.8.7 notebook==6.4.12 -numpy==1.22.4 -oauthlib==3.2.0 +ntlm-auth==1.5.0 +numpy==1.23.3 +oauthlib==3.2.1 okta==1.7.0 openpyxl==3.0.10 -orderedset==2.0.3 +ordered-set==4.1.0 oscrypto==1.3.0 packaging==21.3 -pandas==1.4.2 +pandas==1.5.0 pandavro==1.5.2 pandocfilters==1.5.0 +paramiko==2.11.0 parse==1.19.0 parso==0.8.3 +partd==1.3.0 +pathspec==0.10.1 +pathy==0.10.0 pendulum==2.1.2 +pep517==0.13.0 pexpect==4.8.0 +phonenumbers==8.13.0 pickleshare==0.7.5 -portalocker==2.4.0 +pkginfo==1.8.3 +platformdirs==2.5.2 +pluggy==1.0.0 +portalocker==2.5.1 +preshed==3.0.8 prison==0.2.1 progressbar2==4.0.0 prometheus-client==0.14.1 -prompt-toolkit==3.0.29 -proto-plus==1.19.6 +prompt-toolkit==3.0.31 +proto-plus==1.22.1 protobuf==3.20.1 -psutil==5.9.1 +psutil==5.9.2 psycopg2-binary==2.9.3 ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 +py==1.11.0 py4j==0.10.9 -pyarrow==6.0.1 +pyarrow==8.0.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 pyathena==2.4.1 +pycodestyle==2.9.1 +pycountry==22.3.5 pycparser==2.21 -pycryptodome==3.14.1 -pycryptodomex==3.14.1 -pydantic==1.9.1 -pydash==5.1.0 +pycryptodome==3.15.0 +pycryptodomex==3.15.0 +pydantic==1.9.2 +pydash==5.1.1 pydeequ==1.0.1 -pydruid==0.6.3 -Pygments==2.12.0 +pydruid==0.6.4 +pyflakes==2.5.0 +Pygments==2.13.0 PyJWT==1.7.1 -pymongo==4.1.1 +pymongo==4.3.3 PyMySQL==1.0.2 +PyNaCl==1.5.0 pyOpenSSL==22.0.0 -pyparsing==2.4.7 +pyparsing==3.0.9 pyrsistent==0.18.1 pyspark==3.0.3 -python-daemon==2.3.0 +pytest==7.1.3 +pytest-asyncio==0.19.0 +pytest-cov==4.0.0 +pytest-docker==1.0.1 +python-daemon==2.3.1 python-dateutil==2.8.2 -python-dotenv==0.20.0 +python-dotenv==0.21.0 python-jose==3.3.0 -python-ldap==3.4.0 +python-ldap==3.4.3 python-nvd3==0.15.0 python-slugify==4.0.1 +python-stdnum==1.18 python-tds==1.11.0 python-utils==3.3.3 python3-openid==3.2.0 -pytz==2022.1 +pytz==2022.4 pytz-deprecation-shim==0.1.0.post0 pytzdata==2020.1 -PyYAML==6.0 -pyzmq==23.2.0 +PyYAML==5.4.1 +pyzmq==24.0.1 ratelimiter==1.2.0.post0 +readme-renderer==37.2 redash-toolbelt==0.1.9 -requests==2.28.0 +requests==2.28.1 +requests-file==1.5.1 +requests-mock==1.10.0 +requests-ntlm==1.1.0 requests-oauthlib==1.3.1 +requests-toolbelt==0.9.1 responses==0.21.0 retrying==1.3.3 rfc3986==1.5.0 -rich==12.4.4 -rsa==4.8 +rich==12.6.0 +rsa==4.9 ruamel.yaml==0.17.17 -ruamel.yaml.clib==0.2.6 s3transfer==0.6.0 sasl3==0.2.11 -scipy==1.8.1 +schwifty==2022.9.0 +scipy==1.9.1 Send2Trash==1.8.0 -setproctitle==1.2.3 +setproctitle==1.3.2 +simple-salesforce==1.12.2 six==1.16.0 -smart-open==6.0.0 -sniffio==1.2.0 -snowflake-connector-python==2.7.8 +smart-open==5.2.1 +smmap==5.0.0 +sniffio==1.3.0 +snowflake-connector-python==2.8.0 snowflake-sqlalchemy==1.2.4 soupsieve==2.3.2.post1 +spacy==3.4.3 +spacy-legacy==3.0.10 +spacy-loggers==1.0.3 sql-metadata==2.2.2 SQLAlchemy==1.3.24 -sqlalchemy-bigquery==1.4.4 +sqlalchemy-bigquery==1.4.1 SQLAlchemy-JSONField==1.0.0 sqlalchemy-pytds==0.3.4 -sqlalchemy-redshift==0.8.9 -SQLAlchemy-Utils==0.38.2 +sqlalchemy-redshift==0.8.11 +sqlalchemy-stubs==0.4 +SQLAlchemy-Utils==0.38.3 sqlalchemy-vertica==0.0.5 -sqllineage==1.3.5 -sqlparse==0.4.2 -stack-data==0.3.0 -stackprinter==0.2.6 -starlette==0.19.1 +sqllineage==1.3.6 +sqlparse==0.4.3 +srsly==2.4.5 +stack-data==0.5.1 +stackprinter==0.2.9 +starlette==0.20.4 swagger-ui-bundle==0.0.9 -tableauserverclient==0.19.0 +tableauserverclient==0.23 tableschema==1.20.2 tabulate==0.8.10 tabulator==1.53.5 -tenacity==8.0.1 -tensorflow-metadata==1.9.0 -termcolor==1.1.0 -terminado==0.15.0 +tenacity==8.1.0 +tensorflow-metadata==1.10.0 +termcolor==2.0.1 +terminado==0.16.0 text-unidecode==1.3 +texttable==1.6.4 +thinc==8.1.5 thrift==0.16.0 thrift-sasl==0.4.3 tinycss2==1.1.1 toml==0.10.2 -toolz==0.11.2 -tornado==6.1 -tqdm==4.64.0 +tomli==2.0.1 +toolz==0.12.0 +tornado==6.2 +tqdm==4.64.1 traitlets==5.2.1.post0 -trino==0.313.0 -types-Deprecated==1.2.8 -types-protobuf==3.19.22 -types-termcolor==1.1.4 -types-ujson==5.3.0 -typing-inspect==0.7.1 -typing_extensions==4.2.0 -tzdata==2022.1 +trino==0.319.0 +twine==4.0.1 +typeguard==2.13.3 +typer==0.7.0 +types-awscrt==0.14.7 +types-cachetools==5.2.1 +types-click==0.1.12 +types-click-spinner==0.1.13.1 +types-cryptography==3.3.23 +types-dataclasses==0.6.6 +types-Deprecated==1.2.9 +types-freezegun==1.1.10 +types-pkg-resources==0.1.3 +types-protobuf==4.21.0.1 +types-PyMySQL==1.0.19 +types-pyOpenSSL==22.1.0.0 +types-python-dateutil==2.8.19 +types-pytz==2022.4.0.0 +types-PyYAML==6.0.12 +types-requests==2.28.11.1 +types-s3transfer==0.6.0.post4 +types-six==1.16.21 +types-tabulate==0.8.11 +types-termcolor==1.1.6 +types-toml==0.10.8 +types-ujson==5.5.0 +types-urllib3==1.26.25 +typing-inspect==0.8.0 +typing_extensions==4.3.0 +tzdata==2022.4 tzlocal==4.2 -ujson==5.3.0 +ujson==5.5.0 unicodecsv==0.14.1 -urllib3==1.26.9 -uvicorn==0.17.6 -uvloop==0.16.0 -vertica-python==1.1.0 -watchgod==0.8.2 -wcmatch==8.4 +uritemplate==4.1.1 +urllib3==1.26.12 +uvicorn==0.18.3 +uvloop==0.17.0 +vertica-python==1.1.1 +vininfo==1.7.0 +virtualenv==20.17.0 +volatile==2.1.0 +wasabi==0.10.1 +watchfiles==0.17.0 +wcmatch==8.4.1 wcwidth==0.2.5 webencodings==0.5.1 -websocket-client==1.3.3 +websocket-client==0.59.0 websockets==10.3 Werkzeug==1.0.1 +widgetsnbextension==4.0.3 wrapt==1.14.1 WTForms==2.3.3 xlrd==2.0.1 xmltodict==0.13.0 -yarl==1.7.2 -zipp==3.8.0 \ No newline at end of file +yarl==1.8.1 +zeep==4.1.0 +zipp==3.8.1 \ No newline at end of file