chore(ingestion-base): bump base image packages (#11053)

docker/datahub-ingestion-base/base-requirements.txt

@@ -1,19 +1,20 @@
 # Generated requirements file. Run ./regenerate-base-requirements.sh to regenerate.
-acryl-datahub-classify==0.0.10
+acryl-datahub-classify==0.0.11
 acryl-PyHive==0.6.16
-acryl-sqlglot==24.0.1.dev7
+acryl-sqlglot==25.3.1.dev3
 aenum==3.1.15
-aiohttp==3.9.5
+aiohappyeyeballs==2.3.2
+aiohttp==3.10.0
 aiosignal==1.3.1
-alembic==1.13.1
+alembic==1.13.2
 altair==4.2.0
 anyio==4.4.0
-apache-airflow==2.9.2
+apache-airflow==2.9.3
 apache-airflow-providers-common-io==1.3.2
-apache-airflow-providers-common-sql==1.14.0
+apache-airflow-providers-common-sql==1.14.2
-apache-airflow-providers-fab==1.1.1
+apache-airflow-providers-fab==1.2.2
-apache-airflow-providers-ftp==3.9.1
+apache-airflow-providers-ftp==3.10.0
-apache-airflow-providers-http==4.11.1
+apache-airflow-providers-http==4.12.0
 apache-airflow-providers-imap==3.6.1
 apache-airflow-providers-smtp==1.7.1
 apache-airflow-providers-sqlite==3.8.1
@@ -26,25 +27,30 @@ asgiref==3.8.1
 asn1crypto==1.5.1
 asttokens==2.4.1
 async-timeout==4.0.3
-asynch==0.2.3
+asynch==0.2.4
 attrs==23.2.0
 avro==1.11.3
 avro-gen3==0.7.13
+azure-common==1.1.28
+azure-core==1.29.4
+azure-identity==1.14.1
+azure-storage-blob==12.21.0
+azure-storage-file-datalake==12.16.0
 Babel==2.15.0
 backoff==2.2.1
 beautifulsoup4==4.12.3
 bleach==6.1.0
 blinker==1.8.2
 blis==0.7.11
-boto3==1.34.129
+boto3==1.34.151
-botocore==1.34.129
+botocore==1.34.151
 bracex==2.4
 cached-property==1.5.2
 cachelib==0.9.0
-cachetools==5.3.3
+cachetools==5.4.0
 catalogue==2.0.10
 cattrs==23.2.3
-certifi==2024.6.2
+certifi==2024.7.4
 cffi==1.16.0
 chardet==5.2.0
 charset-normalizer==3.3.2
@@ -55,25 +61,26 @@ click-spinner==0.1.10
 clickclick==20.10.2
 clickhouse-driver==0.2.8
 clickhouse-sqlalchemy==0.2.4
+cloudpathlib==0.18.1
 cloudpickle==3.0.0
 colorama==0.4.6
 colorlog==4.8.0
 comm==0.2.2
 confection==0.1.5
 ConfigUpdater==3.2
-confluent-kafka==2.4.0
+confluent-kafka==2.5.0
-connexion==2.14.1
+connexion==2.14.2
 cron-descriptor==1.4.3
-croniter==2.0.5
+croniter==3.0.3
 cryptography==42.0.8
 cx_Oracle==8.3.0
 cymem==2.0.8
 databricks-dbapi==0.6.0
-databricks-sdk==0.28.0
+databricks-sdk==0.29.0
 databricks-sql-connector==2.9.6
 dataflows-tabulator==1.54.3
 db-dtypes==1.2.0
-debugpy==1.8.1
+debugpy==1.8.2
 decorator==5.1.1
 defusedxml==0.7.1
 deltalake==0.17.4
@@ -84,42 +91,42 @@ docker==7.1.0
 docutils==0.21.2
 ecdsa==0.19.0
 elasticsearch==7.13.4
-email_validator==2.1.2
+email_validator==2.2.0
 entrypoints==0.4
 et-xmlfile==1.1.0
-exceptiongroup==1.2.1
+exceptiongroup==1.2.2
 executing==2.0.1
 expandvars==0.12.0
-fastavro==1.9.4
+fastavro==1.9.5
 fastjsonschema==2.20.0
-filelock==3.15.1
+filelock==3.15.4
 Flask==2.2.5
 flatdict==4.0.1
 frozenlist==1.4.1
 fsspec==2023.12.2
 future==1.0.0
-GeoAlchemy2==0.15.1
+GeoAlchemy2==0.15.2
 gitdb==4.0.11
 GitPython==3.1.43
-google-api-core==2.19.0
+google-api-core==2.19.1
-google-auth==2.30.0
+google-auth==2.32.0
-google-cloud-appengine-logging==1.4.3
+google-cloud-appengine-logging==1.4.5
 google-cloud-audit-log==0.2.5
-google-cloud-bigquery==3.24.0
+google-cloud-bigquery==3.25.0
 google-cloud-core==2.4.1
-google-cloud-datacatalog==3.19.0
+google-cloud-datacatalog==3.20.0
 google-cloud-datacatalog-lineage==0.2.2
 google-cloud-logging==3.5.0
 google-crc32c==1.5.0
-google-re2==1.1.20240601
+google-re2==1.1.20240702
 google-resumable-media==2.7.1
-googleapis-common-protos==1.63.1
+googleapis-common-protos==1.63.2
 gql==3.5.0
 graphql-core==3.2.3
 great-expectations==0.15.50
 greenlet==3.0.3
-grpc-google-iam-v1==0.13.0
+grpc-google-iam-v1==0.13.1
-grpcio==1.64.1
+grpcio==1.65.2
 grpcio-status==1.62.2
 grpcio-tools==1.62.2
 gssapi==1.8.3
@@ -130,7 +137,7 @@ httpx==0.27.0
 humanfriendly==10.0
 idna==3.7
 ijson==3.3.0
-importlib_metadata==7.1.0
+importlib_metadata==7.2.1
 importlib_resources==6.4.0
 inflection==0.5.1
 ipaddress==1.0.23
@@ -148,7 +155,7 @@ jsonlines==4.0.0
 jsonpatch==1.33
 jsonpointer==3.0.0
 jsonref==1.1.0
-jsonschema==4.22.0
+jsonschema==4.23.0
 jsonschema-specifications==2023.12.1
 jupyter-server==1.16.0
 jupyter_client==7.4.9
@@ -159,16 +166,16 @@ langcodes==3.4.0
 language_data==1.2.0
 lark==1.1.4
 lazy-object-proxy==1.10.0
-leb128==1.0.7
+leb128==1.0.8
-limits==3.12.0
+limits==3.13.0
 linear-tsv==1.1.0
 linkify-it-py==2.0.3
-lkml==1.3.4
+lkml==1.3.5
 lockfile==0.12.2
 looker-sdk==23.0.0
 lxml==5.2.2
 lz4==4.3.3
-makefun==1.15.2
+makefun==1.15.4
 Mako==1.3.5
 marisa-trie==1.2.0
 markdown-it-py==3.0.0
@@ -182,11 +189,12 @@ mdurl==0.1.2
 methodtools==0.4.7
 mistune==3.0.2
 mixpanel==4.10.1
-mlflow-skinny==2.14.0
+mlflow-skinny==2.15.0
 mmhash3==3.0.1
 more-itertools==10.3.0
 moto==4.2.14
 msal==1.22.0
+msal-extensions==1.1.0
 multidict==6.0.5
 murmurhash==1.0.10
 mypy-extensions==1.0.0
@@ -201,47 +209,46 @@ notebook_shim==0.2.4
 numpy==1.26.4
 oauthlib==3.2.2
 okta==1.7.0
-openlineage-airflow==1.12.0
+openlineage-airflow==1.18.0
-openlineage-integration-common==1.12.0
+openlineage-integration-common==1.18.0
-openlineage-python==1.12.0
+openlineage-python==1.18.0
-openlineage_sql==1.12.0
+openlineage_sql==1.18.0
-openpyxl==3.1.4
+openpyxl==3.1.5
-opentelemetry-api==1.25.0
+opentelemetry-api==1.26.0
-opentelemetry-exporter-otlp==1.25.0
+opentelemetry-exporter-otlp==1.26.0
-opentelemetry-exporter-otlp-proto-common==1.25.0
+opentelemetry-exporter-otlp-proto-common==1.26.0
-opentelemetry-exporter-otlp-proto-grpc==1.25.0
+opentelemetry-exporter-otlp-proto-grpc==1.26.0
-opentelemetry-exporter-otlp-proto-http==1.25.0
+opentelemetry-exporter-otlp-proto-http==1.26.0
-opentelemetry-proto==1.25.0
+opentelemetry-proto==1.26.0
-opentelemetry-sdk==1.25.0
+opentelemetry-sdk==1.26.0
-opentelemetry-semantic-conventions==0.46b0
+opentelemetry-semantic-conventions==0.47b0
 ordered-set==4.1.0
 packaging==24.1
 pandas==2.1.4
 pandocfilters==1.5.1
 parse==1.20.2
 parso==0.8.4
-pathlib_abc==0.1.1
 pathspec==0.12.1
-pathy==0.11.0
 pendulum==3.0.0
 pexpect==4.9.0
 phonenumbers==8.13.0
 platformdirs==4.2.2
 pluggy==1.5.0
+portalocker==2.10.1
 preshed==3.0.9
 prison==0.2.1
 progressbar2==4.4.2
 prometheus_client==0.20.0
 prompt_toolkit==3.0.47
-proto-plus==1.23.0
+proto-plus==1.24.0
-protobuf==4.25.3
+protobuf==4.25.4
 psutil==6.0.0
 psycopg2-binary==2.9.9
 ptyprocess==0.7.0
-pure-eval==0.2.2
 pure-sasl==0.6.2
+pure_eval==0.2.3
 py-partiql-parser==0.5.0
-pyarrow==16.1.0
+pyarrow==17.0.0
 pyarrow-hotfix==0.6
 pyasn1==0.6.0
 pyasn1_modules==0.4.0
@@ -249,16 +256,16 @@ pyathena==2.25.2
 pycountry==24.6.1
 pycparser==2.22
 pycryptodome==3.20.0
-pydantic==1.10.16
+pydantic==1.10.17
-pydash==8.0.1
+pydash==8.0.3
 pydruid==0.6.9
 Pygments==2.18.0
 pyiceberg==0.4.0
-pymongo==4.7.3
+pymongo==4.8.0
 PyMySQL==1.1.1
-pyOpenSSL==24.1.0
+pyOpenSSL==24.2.1
 pyparsing==3.0.9
-pyspnego==0.11.0
+pyspnego==0.11.1
 python-daemon==3.0.1
 python-dateutil==2.9.0.post0
 python-jose==3.3.0
@@ -273,9 +280,9 @@ pytz==2024.1
 PyYAML==6.0.1
 pyzmq==26.0.3
 redash-toolbelt==0.1.9
-redshift-connector==2.1.1
+redshift-connector==2.1.2
 referencing==0.35.1
-regex==2024.5.15
+regex==2024.7.24
 requests==2.32.3
 requests-file==2.1.0
 requests-gssapi==1.3.0
@@ -286,31 +293,32 @@ rfc3339-validator==0.1.4
 rfc3986==2.0.0
 rich==13.7.1
 rich-argparse==1.5.2
-rpds-py==0.18.1
+rpds-py==0.19.1
 rsa==4.9
 rstr==3.2.2
 ruamel.yaml==0.17.17
-s3transfer==0.10.1
+s3transfer==0.10.2
 schwifty==2024.6.1
-scipy==1.13.1
+scipy==1.14.0
 scramp==1.4.5
 Send2Trash==1.8.3
-sentry-sdk==2.5.1
+sentry-sdk==2.12.0
 setproctitle==1.3.3
+shellingham==1.5.4
 simple-salesforce==1.12.6
 six==1.16.0
 slack-sdk==3.18.1
-smart-open==6.4.0
+smart-open==7.0.4
 smmap==5.0.1
 sniffio==1.3.1
-snowflake-connector-python==3.11.0
+snowflake-connector-python==3.12.0
-snowflake-sqlalchemy==1.5.3
+snowflake-sqlalchemy==1.6.1
 sortedcontainers==2.4.0
 soupsieve==2.5
-spacy==3.5.0
+spacy==3.7.5
 spacy-legacy==3.0.12
 spacy-loggers==1.0.5
-sql-metadata==2.2.2
+sql_metadata==2.12.0
 SQLAlchemy==1.4.44
 sqlalchemy-bigquery==1.11.0
 sqlalchemy-cockroachdb==1.4.4
@@ -318,7 +326,7 @@ SQLAlchemy-JSONField==1.0.2
 sqlalchemy-pytds==0.3.5
 sqlalchemy-redshift==0.8.14
 SQLAlchemy-Utils==0.41.2
-sqlglotrs==0.2.5
+sqlglotrs==0.2.7
 sqllineage==1.3.8
 sqlparse==0.4.4
 srsly==2.4.8
@@ -327,25 +335,25 @@ strictyaml==1.7.3
 tableauserverclient==0.25
 tableschema==1.20.11
 tabulate==0.9.0
-tenacity==8.4.1
+tenacity==9.0.0
-teradatasql==20.0.0.12
+teradatasql==20.0.0.14
 teradatasqlalchemy==20.0.0.1
 termcolor==2.4.0
 terminado==0.18.1
 text-unidecode==1.3
-thinc==8.1.12
+thinc==8.2.5
 thrift==0.16.0
 thrift-sasl==0.4.3
-time-machine==2.14.1
+time-machine==2.14.2
 tinycss2==1.3.0
 toml==0.10.2
-tomlkit==0.12.5
+tomlkit==0.13.0
 toolz==0.12.1
 tornado==6.4.1
 tqdm==4.66.4
 traitlets==5.2.1.post0
-trino==0.328.0
+trino==0.329.0
-typer==0.7.0
+typer==0.12.3
 typing-inspect==0.9.0
 typing_extensions==4.12.2
 tzdata==2024.1
@@ -355,15 +363,16 @@ ujson==5.10.0
 unicodecsv==0.14.1
 universal_pathlib==0.2.2
 urllib3==1.26.19
-vertica-python==1.3.8
+vertica-python==1.4.0
 vertica-sqlalchemy-dialect==0.0.8.2
 vininfo==1.8.0
 wasabi==1.1.3
 wcmatch==8.5.2
 wcwidth==0.2.13
+weasel==0.4.1
 webencodings==0.5.1
 websocket-client==1.8.0
-Werkzeug==2.3.8
+Werkzeug==2.2.3
 widgetsnbextension==4.0.11
 wirerope==0.4.7
 wrapt==1.16.0

docker/datahub-ingestion-base/build.gradle

@@ -12,7 +12,7 @@ ext {
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
     docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
-    revision = 3 // increment to trigger rebuild
+    revision = 4 // increment to trigger rebuild
 }
 
 docker {

docker/datahub-ingestion/build.gradle

@@ -12,7 +12,7 @@ ext {
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
     docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
-    revision = 3 // increment to trigger rebuild
+    revision = 4 // increment to trigger rebuild
 }
 
 dependencies {

docker/docker-compose-with-cassandra.yml

@@ -144,7 +144,7 @@ services:
       - neo4jdata:/data
   schema-registry:
     hostname: schema-registry
-    image: confluentinc/cp-schema-registry:7.4.0
+    image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
       - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
     env_file: schema-registry/env/docker.env
@@ -159,7 +159,7 @@ services:
         condition: service_healthy
   broker:
     hostname: broker
-    image: confluentinc/cp-kafka:7.4.0
+    image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
       - 29092:29092
       - 9092:9092
@@ -177,7 +177,7 @@ services:
       - broker:/var/lib/kafka/data/
   zookeeper:
     hostname: zookeeper
-    image: confluentinc/cp-zookeeper:7.4.0
+    image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
       - 2181:2181
     env_file: zookeeper/env/docker.env

docker/docker-compose-without-neo4j.yml

@@ -123,7 +123,7 @@ services:
     - esdata:/usr/share/elasticsearch/data
   schema-registry:
     hostname: schema-registry
-    image: confluentinc/cp-schema-registry:7.4.0
+    image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
     env_file: schema-registry/env/docker.env
@@ -138,7 +138,7 @@ services:
         condition: service_healthy
   broker:
     hostname: broker
-    image: confluentinc/cp-kafka:7.4.0
+    image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
     env_file: broker/env/docker.env
@@ -155,7 +155,7 @@ services:
     - broker:/var/lib/kafka/data/
   zookeeper:
     hostname: zookeeper
-    image: confluentinc/cp-zookeeper:7.4.0
+    image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     env_file: zookeeper/env/docker.env

docker/docker-compose.yml

@@ -143,7 +143,7 @@ services:
     - neo4jdata:/data
   schema-registry:
     hostname: schema-registry
-    image: confluentinc/cp-schema-registry:7.4.0
+    image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
     env_file: schema-registry/env/docker.env
@@ -158,7 +158,7 @@ services:
         condition: service_healthy
   broker:
     hostname: broker
-    image: confluentinc/cp-kafka:7.4.0
+    image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
     env_file: broker/env/docker.env
@@ -175,7 +175,7 @@ services:
     - broker:/var/lib/kafka/data/
   zookeeper:
     hostname: zookeeper
-    image: confluentinc/cp-zookeeper:7.4.0
+    image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     env_file: zookeeper/env/docker.env

docker/profiles/docker-compose.prerequisites.yml

@@ -210,7 +210,7 @@ services:
       - neo4jdata:/data
   kafka-broker:
     hostname: broker
-    image: confluentinc/cp-kafka:7.4.0
+    image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     command:
       - /bin/bash
       - -c