diff --git a/.github/workflows/docker-ingestion-base.yml b/.github/workflows/docker-ingestion-base.yml index 640398b6b3..c0d4d94914 100644 --- a/.github/workflows/docker-ingestion-base.yml +++ b/.github/workflows/docker-ingestion-base.yml @@ -6,12 +6,14 @@ on: branches: - master paths: + - ".github/workflows/docker-ingestion-base.yml" - "docker/datahub-ingestion/**" - "gradle*" pull_request: branches: - master paths: + - ".github/workflows/docker-ingestion-base.yml" - "docker/datahub-ingestion/**" - "gradle*" workflow_dispatch: @@ -29,23 +31,18 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 + - name: Build and Push image + uses: ./.github/actions/docker-custom-build-and-push with: + images: | + acryldata/datahub-ingestion-base + tags: latest username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and Push image - uses: docker/build-push-action@v3 - with: + publish: ${{ github.ref == 'refs/heads/master' }} context: ./docker/datahub-ingestion file: ./docker/datahub-ingestion/base.Dockerfile platforms: linux/amd64,linux/arm64 - tags: acryldata/datahub-ingestion-base:latest - push: true ingestion-base_scan: permissions: contents: read # for actions/checkout to fetch code @@ -59,6 +56,7 @@ jobs: uses: actions/checkout@v3 - name: Download image uses: ishworkh/docker-image-artifact-download@v1 + if: ${{ github.ref == 'refs/heads/master' }} with: image: acryldata/datahub-ingestion-base:latest - name: Run Trivy vulnerability scanner diff --git a/docker/datahub-ingestion/base-requirements.txt b/docker/datahub-ingestion/base-requirements.txt index cc838a868e..00321f8373 100644 --- a/docker/datahub-ingestion/base-requirements.txt +++ b/docker/datahub-ingestion/base-requirements.txt @@ -3,14 +3,14 @@ acryl-datahub-classify==0.0.4 acryl-iceberg-legacy==0.0.4 acryl-PyHive==0.6.13 aenum==3.1.11 -aiohttp==3.8.3 +aiohttp==3.8.4 aiosignal==1.3.1 -alembic==1.9.2 +alembic==1.9.4 altair==4.2.2 anyio==3.6.2 apache-airflow==2.5.1 apache-airflow-providers-common-sql==1.3.3 -apache-airflow-providers-ftp==3.3.0 +apache-airflow-providers-ftp==3.3.1 apache-airflow-providers-http==4.1.1 apache-airflow-providers-imap==3.1.1 apache-airflow-providers-sqlite==3.3.1 @@ -37,8 +37,8 @@ beautifulsoup4==4.11.2 bleach==6.0.0 blinker==1.5 blis==0.7.9 -boto3==1.26.65 -botocore==1.29.65 +boto3==1.26.75 +botocore==1.29.75 bowler==0.9.0 bracex==2.3.post1 cached-property==1.5.2 @@ -68,7 +68,7 @@ croniter==1.3.8 cryptography==37.0.4 cx-Oracle==8.3.0 cymem==2.0.7 -dask==2023.1.1 +dask==2023.2.0 databricks-cli==0.17.3 databricks-dbapi==0.6.0 debugpy==1.6.6 @@ -88,23 +88,23 @@ et-xmlfile==1.1.0 exceptiongroup==1.1.0 executing==1.2.0 expandvars==0.9.0 -fastapi==0.89.1 +fastapi==0.92.0 fastavro==1.7.1 fastjsonschema==2.16.2 feast==0.29.0 filelock==3.9.0 fissix==21.11.13 -Flask==2.2.2 +Flask==2.2.3 flatdict==4.0.1 frozenlist==1.3.3 fsspec==2023.1.0 future==0.18.3 GeoAlchemy2==0.13.1 gitdb==4.0.10 -GitPython==3.1.30 +GitPython==3.1.31 google-api-core==2.11.0 -google-api-python-client==2.76.0 -google-auth==2.16.0 +google-api-python-client==2.78.0 +google-auth==2.16.1 google-auth-httplib2==0.1.0 google-cloud-appengine-logging==1.3.0 google-cloud-audit-log==0.2.5 @@ -112,7 +112,7 @@ google-cloud-bigquery==3.5.0 google-cloud-bigquery-storage==2.18.1 google-cloud-core==2.3.2 google-cloud-datacatalog-lineage==0.2.0 -google-cloud-logging==3.1.1 +google-cloud-logging==3.5.0 google-crc32c==1.5.0 google-resumable-media==2.4.1 googleapis-common-protos==1.58.0 @@ -140,7 +140,7 @@ importlib-metadata==6.0.0 inflection==0.5.1 ipaddress==1.0.23 ipykernel==6.17.1 -ipython==8.9.0 +ipython==8.10.0 ipython-genutils==0.2.0 ipywidgets==8.0.4 iso3166==2.1.1 @@ -153,8 +153,9 @@ JPype1==1.4.1 jsonlines==3.1.0 jsonpatch==1.32 jsonpointer==2.3 +jsonref==1.1.0 jsonschema==4.17.3 -jupyter-server==1.23.5 +jupyter-server==1.23.6 jupyter_client==7.4.9 jupyter_core==4.12.0 jupyterlab-pygments==0.2.2 @@ -179,22 +180,22 @@ marshmallow-enum==1.5.1 marshmallow-oneofschema==3.0.1 marshmallow-sqlalchemy==0.26.1 matplotlib-inline==0.1.6 -mdit-py-plugins==0.3.3 +mdit-py-plugins==0.3.4 mdurl==0.1.2 mistune==2.0.5 mixpanel==4.10.0 mmh3==3.0.0 more-itertools==9.0.0 moreorless==0.4.0 -moto==4.1.2 +moto==4.1.3 msal==1.16.0 msal-extensions==1.0.0 msrest==0.7.1 multidict==6.0.4 murmurhash==1.0.9 -mypy==1.0.0 +mypy==1.0.1 mypy-extensions==1.0.0 -nbclassic==0.5.1 +nbclassic==0.5.2 nbclient==0.6.3 nbconvert==7.2.9 nbformat==5.7.3 @@ -202,11 +203,10 @@ nest-asyncio==1.5.6 networkx==3.0 notebook==6.5.2 notebook_shim==0.2.2 -ntlm-auth==1.5.0 numpy==1.24.2 oauthlib==3.2.2 okta==1.7.0 -openpyxl==3.1.0 +openpyxl==3.1.1 oscrypto==1.3.0 packaging==21.3 pandas==1.5.3 @@ -230,7 +230,7 @@ progressbar2==4.2.0 prometheus-client==0.16.0 prompt-toolkit==3.0.36 proto-plus==1.22.2 -protobuf==4.21.12 +protobuf==4.22.0 psutil==5.9.4 psycopg2-binary==2.9.5 ptyprocess==0.7.0 @@ -244,8 +244,8 @@ pycountry==22.3.5 pycparser==2.21 pycryptodome==3.17 pycryptodomex==3.17 -pydantic==1.10.4 -pydash==6.0.0 +pydantic==1.10.5 +pydash==6.0.1 pydeequ==1.0.1 pydruid==0.6.5 Pygments==2.14.0 @@ -255,6 +255,7 @@ pyOpenSSL==22.0.0 pyparsing==3.0.9 pyrsistent==0.19.3 pyspark==3.0.3 +pyspnego==0.8.0 python-daemon==2.3.2 python-dateutil==2.8.2 python-dotenv==0.21.1 @@ -263,8 +264,8 @@ python-ldap==3.4.3 python-nvd3==0.15.0 python-slugify==8.0.0 python-stdnum==1.18 -python-tds==1.11.0 -python-utils==3.4.5 +python-tds==1.12.0 +python-utils==3.5.2 python3-openid==3.2.0 pytz==2022.7.1 pytz-deprecation-shim==0.1.0.post0 @@ -276,7 +277,7 @@ redash-toolbelt==0.1.9 regex==2022.10.31 requests==2.28.2 requests-file==1.5.1 -requests-ntlm==1.1.0 +requests-ntlm==1.2.0 requests-oauthlib==1.3.1 requests-toolbelt==0.10.1 responses==0.22.0 @@ -288,7 +289,7 @@ ruamel.yaml==0.17.17 s3transfer==0.6.0 sasl3==0.2.11 schwifty==2023.2.0 -scipy==1.10.0 +scipy==1.10.1 Send2Trash==1.8.0 setproctitle==1.3.2 simple-salesforce==1.12.3 @@ -297,8 +298,8 @@ smart-open==6.3.0 smmap==5.0.0 sniffio==1.3.0 snowflake-connector-python==2.9.0 -snowflake-sqlalchemy==1.4.4 -soupsieve==2.3.2.post1 +snowflake-sqlalchemy==1.4.6 +soupsieve==2.4 spacy==3.4.3 spacy-legacy==3.0.12 spacy-loggers==1.0.4 @@ -306,21 +307,20 @@ sql-metadata==2.2.2 SQLAlchemy==1.4.41 sqlalchemy-bigquery==1.6.1 SQLAlchemy-JSONField==1.0.1.post0 -sqlalchemy-pytds==0.3.4 +sqlalchemy-pytds==0.3.5 sqlalchemy-redshift==0.8.12 -SQLAlchemy-Utils==0.39.0 +SQLAlchemy-Utils==0.40.0 sqlalchemy2-stubs==0.0.2a32 sqllineage==1.3.6 sqlparse==0.4.3 srsly==2.4.5 stack-data==0.6.2 -stackprinter==0.2.10 -starlette==0.22.0 -tableauserverclient==0.23.4 +starlette==0.25.0 +tableauserverclient==0.24 tableschema==1.20.2 tabulate==0.9.0 tabulator==1.53.5 -tenacity==8.1.0 +tenacity==8.2.1 termcolor==2.2.0 terminado==0.17.1 text-unidecode==1.3 @@ -337,9 +337,9 @@ traitlets==5.2.1.post0 trino==0.321.0 typeguard==2.13.3 typer==0.7.0 -types-toml==0.10.8.3 +types-toml==0.10.8.5 typing-inspect==0.8.0 -typing_extensions==4.4.0 +typing_extensions==4.5.0 tzdata==2022.7 tzlocal==4.2 uc-micro-py==1.0.1 @@ -349,7 +349,7 @@ uritemplate==4.1.1 urllib3==1.26.14 uvicorn==0.20.0 uvloop==0.17.0 -vertica-python==1.2.0 +vertica-python==1.3.0 vertica-sqlalchemy-dialect==0.0.1 vininfo==1.7.0 volatile==2.1.0 @@ -360,7 +360,7 @@ wcwidth==0.2.6 webencodings==0.5.1 websocket-client==1.5.1 websockets==10.4 -Werkzeug==2.2.2 +Werkzeug==2.2.3 widgetsnbextension==4.0.5 wrapt==1.14.1 WTForms==3.0.1