yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-12 18:47:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ci): use local ingestion in actions (#13408)

Browse Source
This commit is contained in:
Harshal Sheth 2025-05-02 19:22:09 -07:00 committed by GitHub
parent 096e6d9af4
commit 24f9bc0f18
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
datahub-actions/build.gradle
View File

@ -122,7 +122,7 @@ task cleanPythonCache(type: Exec) {
} }
docker { docker {
//dependsOn(build) dependsOn ':metadata-ingestion:codegen'
name "${docker_registry}/${docker_repo}:${versionTag}" name "${docker_registry}/${docker_repo}:${versionTag}"
dockerfile file("${rootProject.projectDir}/docker/datahub-actions/Dockerfile") dockerfile file("${rootProject.projectDir}/docker/datahub-actions/Dockerfile")
files fileTree(rootProject.projectDir) { files fileTree(rootProject.projectDir) {
@ -134,6 +134,7 @@ docker {
include ".dockerignore" include ".dockerignore"
include "docker/datahub-actions/**" include "docker/datahub-actions/**"
include "docker/snippets/**" include "docker/snippets/**"
include "metadata-ingestion/**"
include "datahub-actions/**" include "datahub-actions/**"
include "python-build/**" include "python-build/**"
}.exclude { }.exclude {

4
docker/datahub-actions/Dockerfile
View File

@ -139,6 +139,7 @@ RUN chmod a+x /start_datahub_actions.sh && \
mkdir -p /tmp/datahub/logs/actions/system && \ mkdir -p /tmp/datahub/logs/actions/system && \
chown -R datahub:datahub /etc/datahub /tmp/datahub chown -R datahub:datahub /etc/datahub /tmp/datahub
COPY --chown=datahub:datahub ./metadata-ingestion /metadata-ingestion
COPY --chown=datahub:datahub ./datahub-actions /datahub-actions COPY --chown=datahub:datahub ./datahub-actions /datahub-actions
# Add other default configurations into this! # Add other default configurations into this!
COPY --chown=datahub:datahub ./docker/datahub-actions/config /etc/datahub/actions/system/conf COPY --chown=datahub:datahub ./docker/datahub-actions/config /etc/datahub/actions/system/conf
@ -148,13 +149,14 @@ USER datahub
ARG RELEASE_VERSION ARG RELEASE_VERSION
RUN test -n "$RELEASE_VERSION" # RELEASE_VERSION is a required build arg RUN test -n "$RELEASE_VERSION" # RELEASE_VERSION is a required build arg
RUN --mount=type=bind,source=./python-build/version_updater.py,target=/version_updater.py \ RUN --mount=type=bind,source=./python-build/version_updater.py,target=/version_updater.py \
python /version_updater.py --directory /metadata-ingestion/ --version "$RELEASE_VERSION" --expected-update-count 1 && \
python /version_updater.py --directory /datahub-actions/ --version "$RELEASE_VERSION" --expected-update-count 1 python /version_updater.py --directory /datahub-actions/ --version "$RELEASE_VERSION" --expected-update-count 1
# For the datahub-actions build, we explicitly want to retain the uv cache. # For the datahub-actions build, we explicitly want to retain the uv cache.
# This speeds up the process of creating venvs at runtime. # This speeds up the process of creating venvs at runtime.
# Because uv uses hardlinks for installing packages, keeping the cache around does not # Because uv uses hardlinks for installing packages, keeping the cache around does not
# really impact image size. # really impact image size.
RUN uv pip install -e "/datahub-actions/[all]" RUN uv pip install -e '/metadata-ingestion/' -e '/datahub-actions/[all]'
ENTRYPOINT [ ] ENTRYPOINT [ ]
CMD dockerize -wait ${DATAHUB_GMS_PROTOCOL:-http}://$DATAHUB_GMS_HOST:$DATAHUB_GMS_PORT/health -timeout 240s /start_datahub_actions.sh CMD dockerize -wait ${DATAHUB_GMS_PROTOCOL:-http}://$DATAHUB_GMS_HOST:$DATAHUB_GMS_PORT/health -timeout 240s /start_datahub_actions.sh

4
docker/profiles/docker-compose.actions.yml
View File

@ -20,6 +20,10 @@ x-datahub-actions-service: &datahub-actions-service
x-datahub-actions-service-dev: &datahub-actions-service-dev x-datahub-actions-service-dev: &datahub-actions-service-dev
<<: *datahub-actions-service <<: *datahub-actions-service
volumes:
- ./datahub-actions/start.sh:/start_datahub_actions.sh
- ../../metadata-ingestion/src:/metadata-ingestion/src
- ../../datahub-actions/src:/datahub-actions/src
services: services:
datahub-actions-quickstart: datahub-actions-quickstart: