feat(ci): use local ingestion in actions (#13408)

datahub-actions/build.gradle

@@ -122,7 +122,7 @@ task cleanPythonCache(type: Exec) {
 }
 
 docker {
-  //dependsOn(build)
+  dependsOn ':metadata-ingestion:codegen'
   name "${docker_registry}/${docker_repo}:${versionTag}"
   dockerfile file("${rootProject.projectDir}/docker/datahub-actions/Dockerfile")
   files fileTree(rootProject.projectDir) {
@@ -134,12 +134,13 @@ docker {
     include ".dockerignore"
     include "docker/datahub-actions/**"
     include "docker/snippets/**"
+    include "metadata-ingestion/**"
     include "datahub-actions/**"
     include "python-build/**"
   }.exclude {
     i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
   }
-  
+
   additionalTag("Debug", "${docker_registry}/${docker_repo}:debug")
 
   defaultVariant = "slim"

docker/datahub-actions/Dockerfile

@@ -139,6 +139,7 @@ RUN chmod a+x /start_datahub_actions.sh && \
     mkdir -p /tmp/datahub/logs/actions/system && \
     chown -R datahub:datahub /etc/datahub /tmp/datahub
 
+COPY --chown=datahub:datahub ./metadata-ingestion /metadata-ingestion
 COPY --chown=datahub:datahub ./datahub-actions /datahub-actions
 # Add other default configurations into this!
 COPY --chown=datahub:datahub ./docker/datahub-actions/config /etc/datahub/actions/system/conf
@@ -148,13 +149,14 @@ USER datahub
 ARG RELEASE_VERSION
 RUN test -n "$RELEASE_VERSION"  # RELEASE_VERSION is a required build arg
 RUN --mount=type=bind,source=./python-build/version_updater.py,target=/version_updater.py \
+    python /version_updater.py --directory /metadata-ingestion/ --version "$RELEASE_VERSION" --expected-update-count 1 && \
     python /version_updater.py --directory /datahub-actions/ --version "$RELEASE_VERSION" --expected-update-count 1
 
 # For the datahub-actions build, we explicitly want to retain the uv cache.
 # This speeds up the process of creating venvs at runtime.
 # Because uv uses hardlinks for installing packages, keeping the cache around does not
 # really impact image size.
-RUN uv pip install -e "/datahub-actions/[all]"
+RUN uv pip install -e '/metadata-ingestion/' -e '/datahub-actions/[all]'
 
 ENTRYPOINT [ ]
 CMD dockerize -wait ${DATAHUB_GMS_PROTOCOL:-http}://$DATAHUB_GMS_HOST:$DATAHUB_GMS_PORT/health -timeout 240s /start_datahub_actions.sh

docker/profiles/docker-compose.actions.yml

@@ -20,6 +20,10 @@ x-datahub-actions-service: &datahub-actions-service
 
 x-datahub-actions-service-dev: &datahub-actions-service-dev
   <<: *datahub-actions-service
+  volumes:
+    - ./datahub-actions/start.sh:/start_datahub_actions.sh
+    - ../../metadata-ingestion/src:/metadata-ingestion/src
+    - ../../datahub-actions/src:/datahub-actions/src
 
 services:
   datahub-actions-quickstart: