yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-18 14:16:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(build): run codegen when building datahub-ingestion image (#8869)

Browse Source
This commit is contained in:
Harshal Sheth 2023-09-26 00:10:49 -04:00 committed by GitHub
parent 53eaac5963
commit ed1d35c79b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/docker-unified.yml vendored
View File

@ -58,7 +58,7 @@ jobs:
echo "full_tag=$(get_tag)-full" >> $GITHUB_OUTPUT echo "full_tag=$(get_tag)-full" >> $GITHUB_OUTPUT
echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
echo "unique_slim_tag=$(get_unique_tag)-slim" >> $GITHUB_OUTPUT echo "unique_slim_tag=$(get_unique_tag)-slim" >> $GITHUB_OUTPUT
echo "unique_full_tag=$(get_unique_tag)-full" >> $GITHUB_OUTPUT echo "unique_full_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
- name: Check whether publishing enabled - name: Check whether publishing enabled
id: publish id: publish
@ -501,7 +501,7 @@ jobs:
platforms: linux/amd64,linux/arm64/v8 platforms: linux/amd64,linux/arm64/v8
- name: Compute DataHub Ingestion (Base-Slim) Tag - name: Compute DataHub Ingestion (Base-Slim) Tag
id: tag id: tag
run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}" >> $GITHUB_OUTPUT run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
datahub_ingestion_base_full_build: datahub_ingestion_base_full_build:
name: Build and Push DataHub Ingestion (Base-Full) Docker Image name: Build and Push DataHub Ingestion (Base-Full) Docker Image
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -567,13 +567,13 @@ jobs:
datahub-ingestion: datahub-ingestion:
- 'docker/datahub-ingestion/**' - 'docker/datahub-ingestion/**'
- name: Build codegen - name: Build codegen
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }} if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
run: ./gradlew :metadata-ingestion:codegen run: ./gradlew :metadata-ingestion:codegen
- name: Download Base Image - name: Download Base Image
uses: ishworkh/docker-image-artifact-download@v1 uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }}
with: with:
image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }} image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
- name: Build and push Slim Image - name: Build and push Slim Image
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
uses: ./.github/actions/docker-custom-build-and-push uses: ./.github/actions/docker-custom-build-and-push
@ -583,7 +583,7 @@ jobs:
${{ env.DATAHUB_INGESTION_IMAGE }} ${{ env.DATAHUB_INGESTION_IMAGE }}
build-args: | build-args: |
BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }} BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }} DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
APP_ENV=slim APP_ENV=slim
tags: ${{ needs.setup.outputs.slim_tag }} tags: ${{ needs.setup.outputs.slim_tag }}
@ -595,7 +595,7 @@ jobs:
platforms: linux/amd64,linux/arm64/v8 platforms: linux/amd64,linux/arm64/v8
- name: Compute Tag - name: Compute Tag
id: tag id: tag
run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head' }}" >> $GITHUB_OUTPUT run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
datahub_ingestion_slim_scan: datahub_ingestion_slim_scan:
permissions: permissions:
contents: read # for actions/checkout to fetch code contents: read # for actions/checkout to fetch code
@ -650,7 +650,7 @@ jobs:
datahub-ingestion: datahub-ingestion:
- 'docker/datahub-ingestion/**' - 'docker/datahub-ingestion/**'
- name: Build codegen - name: Build codegen
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }} if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
run: ./gradlew :metadata-ingestion:codegen run: ./gradlew :metadata-ingestion:codegen
- name: Download Base Image - name: Download Base Image
uses: ishworkh/docker-image-artifact-download@v1 uses: ishworkh/docker-image-artifact-download@v1

4
docker/datahub-ingestion-base/Dockerfile
View File

@ -1,7 +1,7 @@
ARG APP_ENV=full ARG APP_ENV=full
ARG BASE_IMAGE=base ARG BASE_IMAGE=base
FROM golang:1-alpine3.17 AS binary FROM golang:1-alpine3.17 AS dockerize-binary
ENV DOCKERIZE_VERSION v0.6.1 ENV DOCKERIZE_VERSION v0.6.1
WORKDIR /go/src/github.com/jwilder WORKDIR /go/src/github.com/jwilder
@ -41,7 +41,7 @@ RUN apt-get update && apt-get install -y -qq \
&& rm -rf /var/lib/apt/lists/* /var/cache/apk/* && rm -rf /var/lib/apt/lists/* /var/cache/apk/*
# compiled against newer golang for security fixes # compiled against newer golang for security fixes
COPY --from=binary /go/bin/dockerize /usr/local/bin COPY --from=dockerize-binary /go/bin/dockerize /usr/local/bin
COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt
COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh

2
docker/datahub-ingestion/Dockerfile
View File

@ -1,7 +1,7 @@
# Defining environment # Defining environment
ARG APP_ENV=full ARG APP_ENV=full
ARG BASE_IMAGE=acryldata/datahub-ingestion-base ARG BASE_IMAGE=acryldata/datahub-ingestion-base
ARG DOCKER_VERSION=latest ARG DOCKER_VERSION=head
FROM $BASE_IMAGE:$DOCKER_VERSION as base FROM $BASE_IMAGE:$DOCKER_VERSION as base
USER 0 USER 0

2
docker/datahub-ingestion/Dockerfile-slim-only
View File

@ -1,6 +1,6 @@
# Defining environment # Defining environment
ARG BASE_IMAGE=acryldata/datahub-ingestion-base ARG BASE_IMAGE=acryldata/datahub-ingestion-base
ARG DOCKER_VERSION=latest ARG DOCKER_VERSION=head-slim
FROM $BASE_IMAGE:$DOCKER_VERSION as base FROM $BASE_IMAGE:$DOCKER_VERSION as base
USER 0 USER 0