mirror of
https://github.com/datahub-project/datahub.git
synced 2026-01-05 14:24:43 +00:00
449 lines
16 KiB
YAML
449 lines
16 KiB
YAML
name: Unified Docker Workflow
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
paths-ignore:
|
|
- 'docs/**'
|
|
- '**.md'
|
|
pull_request:
|
|
branches:
|
|
- master
|
|
paths-ignore:
|
|
- 'docs/**'
|
|
- '**.md'
|
|
release:
|
|
types: [published, edited]
|
|
|
|
concurrency:
|
|
# Using `github.run_id` (unique val) instead of `github.ref` here
|
|
# because we don't want to cancel this workflow on master only for PRs
|
|
# as that makes reproducing issues easier
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
DATAHUB_GMS_IMAGE: 'linkedin/datahub-gms'
|
|
DATAHUB_FRONTEND_IMAGE: 'linkedin/datahub-frontend-react'
|
|
DATAHUB_MAE_CONSUMER_IMAGE: 'linkedin/datahub-mae-consumer'
|
|
DATAHUB_MCE_CONSUMER_IMAGE: 'linkedin/datahub-mce-consumer'
|
|
DATAHUB_KAFKA_SETUP_IMAGE: 'linkedin/datahub-kafka-setup'
|
|
DATAHUB_ELASTIC_SETUP_IMAGE: 'linkedin/datahub-elasticsearch-setup'
|
|
DATAHUB_MYSQL_SETUP_IMAGE: 'acryldata/datahub-mysql-setup'
|
|
DATAHUB_UPGRADE_IMAGE: 'acryldata/datahub-upgrade'
|
|
|
|
|
|
jobs:
|
|
setup:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
tag: ${{ steps.tag.outputs.tag }}
|
|
unique_tag: ${{ steps.tag.outputs.unique_tag }}
|
|
publish: ${{ steps.publish.outputs.publish }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v2
|
|
- name: Compute Tag
|
|
id: tag
|
|
run: |
|
|
echo "GITHUB_REF: $GITHUB_REF"
|
|
SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
|
|
TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
|
|
UNIQUE_TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
|
|
echo "tag=$TAG"
|
|
echo "unique_tag=$UNIQUE_TAG"
|
|
echo "::set-output name=tag::$TAG"
|
|
echo "::set-output name=unique_tag::$UNIQUE_TAG"
|
|
- name: Check whether publishing enabled
|
|
id: publish
|
|
env:
|
|
ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }}
|
|
run: |
|
|
echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
|
|
echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
|
|
|
|
gms_build:
|
|
name: Build and Push DataHub GMS Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: setup
|
|
outputs:
|
|
image_tag: ${{ steps.docker_meta.outputs.tags }}
|
|
image_name: ${{ env.DATAHUB_GMS_IMAGE }}
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build and push
|
|
uses: ./.github/actions/docker-custom-build-and-push
|
|
with:
|
|
images: |
|
|
${{ env.DATAHUB_GMS_IMAGE }}
|
|
tags: ${{ needs.setup.outputs.tag }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
publish: ${{ needs.setup.outputs.publish }}
|
|
context: .
|
|
file: ./docker/datahub-gms/Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
gms_scan:
|
|
name: "[Monitoring] Scan GMS images for vulnerabilities"
|
|
runs-on: ubuntu-latest
|
|
needs: [setup, gms_build]
|
|
steps:
|
|
- name: Download image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Run Trivy vulnerability scanner
|
|
uses: aquasecurity/trivy-action@master
|
|
with:
|
|
image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
format: 'table'
|
|
ignore-unfixed: true
|
|
vuln-type: 'os,library'
|
|
severity: 'CRITICAL,HIGH'
|
|
|
|
mae_consumer_build:
|
|
name: Build and Push DataHub MAE Consumer Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: setup
|
|
outputs:
|
|
image_tag: ${{ steps.docker_meta.outputs.tags }}
|
|
image_name: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build and push
|
|
uses: ./.github/actions/docker-custom-build-and-push
|
|
with:
|
|
images: |
|
|
${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
|
|
tags: ${{ needs.setup.outputs.tag }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
publish: ${{ needs.setup.outputs.publish }}
|
|
context: .
|
|
file: ./docker/datahub-mae-consumer/Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
mae_consumer_scan:
|
|
name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
|
|
runs-on: ubuntu-latest
|
|
needs: [setup,mae_consumer_build]
|
|
steps:
|
|
- name: Download image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Run Trivy vulnerability scanner
|
|
uses: aquasecurity/trivy-action@master
|
|
with:
|
|
image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
format: 'table'
|
|
ignore-unfixed: true
|
|
vuln-type: 'os,library'
|
|
severity: 'CRITICAL,HIGH'
|
|
|
|
mce_consumer_build:
|
|
name: Build and Push DataHub MCE Consumer Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: setup
|
|
outputs:
|
|
image_tag: ${{ steps.docker_meta.outputs.tags }}
|
|
image_name: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build and push
|
|
uses: ./.github/actions/docker-custom-build-and-push
|
|
with:
|
|
images: |
|
|
${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
|
|
tags: ${{ needs.setup.outputs.tag }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
publish: ${{ needs.setup.outputs.publish }}
|
|
context: .
|
|
file: ./docker/datahub-mce-consumer/Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
mce_consumer_scan:
|
|
name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
|
|
runs-on: ubuntu-latest
|
|
needs: [setup,mce_consumer_build]
|
|
steps:
|
|
- name: Download image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Run Trivy vulnerability scanner
|
|
uses: aquasecurity/trivy-action@master
|
|
with:
|
|
image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
format: 'table'
|
|
ignore-unfixed: true
|
|
vuln-type: 'os,library'
|
|
severity: 'CRITICAL,HIGH'
|
|
|
|
datahub_upgrade_build:
|
|
name: Build and Push DataHub Upgrade Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: setup
|
|
outputs:
|
|
image_tag: ${{ steps.docker_meta.outputs.tags }}
|
|
image_name: ${{ env.DATAHUB_UPGRADE_IMAGE }}
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build and push
|
|
uses: ./.github/actions/docker-custom-build-and-push
|
|
with:
|
|
images: |
|
|
${{ env.DATAHUB_UPGRADE_IMAGE }}
|
|
tags: ${{ needs.setup.outputs.tag }}
|
|
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
|
|
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
|
|
publish: ${{ needs.setup.outputs.publish }}
|
|
context: .
|
|
file: ./docker/datahub-upgrade/Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
datahub_upgrade_scan:
|
|
name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities"
|
|
runs-on: ubuntu-latest
|
|
needs: [setup, datahub_upgrade_build]
|
|
steps:
|
|
- name: Download image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Run Trivy vulnerability scanner
|
|
uses: aquasecurity/trivy-action@master
|
|
with:
|
|
image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
format: 'table'
|
|
ignore-unfixed: true
|
|
vuln-type: 'os,library'
|
|
severity: 'CRITICAL,HIGH'
|
|
|
|
frontend_build:
|
|
name: Build and Push DataHub Frontend Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: setup
|
|
outputs:
|
|
image_tag: ${{ steps.docker_meta.outputs.tags }}
|
|
image_name: ${{ env.DATAHUB_FRONTEND_IMAGE }}
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build and push
|
|
uses: ./.github/actions/docker-custom-build-and-push
|
|
with:
|
|
images: |
|
|
${{ env.DATAHUB_FRONTEND_IMAGE }}
|
|
tags: ${{ needs.setup.outputs.tag }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
publish: ${{ needs.setup.outputs.publish }}
|
|
context: .
|
|
file: ./docker/datahub-frontend/Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
frontend_scan:
|
|
name: "[Monitoring] Scan Frontend images for vulnerabilities"
|
|
runs-on: ubuntu-latest
|
|
needs: [setup, frontend_build]
|
|
steps:
|
|
- name: Download image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Run Trivy vulnerability scanner
|
|
uses: aquasecurity/trivy-action@master
|
|
with:
|
|
image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
format: 'table'
|
|
ignore-unfixed: true
|
|
vuln-type: 'os,library'
|
|
severity: 'CRITICAL,HIGH'
|
|
|
|
kafka_setup_build:
|
|
name: Build and Push DataHub Kafka Setup Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: setup
|
|
outputs:
|
|
image_tag: ${{ steps.docker_meta.outputs.tags }}
|
|
image_name: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build and push
|
|
uses: ./.github/actions/docker-custom-build-and-push
|
|
with:
|
|
images: |
|
|
${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
|
|
tags: ${{ needs.setup.outputs.tag }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
publish: ${{ needs.setup.outputs.publish }}
|
|
context: ./docker/kafka-setup
|
|
file: ./docker/kafka-setup/Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
|
|
mysql_setup_build:
|
|
name: Build and Push DataHub MySQL Setup Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: setup
|
|
outputs:
|
|
image_tag: ${{ steps.docker_meta.outputs.tags }}
|
|
image_name: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build and push
|
|
uses: ./.github/actions/docker-custom-build-and-push
|
|
with:
|
|
images: |
|
|
${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
|
|
tags: ${{ needs.setup.outputs.tag }}
|
|
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
|
|
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
|
|
publish: ${{ needs.setup.outputs.publish }}
|
|
context: .
|
|
file: ./docker/mysql-setup/Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
|
|
elasticsearch_setup_build:
|
|
name: Build and Push DataHub Elasticsearch Setup Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: setup
|
|
outputs:
|
|
image_tag: ${{ steps.docker_meta.outputs.tags }}
|
|
image_name: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build and push
|
|
uses: ./.github/actions/docker-custom-build-and-push
|
|
with:
|
|
images: |
|
|
${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
|
|
tags: ${{ needs.setup.outputs.tag }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
publish: ${{ needs.setup.outputs.publish }}
|
|
context: .
|
|
file: ./docker/elasticsearch-setup/Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
|
|
smoke_test:
|
|
name: Run Smoke Tests
|
|
runs-on: ubuntu-latest
|
|
needs: [setup,gms_build,frontend_build,kafka_setup_build,mysql_setup_build,elasticsearch_setup_build,mae_consumer_build,mce_consumer_build]
|
|
steps:
|
|
- name: Check out the repo
|
|
uses: actions/checkout@v2
|
|
- name: Set up JDK 11
|
|
uses: actions/setup-java@v1
|
|
with:
|
|
java-version: 11
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: "3.7"
|
|
- name: Install dependencies
|
|
run: ./metadata-ingestion/scripts/install_deps.sh
|
|
- name: Build datahub cli
|
|
run: |
|
|
./gradlew :metadata-ingestion:install
|
|
- name: Download GMS image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Download Frontend image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Download Kafka Setup image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Download Mysql Setup image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Download Elastic Setup image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Download MCE Consumer image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Download MCE Consumer image
|
|
uses: ishworkh/docker-image-artifact-download@v1
|
|
if: ${{ needs.setup.outputs.publish != 'true' }}
|
|
with:
|
|
image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
|
- name: Smoke test
|
|
env:
|
|
DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
|
|
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
|
run: |
|
|
echo "$DATAHUB_VERSION"
|
|
./smoke-test/smoke.sh
|
|
- name: store logs
|
|
if: failure()
|
|
run: |
|
|
docker ps -a
|
|
docker logs datahub-gms >& gms.log
|
|
- name: Upload logs
|
|
uses: actions/upload-artifact@v2
|
|
if: failure()
|
|
with:
|
|
name: docker logs
|
|
path: "*.log"
|
|
- name: Upload screenshots
|
|
uses: actions/upload-artifact@v2
|
|
if: failure()
|
|
with:
|
|
name: cypress-snapshots
|
|
path: smoke-test/tests/cypress/cypress/screenshots/
|
|
- uses: actions/upload-artifact@v2
|
|
if: always()
|
|
with:
|
|
name: Test Results (smoke tests)
|
|
path: |
|
|
**/build/reports/tests/test/**
|
|
**/build/test-results/test/**
|
|
**/junit.*.xml
|
|
- name: Slack failure notification
|
|
if: failure() && github.event_name == 'push'
|
|
uses: kpritam/slack-job-status-action@v1
|
|
with:
|
|
job-status: ${{ job.status }}
|
|
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
channel: github-activities
|
|
|