mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-27 09:58:14 +00:00
chore(ci): update actions versions addressing warnings in builds. (#6308)
This commit is contained in:
parent
dfc20b3d40
commit
3eae087a30
16
.github/pr-labeler-config.yml
vendored
16
.github/pr-labeler-config.yml
vendored
@ -1,15 +1,15 @@
|
||||
ingestion:
|
||||
- 'metadata-ingestion/**/*'
|
||||
- "metadata-ingestion/**/*"
|
||||
|
||||
devops:
|
||||
- 'docker/**/*'
|
||||
- '.github/**/*'
|
||||
- "docker/**/*"
|
||||
- ".github/**/*"
|
||||
|
||||
product:
|
||||
- 'datahub-web-react/**/*'
|
||||
- 'datahub-frontend/**/*'
|
||||
- 'datahub-graphql-core/**/*'
|
||||
- 'metadata-io/**/*'
|
||||
- "datahub-web-react/**/*"
|
||||
- "datahub-frontend/**/*"
|
||||
- "datahub-graphql-core/**/*"
|
||||
- "metadata-io/**/*"
|
||||
|
||||
docs:
|
||||
- 'docs/**/*'
|
||||
- "docs/**/*"
|
||||
|
||||
4
.github/pull_request_template.md
vendored
4
.github/pull_request_template.md
vendored
@ -1,8 +1,8 @@
|
||||
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] The PR conforms to DataHub's [Contributing Guideline](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md) (particularly [Commit Message Format](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md#commit-message-format))
|
||||
- [ ] Links to related issues (if applicable)
|
||||
- [ ] Tests for the changes have been added/updated (if applicable)
|
||||
- [ ] Docs related to the changes have been added/updated (if applicable). If a new feature has been added a Usage Guide has been added for the same.
|
||||
- [ ] For any breaking change/potential downtime/deprecation/big changes an entry has been made in [Updating DataHub](https://github.com/datahub-project/datahub/blob/master/docs/how/updating-datahub.md)
|
||||
- [ ] For any breaking change/potential downtime/deprecation/big changes an entry has been made in [Updating DataHub](https://github.com/datahub-project/datahub/blob/master/docs/how/updating-datahub.md)
|
||||
|
||||
15
.github/workflows/build-and-test.yml
vendored
15
.github/workflows/build-and-test.yml
vendored
@ -24,20 +24,21 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: 11
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
- name: Gradle build (and test)
|
||||
run: |
|
||||
./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test
|
||||
- uses: actions/upload-artifact@v2
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: Test Results (build)
|
||||
@ -58,8 +59,8 @@ jobs:
|
||||
quickstart-compose-validation:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
- name: Quickstart Compose Validation
|
||||
@ -69,7 +70,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Upload
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Event File
|
||||
path: ${{ github.event_path }}
|
||||
|
||||
8
.github/workflows/check-datahub-jars.yml
vendored
8
.github/workflows/check-datahub-jars.yml
vendored
@ -23,18 +23,18 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
|
||||
check_jars:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: 11
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
- name: check datahub-client jar
|
||||
|
||||
55
.github/workflows/check-quickstart.yml
vendored
55
.github/workflows/check-quickstart.yml
vendored
@ -9,7 +9,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
|
||||
jobs:
|
||||
test-quickstart:
|
||||
strategy:
|
||||
@ -18,30 +17,30 @@ jobs:
|
||||
os: [ubuntu-20.04]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.10"
|
||||
- name: Install acryl-datahub
|
||||
run: |
|
||||
pip install --upgrade acryl-datahub
|
||||
datahub version
|
||||
python -c "import platform; print(platform.platform())"
|
||||
- name: Run quickstart
|
||||
run: |
|
||||
datahub docker quickstart
|
||||
- name: Ingest sample data
|
||||
run: |
|
||||
datahub docker ingest-sample-data
|
||||
- name: See status
|
||||
run: |
|
||||
docker ps -a && datahub docker check
|
||||
- name: store logs
|
||||
if: failure()
|
||||
run: |
|
||||
docker logs datahub-gms >& quickstart-gms.log
|
||||
- name: Upload logs
|
||||
uses: actions/upload-artifact@v2
|
||||
if: failure()
|
||||
with:
|
||||
name: docker-quickstart-logs-${{ matrix.os }}
|
||||
path: "*.log"
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
- name: Install acryl-datahub
|
||||
run: |
|
||||
pip install --upgrade acryl-datahub
|
||||
datahub version
|
||||
python -c "import platform; print(platform.platform())"
|
||||
- name: Run quickstart
|
||||
run: |
|
||||
datahub docker quickstart
|
||||
- name: Ingest sample data
|
||||
run: |
|
||||
datahub docker ingest-sample-data
|
||||
- name: See status
|
||||
run: |
|
||||
docker ps -a && datahub docker check
|
||||
- name: store logs
|
||||
if: failure()
|
||||
run: |
|
||||
docker logs datahub-gms >& quickstart-gms.log
|
||||
- name: Upload logs
|
||||
uses: actions/upload-artifact@v3
|
||||
if: failure()
|
||||
with:
|
||||
name: docker-quickstart-logs-${{ matrix.os }}
|
||||
path: "*.log"
|
||||
|
||||
2
.github/workflows/close-stale-issues.yml
vendored
2
.github/workflows/close-stale-issues.yml
vendored
@ -10,7 +10,7 @@ jobs:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/stale@v5
|
||||
- uses: actions/stale@v6
|
||||
with:
|
||||
ascending: true
|
||||
operations-per-run: 100
|
||||
|
||||
8
.github/workflows/delete-artifacts.yaml
vendored
8
.github/workflows/delete-artifacts.yaml
vendored
@ -12,7 +12,7 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Remove old artifacts
|
||||
uses: c-hive/gha-remove-artifacts@v1
|
||||
with:
|
||||
age: '1 second'
|
||||
- name: Remove old artifacts
|
||||
uses: c-hive/gha-remove-artifacts@v1
|
||||
with:
|
||||
age: "1 second"
|
||||
|
||||
12
.github/workflows/docker-feast-source.yml
vendored
12
.github/workflows/docker-feast-source.yml
vendored
@ -4,14 +4,14 @@ on:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**'
|
||||
- '.github/workflows/docker-feast-source.yml'
|
||||
- "metadata-ingestion/src/datahub/ingestion/source/feast_image/**"
|
||||
- ".github/workflows/docker-feast-source.yml"
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**'
|
||||
- '.github/workflows/docker-feast-source.yml'
|
||||
- "metadata-ingestion/src/datahub/ingestion/source/feast_image/**"
|
||||
- ".github/workflows/docker-feast-source.yml"
|
||||
release:
|
||||
types: [published, edited]
|
||||
|
||||
@ -27,7 +27,7 @@ jobs:
|
||||
publish: ${{ steps.publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
- name: Compute Tag
|
||||
id: tag
|
||||
run: |
|
||||
@ -51,7 +51,7 @@ jobs:
|
||||
needs: setup
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Docker meta
|
||||
|
||||
3
.github/workflows/docker-ingestion-base.yml
vendored
3
.github/workflows/docker-ingestion-base.yml
vendored
@ -15,13 +15,12 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
|
||||
build-base:
|
||||
name: Build and Push Docker Image to Docker Hub
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up QEMU
|
||||
|
||||
4
.github/workflows/docker-ingestion-smoke.yml
vendored
4
.github/workflows/docker-ingestion-smoke.yml
vendored
@ -27,7 +27,7 @@ jobs:
|
||||
publish: ${{ steps.publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
- name: Compute Tag
|
||||
id: tag
|
||||
run: |
|
||||
@ -55,7 +55,7 @@ jobs:
|
||||
image_name: ${{ env.IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
|
||||
4
.github/workflows/docker-ingestion.yml
vendored
4
.github/workflows/docker-ingestion.yml
vendored
@ -30,7 +30,7 @@ jobs:
|
||||
python_release_version: ${{ steps.python_release_version.outputs.release_version }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
- name: Compute Tag
|
||||
id: tag
|
||||
run: |
|
||||
@ -60,7 +60,7 @@ jobs:
|
||||
needs: setup
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Docker meta
|
||||
|
||||
12
.github/workflows/docker-postgres-setup.yml
vendored
12
.github/workflows/docker-postgres-setup.yml
vendored
@ -4,14 +4,14 @@ on:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'docker/postgres-setup/**'
|
||||
- '.github/workflows/docker-postgres-setup.yml'
|
||||
- "docker/postgres-setup/**"
|
||||
- ".github/workflows/docker-postgres-setup.yml"
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'docker/postgres-setup/**'
|
||||
- '.github/workflows/docker-postgres-setup.yml'
|
||||
- "docker/postgres-setup/**"
|
||||
- ".github/workflows/docker-postgres-setup.yml"
|
||||
release:
|
||||
types: [published, edited]
|
||||
|
||||
@ -27,7 +27,7 @@ jobs:
|
||||
publish: ${{ steps.publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
- name: Compute Tag
|
||||
id: tag
|
||||
run: |
|
||||
@ -50,7 +50,7 @@ jobs:
|
||||
needs: setup
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Docker meta
|
||||
|
||||
105
.github/workflows/docker-unified.yml
vendored
105
.github/workflows/docker-unified.yml
vendored
@ -4,14 +4,14 @@ on:
|
||||
branches:
|
||||
- master
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- '**.md'
|
||||
- "docs/**"
|
||||
- "**.md"
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- '**.md'
|
||||
- "docs/**"
|
||||
- "**.md"
|
||||
release:
|
||||
types: [published, edited]
|
||||
|
||||
@ -23,15 +23,14 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
DATAHUB_GMS_IMAGE: 'linkedin/datahub-gms'
|
||||
DATAHUB_FRONTEND_IMAGE: 'linkedin/datahub-frontend-react'
|
||||
DATAHUB_MAE_CONSUMER_IMAGE: 'linkedin/datahub-mae-consumer'
|
||||
DATAHUB_MCE_CONSUMER_IMAGE: 'linkedin/datahub-mce-consumer'
|
||||
DATAHUB_KAFKA_SETUP_IMAGE: 'linkedin/datahub-kafka-setup'
|
||||
DATAHUB_ELASTIC_SETUP_IMAGE: 'linkedin/datahub-elasticsearch-setup'
|
||||
DATAHUB_MYSQL_SETUP_IMAGE: 'acryldata/datahub-mysql-setup'
|
||||
DATAHUB_UPGRADE_IMAGE: 'acryldata/datahub-upgrade'
|
||||
|
||||
DATAHUB_GMS_IMAGE: "linkedin/datahub-gms"
|
||||
DATAHUB_FRONTEND_IMAGE: "linkedin/datahub-frontend-react"
|
||||
DATAHUB_MAE_CONSUMER_IMAGE: "linkedin/datahub-mae-consumer"
|
||||
DATAHUB_MCE_CONSUMER_IMAGE: "linkedin/datahub-mce-consumer"
|
||||
DATAHUB_KAFKA_SETUP_IMAGE: "linkedin/datahub-kafka-setup"
|
||||
DATAHUB_ELASTIC_SETUP_IMAGE: "linkedin/datahub-elasticsearch-setup"
|
||||
DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup"
|
||||
DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade"
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
@ -42,7 +41,7 @@ jobs:
|
||||
publish: ${{ steps.publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
- name: Compute Tag
|
||||
id: tag
|
||||
run: |
|
||||
@ -71,7 +70,7 @@ jobs:
|
||||
image_name: ${{ env.DATAHUB_GMS_IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
@ -100,10 +99,10 @@ jobs:
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
||||
format: 'table'
|
||||
format: "table"
|
||||
ignore-unfixed: true
|
||||
vuln-type: 'os,library'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
vuln-type: "os,library"
|
||||
severity: "CRITICAL,HIGH"
|
||||
|
||||
mae_consumer_build:
|
||||
name: Build and Push DataHub MAE Consumer Docker Image
|
||||
@ -114,7 +113,7 @@ jobs:
|
||||
image_name: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
@ -132,7 +131,7 @@ jobs:
|
||||
mae_consumer_scan:
|
||||
name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup,mae_consumer_build]
|
||||
needs: [setup, mae_consumer_build]
|
||||
steps:
|
||||
- name: Download image
|
||||
uses: ishworkh/docker-image-artifact-download@v1
|
||||
@ -143,10 +142,10 @@ jobs:
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
||||
format: 'table'
|
||||
format: "table"
|
||||
ignore-unfixed: true
|
||||
vuln-type: 'os,library'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
vuln-type: "os,library"
|
||||
severity: "CRITICAL,HIGH"
|
||||
|
||||
mce_consumer_build:
|
||||
name: Build and Push DataHub MCE Consumer Docker Image
|
||||
@ -157,7 +156,7 @@ jobs:
|
||||
image_name: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
@ -175,7 +174,7 @@ jobs:
|
||||
mce_consumer_scan:
|
||||
name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup,mce_consumer_build]
|
||||
needs: [setup, mce_consumer_build]
|
||||
steps:
|
||||
- name: Download image
|
||||
uses: ishworkh/docker-image-artifact-download@v1
|
||||
@ -186,10 +185,10 @@ jobs:
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
||||
format: 'table'
|
||||
format: "table"
|
||||
ignore-unfixed: true
|
||||
vuln-type: 'os,library'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
vuln-type: "os,library"
|
||||
severity: "CRITICAL,HIGH"
|
||||
|
||||
datahub_upgrade_build:
|
||||
name: Build and Push DataHub Upgrade Docker Image
|
||||
@ -200,7 +199,7 @@ jobs:
|
||||
image_name: ${{ env.DATAHUB_UPGRADE_IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
@ -229,10 +228,10 @@ jobs:
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
||||
format: 'table'
|
||||
format: "table"
|
||||
ignore-unfixed: true
|
||||
vuln-type: 'os,library'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
vuln-type: "os,library"
|
||||
severity: "CRITICAL,HIGH"
|
||||
|
||||
frontend_build:
|
||||
name: Build and Push DataHub Frontend Docker Image
|
||||
@ -243,7 +242,7 @@ jobs:
|
||||
image_name: ${{ env.DATAHUB_FRONTEND_IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
@ -272,10 +271,10 @@ jobs:
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
||||
format: 'table'
|
||||
format: "table"
|
||||
ignore-unfixed: true
|
||||
vuln-type: 'os,library'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
vuln-type: "os,library"
|
||||
severity: "CRITICAL,HIGH"
|
||||
|
||||
kafka_setup_build:
|
||||
name: Build and Push DataHub Kafka Setup Docker Image
|
||||
@ -286,7 +285,7 @@ jobs:
|
||||
image_name: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
@ -311,7 +310,7 @@ jobs:
|
||||
image_name: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
@ -336,7 +335,7 @@ jobs:
|
||||
image_name: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and push
|
||||
@ -355,15 +354,26 @@ jobs:
|
||||
smoke_test:
|
||||
name: Run Smoke Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup,gms_build,frontend_build,kafka_setup_build,mysql_setup_build,elasticsearch_setup_build,mae_consumer_build,mce_consumer_build]
|
||||
needs:
|
||||
[
|
||||
setup,
|
||||
gms_build,
|
||||
frontend_build,
|
||||
kafka_setup_build,
|
||||
mysql_setup_build,
|
||||
elasticsearch_setup_build,
|
||||
mae_consumer_build,
|
||||
mce_consumer_build,
|
||||
]
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: 11
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
- name: Install dependencies
|
||||
@ -380,7 +390,7 @@ jobs:
|
||||
uses: ishworkh/docker-image-artifact-download@v1
|
||||
if: ${{ needs.setup.outputs.publish != 'true' }}
|
||||
with:
|
||||
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
||||
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
|
||||
- name: Download Kafka Setup image
|
||||
uses: ishworkh/docker-image-artifact-download@v1
|
||||
if: ${{ needs.setup.outputs.publish != 'true' }}
|
||||
@ -419,18 +429,18 @@ jobs:
|
||||
docker ps -a
|
||||
docker logs datahub-gms >& gms.log
|
||||
- name: Upload logs
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
if: failure()
|
||||
with:
|
||||
name: docker logs
|
||||
path: "*.log"
|
||||
- name: Upload screenshots
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
if: failure()
|
||||
with:
|
||||
name: cypress-snapshots
|
||||
path: smoke-test/tests/cypress/cypress/screenshots/
|
||||
- uses: actions/upload-artifact@v2
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: Test Results (smoke tests)
|
||||
@ -445,4 +455,3 @@ jobs:
|
||||
job-status: ${{ job.status }}
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
channel: github-activities
|
||||
|
||||
|
||||
7
.github/workflows/documentation.yml
vendored
7
.github/workflows/documentation.yml
vendored
@ -18,12 +18,13 @@ jobs:
|
||||
gh-pages:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: 11
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
- name: Install Python dependencies
|
||||
|
||||
14
.github/workflows/metadata-ingestion.yml
vendored
14
.github/workflows/metadata-ingestion.yml
vendored
@ -29,15 +29,15 @@ jobs:
|
||||
matrix:
|
||||
python-version: ["3.7", "3.10.6"]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: ./metadata-ingestion/scripts/install_deps.sh
|
||||
- name: Run metadata-ingestion tests
|
||||
run: ./gradlew :metadata-ingestion:build :metadata-ingestion:testQuick :metadata-ingestion:check
|
||||
- uses: actions/upload-artifact@v2
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: Test Results (metadata ingestion ${{ matrix.python-version }} testQuick)
|
||||
@ -64,8 +64,8 @@ jobs:
|
||||
]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: vemonet/setup-spark@v1 # spark is required for pyspark+pydeequ data lake profiling
|
||||
@ -79,7 +79,7 @@ jobs:
|
||||
- name: pip freeze show list installed
|
||||
if: always()
|
||||
run: source metadata-ingestion/venv/bin/activate && pip freeze
|
||||
- uses: actions/upload-artifact@v2
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: Test Results (metadata ingestion ${{ matrix.python-version }})
|
||||
@ -92,7 +92,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Upload
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Event File
|
||||
path: ${{ github.event_path }}
|
||||
|
||||
11
.github/workflows/metadata-io.yml
vendored
11
.github/workflows/metadata-io.yml
vendored
@ -28,12 +28,13 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: 11
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
- name: Gradle build (and test)
|
||||
@ -42,7 +43,7 @@ jobs:
|
||||
# datahub-frontend:unzipAssets depends on datahub-web-react:yarnBuild but gradle does not know about it
|
||||
run: |
|
||||
./gradlew :metadata-io:test
|
||||
- uses: actions/upload-artifact@v2
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: Test Results (metadata-io)
|
||||
@ -57,7 +58,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Upload
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Event File
|
||||
path: ${{ github.event_path }}
|
||||
|
||||
6
.github/workflows/metadata-model.yml
vendored
6
.github/workflows/metadata-model.yml
vendored
@ -30,8 +30,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: setup
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
- name: Install dependencies
|
||||
@ -44,5 +44,3 @@ jobs:
|
||||
DATAHUB_SERVER: ${{ secrets.DataHubServer }}
|
||||
DATAHUB_TOKEN: ${{ secrets.DataHubToken }}
|
||||
run: ./gradlew :metadata-ingestion:modelDocUpload
|
||||
|
||||
|
||||
|
||||
8
.github/workflows/pr-labeler.yml
vendored
8
.github/workflows/pr-labeler.yml
vendored
@ -10,7 +10,7 @@ jobs:
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/labeler@v4
|
||||
with:
|
||||
repo-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
configuration-path: '.github/pr-labeler-config.yml'
|
||||
- uses: actions/labeler@v4
|
||||
with:
|
||||
repo-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
configuration-path: ".github/pr-labeler-config.yml"
|
||||
|
||||
9
.github/workflows/publish-datahub-jars.yml
vendored
9
.github/workflows/publish-datahub-jars.yml
vendored
@ -36,7 +36,7 @@ jobs:
|
||||
tag: ${{ steps.tag.outputs.tag }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
- name: Compute Tag
|
||||
id: tag
|
||||
run: |
|
||||
@ -50,14 +50,15 @@ jobs:
|
||||
needs: ["check-secret", "setup"]
|
||||
if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: 11
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
- name: checkout upstream repo
|
||||
|
||||
10
.github/workflows/spark-smoke-test.yml
vendored
10
.github/workflows/spark-smoke-test.yml
vendored
@ -27,12 +27,13 @@ jobs:
|
||||
spark-smoke-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: 11
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
- name: Install dependencies
|
||||
@ -41,7 +42,7 @@ jobs:
|
||||
run: ./gradlew build -x check -x docs-website:build -x test
|
||||
- name: Smoke test
|
||||
run: ./gradlew metadata-integration:java:spark-lineage:integrationTest
|
||||
- uses: actions/upload-artifact@v2
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: Test Results (smoke tests)
|
||||
@ -56,4 +57,3 @@ jobs:
|
||||
job-status: ${{ job.status }}
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
channel: github-activities
|
||||
|
||||
|
||||
1
.github/workflows/test-results.yml
vendored
1
.github/workflows/test-results.yml
vendored
@ -36,4 +36,3 @@ jobs:
|
||||
event_file: artifacts/Event File/event.json
|
||||
event_name: ${{ github.event.workflow_run.event }}
|
||||
files: "artifacts/**/*.xml"
|
||||
|
||||
|
||||
@ -2,9 +2,10 @@
|
||||
|
||||
#### [Recommended] Create a GitHub Deploy Key
|
||||
|
||||
To use LookML ingestion through the UI, or automate github checkout through the cli, you must set up a GitHub deploy key for your Looker GitHub repository. Read [this](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) document for how to set up deploy keys for your Looker git repo.
|
||||
To use LookML ingestion through the UI, or automate github checkout through the cli, you must set up a GitHub deploy key for your Looker GitHub repository. Read [this](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) document for how to set up deploy keys for your Looker git repo.
|
||||
|
||||
In a nutshell, there are three steps:
|
||||
|
||||
In a nutshell, there are three steps:
|
||||
1. Generate a private-public ssh key pair. This will typically generate two files, e.g. looker_datahub_deploy_key (this is the private key) and looker_datahub_deploy_key.pub (this is the public key)
|
||||

|
||||
|
||||
@ -16,18 +17,19 @@ In a nutshell, there are three steps:
|
||||
#### [Optional] Create an API key with admin privileges
|
||||
|
||||
See the [Looker authentication docs](https://docs.looker.com/reference/api-and-integration/api-auth#authentication_with_an_sdk) for the steps to create a client ID and secret.
|
||||
You need to ensure that the API key is attached to a user that has Admin privileges.
|
||||
You need to ensure that the API key is attached to a user that has Admin privileges.
|
||||
|
||||
If that is not possible, read the configuration section and provide an offline specification of the `connection_to_platform_map` and the `project_name`.
|
||||
|
||||
### Ingestion Options
|
||||
|
||||
You have 3 options for controlling where your ingestion of LookML is run.
|
||||
You have 3 options for controlling where your ingestion of LookML is run.
|
||||
|
||||
- The DataHub UI (recommended for the easiest out-of-the-box experience)
|
||||
- As a GitHub Action (recommended to ensure that you have the freshest metadata pushed on change)
|
||||
- Using the CLI (scheduled via an orchestrator like Airflow)
|
||||
|
||||
Read on to learn more about these options.
|
||||
Read on to learn more about these options.
|
||||
|
||||
### UI-based Ingestion [Recommended for ease of use]
|
||||
|
||||
@ -60,9 +62,10 @@ The following sample GitHub action file can be modified to emit LookML metadata
|
||||
|
||||
Drop this file into your `.github/workflows` directory inside your Looker GitHub repo.
|
||||
You need to set up the following secrets in your GitHub repository to get this workflow to work:
|
||||
|
||||
- DATAHUB_GMS_HOST: The endpoint where your DataHub host is running
|
||||
- DATAHUB_TOKEN: An authentication token provisioned for DataHub ingestion
|
||||
- LOOKER_BASE_URL: The base url where your Looker assets are hosted (e.g. https://acryl.cloud.looker.com)
|
||||
- LOOKER_BASE_URL: The base url where your Looker assets are hosted (e.g. <https://acryl.cloud.looker.com>)
|
||||
- LOOKER_CLIENT_ID: A provisioned Looker Client ID
|
||||
- LOOKER_CLIENT_SECRET: A provisioned Looker Client Secret
|
||||
|
||||
@ -84,7 +87,7 @@ jobs:
|
||||
lookml-metadata-upload:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
@ -125,4 +128,4 @@ jobs:
|
||||
LOOKER_CLIENT_SECRET: ${{ secrets.LOOKER_CLIENT_SECRET }}
|
||||
```
|
||||
|
||||
If you want to ingest lookml using the **datahub** cli directly, read on for instructions and configuration details.
|
||||
If you want to ingest lookml using the **datahub** cli directly, read on for instructions and configuration details.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user