chore(ci): update actions versions addressing warnings in builds. (#6308)

This commit is contained in:
Kevin Neville 2022-10-29 02:10:30 +02:00 committed by GitHub
parent dfc20b3d40
commit 3eae087a30
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 172 additions and 161 deletions

View File

@ -1,15 +1,15 @@
ingestion:
- 'metadata-ingestion/**/*'
- "metadata-ingestion/**/*"
devops:
- 'docker/**/*'
- '.github/**/*'
- "docker/**/*"
- ".github/**/*"
product:
- 'datahub-web-react/**/*'
- 'datahub-frontend/**/*'
- 'datahub-graphql-core/**/*'
- 'metadata-io/**/*'
- "datahub-web-react/**/*"
- "datahub-frontend/**/*"
- "datahub-graphql-core/**/*"
- "metadata-io/**/*"
docs:
- 'docs/**/*'
- "docs/**/*"

View File

@ -1,8 +1,8 @@
## Checklist
- [ ] The PR conforms to DataHub's [Contributing Guideline](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md) (particularly [Commit Message Format](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md#commit-message-format))
- [ ] Links to related issues (if applicable)
- [ ] Tests for the changes have been added/updated (if applicable)
- [ ] Docs related to the changes have been added/updated (if applicable). If a new feature has been added a Usage Guide has been added for the same.
- [ ] For any breaking change/potential downtime/deprecation/big changes an entry has been made in [Updating DataHub](https://github.com/datahub-project/datahub/blob/master/docs/how/updating-datahub.md)
- [ ] For any breaking change/potential downtime/deprecation/big changes an entry has been made in [Updating DataHub](https://github.com/datahub-project/datahub/blob/master/docs/how/updating-datahub.md)

View File

@ -24,20 +24,21 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up JDK 11
uses: actions/setup-java@v1
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
- uses: actions/setup-python@v2
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- name: Gradle build (and test)
run: |
./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test
- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (build)
@ -58,8 +59,8 @@ jobs:
quickstart-compose-validation:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- name: Quickstart Compose Validation
@ -69,7 +70,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Upload
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: Event File
path: ${{ github.event_path }}

View File

@ -23,18 +23,18 @@ concurrency:
cancel-in-progress: true
jobs:
check_jars:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up JDK 11
uses: actions/setup-java@v1
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
- uses: actions/setup-python@v2
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- name: check datahub-client jar

View File

@ -9,7 +9,6 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test-quickstart:
strategy:
@ -18,30 +17,30 @@ jobs:
os: [ubuntu-20.04]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Install acryl-datahub
run: |
pip install --upgrade acryl-datahub
datahub version
python -c "import platform; print(platform.platform())"
- name: Run quickstart
run: |
datahub docker quickstart
- name: Ingest sample data
run: |
datahub docker ingest-sample-data
- name: See status
run: |
docker ps -a && datahub docker check
- name: store logs
if: failure()
run: |
docker logs datahub-gms >& quickstart-gms.log
- name: Upload logs
uses: actions/upload-artifact@v2
if: failure()
with:
name: docker-quickstart-logs-${{ matrix.os }}
path: "*.log"
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install acryl-datahub
run: |
pip install --upgrade acryl-datahub
datahub version
python -c "import platform; print(platform.platform())"
- name: Run quickstart
run: |
datahub docker quickstart
- name: Ingest sample data
run: |
datahub docker ingest-sample-data
- name: See status
run: |
docker ps -a && datahub docker check
- name: store logs
if: failure()
run: |
docker logs datahub-gms >& quickstart-gms.log
- name: Upload logs
uses: actions/upload-artifact@v3
if: failure()
with:
name: docker-quickstart-logs-${{ matrix.os }}
path: "*.log"

View File

@ -10,7 +10,7 @@ jobs:
issues: write
pull-requests: write
steps:
- uses: actions/stale@v5
- uses: actions/stale@v6
with:
ascending: true
operations-per-run: 100

View File

@ -12,7 +12,7 @@ jobs:
timeout-minutes: 10
steps:
- name: Remove old artifacts
uses: c-hive/gha-remove-artifacts@v1
with:
age: '1 second'
- name: Remove old artifacts
uses: c-hive/gha-remove-artifacts@v1
with:
age: "1 second"

View File

@ -4,14 +4,14 @@ on:
branches:
- master
paths:
- 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**'
- '.github/workflows/docker-feast-source.yml'
- "metadata-ingestion/src/datahub/ingestion/source/feast_image/**"
- ".github/workflows/docker-feast-source.yml"
pull_request:
branches:
- master
paths:
- 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**'
- '.github/workflows/docker-feast-source.yml'
- "metadata-ingestion/src/datahub/ingestion/source/feast_image/**"
- ".github/workflows/docker-feast-source.yml"
release:
types: [published, edited]
@ -27,7 +27,7 @@ jobs:
publish: ${{ steps.publish.outputs.publish }}
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Compute Tag
id: tag
run: |
@ -51,7 +51,7 @@ jobs:
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Docker meta

View File

@ -15,13 +15,12 @@ concurrency:
cancel-in-progress: true
jobs:
build-base:
name: Build and Push Docker Image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up QEMU

View File

@ -27,7 +27,7 @@ jobs:
publish: ${{ steps.publish.outputs.publish }}
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Compute Tag
id: tag
run: |
@ -55,7 +55,7 @@ jobs:
image_name: ${{ env.IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push

View File

@ -30,7 +30,7 @@ jobs:
python_release_version: ${{ steps.python_release_version.outputs.release_version }}
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Compute Tag
id: tag
run: |
@ -60,7 +60,7 @@ jobs:
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Docker meta

View File

@ -4,14 +4,14 @@ on:
branches:
- master
paths:
- 'docker/postgres-setup/**'
- '.github/workflows/docker-postgres-setup.yml'
- "docker/postgres-setup/**"
- ".github/workflows/docker-postgres-setup.yml"
pull_request:
branches:
- master
paths:
- 'docker/postgres-setup/**'
- '.github/workflows/docker-postgres-setup.yml'
- "docker/postgres-setup/**"
- ".github/workflows/docker-postgres-setup.yml"
release:
types: [published, edited]
@ -27,7 +27,7 @@ jobs:
publish: ${{ steps.publish.outputs.publish }}
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Compute Tag
id: tag
run: |
@ -50,7 +50,7 @@ jobs:
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Docker meta

View File

@ -4,14 +4,14 @@ on:
branches:
- master
paths-ignore:
- 'docs/**'
- '**.md'
- "docs/**"
- "**.md"
pull_request:
branches:
- master
paths-ignore:
- 'docs/**'
- '**.md'
- "docs/**"
- "**.md"
release:
types: [published, edited]
@ -23,15 +23,14 @@ concurrency:
cancel-in-progress: true
env:
DATAHUB_GMS_IMAGE: 'linkedin/datahub-gms'
DATAHUB_FRONTEND_IMAGE: 'linkedin/datahub-frontend-react'
DATAHUB_MAE_CONSUMER_IMAGE: 'linkedin/datahub-mae-consumer'
DATAHUB_MCE_CONSUMER_IMAGE: 'linkedin/datahub-mce-consumer'
DATAHUB_KAFKA_SETUP_IMAGE: 'linkedin/datahub-kafka-setup'
DATAHUB_ELASTIC_SETUP_IMAGE: 'linkedin/datahub-elasticsearch-setup'
DATAHUB_MYSQL_SETUP_IMAGE: 'acryldata/datahub-mysql-setup'
DATAHUB_UPGRADE_IMAGE: 'acryldata/datahub-upgrade'
DATAHUB_GMS_IMAGE: "linkedin/datahub-gms"
DATAHUB_FRONTEND_IMAGE: "linkedin/datahub-frontend-react"
DATAHUB_MAE_CONSUMER_IMAGE: "linkedin/datahub-mae-consumer"
DATAHUB_MCE_CONSUMER_IMAGE: "linkedin/datahub-mce-consumer"
DATAHUB_KAFKA_SETUP_IMAGE: "linkedin/datahub-kafka-setup"
DATAHUB_ELASTIC_SETUP_IMAGE: "linkedin/datahub-elasticsearch-setup"
DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup"
DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade"
jobs:
setup:
@ -42,7 +41,7 @@ jobs:
publish: ${{ steps.publish.outputs.publish }}
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Compute Tag
id: tag
run: |
@ -71,7 +70,7 @@ jobs:
image_name: ${{ env.DATAHUB_GMS_IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
@ -100,10 +99,10 @@ jobs:
uses: aquasecurity/trivy-action@master
with:
image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: 'table'
format: "table"
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
vuln-type: "os,library"
severity: "CRITICAL,HIGH"
mae_consumer_build:
name: Build and Push DataHub MAE Consumer Docker Image
@ -114,7 +113,7 @@ jobs:
image_name: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
@ -132,7 +131,7 @@ jobs:
mae_consumer_scan:
name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup,mae_consumer_build]
needs: [setup, mae_consumer_build]
steps:
- name: Download image
uses: ishworkh/docker-image-artifact-download@v1
@ -143,10 +142,10 @@ jobs:
uses: aquasecurity/trivy-action@master
with:
image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: 'table'
format: "table"
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
vuln-type: "os,library"
severity: "CRITICAL,HIGH"
mce_consumer_build:
name: Build and Push DataHub MCE Consumer Docker Image
@ -157,7 +156,7 @@ jobs:
image_name: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
@ -175,7 +174,7 @@ jobs:
mce_consumer_scan:
name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup,mce_consumer_build]
needs: [setup, mce_consumer_build]
steps:
- name: Download image
uses: ishworkh/docker-image-artifact-download@v1
@ -186,10 +185,10 @@ jobs:
uses: aquasecurity/trivy-action@master
with:
image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: 'table'
format: "table"
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
vuln-type: "os,library"
severity: "CRITICAL,HIGH"
datahub_upgrade_build:
name: Build and Push DataHub Upgrade Docker Image
@ -200,7 +199,7 @@ jobs:
image_name: ${{ env.DATAHUB_UPGRADE_IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
@ -229,10 +228,10 @@ jobs:
uses: aquasecurity/trivy-action@master
with:
image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: 'table'
format: "table"
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
vuln-type: "os,library"
severity: "CRITICAL,HIGH"
frontend_build:
name: Build and Push DataHub Frontend Docker Image
@ -243,7 +242,7 @@ jobs:
image_name: ${{ env.DATAHUB_FRONTEND_IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
@ -272,10 +271,10 @@ jobs:
uses: aquasecurity/trivy-action@master
with:
image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: 'table'
format: "table"
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
vuln-type: "os,library"
severity: "CRITICAL,HIGH"
kafka_setup_build:
name: Build and Push DataHub Kafka Setup Docker Image
@ -286,7 +285,7 @@ jobs:
image_name: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
@ -311,7 +310,7 @@ jobs:
image_name: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
@ -336,7 +335,7 @@ jobs:
image_name: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
@ -355,15 +354,26 @@ jobs:
smoke_test:
name: Run Smoke Tests
runs-on: ubuntu-latest
needs: [setup,gms_build,frontend_build,kafka_setup_build,mysql_setup_build,elasticsearch_setup_build,mae_consumer_build,mce_consumer_build]
needs:
[
setup,
gms_build,
frontend_build,
kafka_setup_build,
mysql_setup_build,
elasticsearch_setup_build,
mae_consumer_build,
mce_consumer_build,
]
steps:
- name: Check out the repo
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v1
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
- uses: actions/setup-python@v2
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- name: Install dependencies
@ -380,7 +390,7 @@ jobs:
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Download Kafka Setup image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
@ -419,18 +429,18 @@ jobs:
docker ps -a
docker logs datahub-gms >& gms.log
- name: Upload logs
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
if: failure()
with:
name: docker logs
path: "*.log"
- name: Upload screenshots
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
if: failure()
with:
name: cypress-snapshots
path: smoke-test/tests/cypress/cypress/screenshots/
- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (smoke tests)
@ -445,4 +455,3 @@ jobs:
job-status: ${{ job.status }}
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
channel: github-activities

View File

@ -18,12 +18,13 @@ jobs:
gh-pages:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v1
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
- uses: actions/setup-python@v2
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install Python dependencies

View File

@ -29,15 +29,15 @@ jobs:
matrix:
python-version: ["3.7", "3.10.6"]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Run metadata-ingestion tests
run: ./gradlew :metadata-ingestion:build :metadata-ingestion:testQuick :metadata-ingestion:check
- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (metadata ingestion ${{ matrix.python-version }} testQuick)
@ -64,8 +64,8 @@ jobs:
]
fail-fast: false
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: vemonet/setup-spark@v1 # spark is required for pyspark+pydeequ data lake profiling
@ -79,7 +79,7 @@ jobs:
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion/venv/bin/activate && pip freeze
- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (metadata ingestion ${{ matrix.python-version }})
@ -92,7 +92,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Upload
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: Event File
path: ${{ github.event_path }}

View File

@ -28,12 +28,13 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v1
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
- uses: actions/setup-python@v2
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- name: Gradle build (and test)
@ -42,7 +43,7 @@ jobs:
# datahub-frontend:unzipAssets depends on datahub-web-react:yarnBuild but gradle does not know about it
run: |
./gradlew :metadata-io:test
- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (metadata-io)
@ -57,7 +58,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Upload
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: Event File
path: ${{ github.event_path }}

View File

@ -30,8 +30,8 @@ jobs:
runs-on: ubuntu-latest
needs: setup
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install dependencies
@ -44,5 +44,3 @@ jobs:
DATAHUB_SERVER: ${{ secrets.DataHubServer }}
DATAHUB_TOKEN: ${{ secrets.DataHubToken }}
run: ./gradlew :metadata-ingestion:modelDocUpload

View File

@ -10,7 +10,7 @@ jobs:
pull-requests: write
runs-on: ubuntu-latest
steps:
- uses: actions/labeler@v4
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
configuration-path: '.github/pr-labeler-config.yml'
- uses: actions/labeler@v4
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
configuration-path: ".github/pr-labeler-config.yml"

View File

@ -36,7 +36,7 @@ jobs:
tag: ${{ steps.tag.outputs.tag }}
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Compute Tag
id: tag
run: |
@ -50,14 +50,15 @@ jobs:
needs: ["check-secret", "setup"]
if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up JDK 11
uses: actions/setup-java@v1
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
- uses: actions/setup-python@v2
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- name: checkout upstream repo

View File

@ -27,12 +27,13 @@ jobs:
spark-smoke-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v1
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
- uses: actions/setup-python@v2
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- name: Install dependencies
@ -41,7 +42,7 @@ jobs:
run: ./gradlew build -x check -x docs-website:build -x test
- name: Smoke test
run: ./gradlew metadata-integration:java:spark-lineage:integrationTest
- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (smoke tests)
@ -56,4 +57,3 @@ jobs:
job-status: ${{ job.status }}
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
channel: github-activities

View File

@ -36,4 +36,3 @@ jobs:
event_file: artifacts/Event File/event.json
event_name: ${{ github.event.workflow_run.event }}
files: "artifacts/**/*.xml"

View File

@ -2,9 +2,10 @@
#### [Recommended] Create a GitHub Deploy Key
To use LookML ingestion through the UI, or automate github checkout through the cli, you must set up a GitHub deploy key for your Looker GitHub repository. Read [this](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) document for how to set up deploy keys for your Looker git repo.
To use LookML ingestion through the UI, or automate github checkout through the cli, you must set up a GitHub deploy key for your Looker GitHub repository. Read [this](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) document for how to set up deploy keys for your Looker git repo.
In a nutshell, there are three steps:
In a nutshell, there are three steps:
1. Generate a private-public ssh key pair. This will typically generate two files, e.g. looker_datahub_deploy_key (this is the private key) and looker_datahub_deploy_key.pub (this is the public key)
![Image](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/gitssh/ssh-key-generation.png)
@ -16,18 +17,19 @@ In a nutshell, there are three steps:
#### [Optional] Create an API key with admin privileges
See the [Looker authentication docs](https://docs.looker.com/reference/api-and-integration/api-auth#authentication_with_an_sdk) for the steps to create a client ID and secret.
You need to ensure that the API key is attached to a user that has Admin privileges.
You need to ensure that the API key is attached to a user that has Admin privileges.
If that is not possible, read the configuration section and provide an offline specification of the `connection_to_platform_map` and the `project_name`.
### Ingestion Options
You have 3 options for controlling where your ingestion of LookML is run.
You have 3 options for controlling where your ingestion of LookML is run.
- The DataHub UI (recommended for the easiest out-of-the-box experience)
- As a GitHub Action (recommended to ensure that you have the freshest metadata pushed on change)
- Using the CLI (scheduled via an orchestrator like Airflow)
Read on to learn more about these options.
Read on to learn more about these options.
### UI-based Ingestion [Recommended for ease of use]
@ -60,9 +62,10 @@ The following sample GitHub action file can be modified to emit LookML metadata
Drop this file into your `.github/workflows` directory inside your Looker GitHub repo.
You need to set up the following secrets in your GitHub repository to get this workflow to work:
- DATAHUB_GMS_HOST: The endpoint where your DataHub host is running
- DATAHUB_TOKEN: An authentication token provisioned for DataHub ingestion
- LOOKER_BASE_URL: The base url where your Looker assets are hosted (e.g. https://acryl.cloud.looker.com)
- LOOKER_BASE_URL: The base url where your Looker assets are hosted (e.g. <https://acryl.cloud.looker.com>)
- LOOKER_CLIENT_ID: A provisioned Looker Client ID
- LOOKER_CLIENT_SECRET: A provisioned Looker Client Secret
@ -84,7 +87,7 @@ jobs:
lookml-metadata-upload:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.9'
@ -125,4 +128,4 @@ jobs:
LOOKER_CLIENT_SECRET: ${{ secrets.LOOKER_CLIENT_SECRET }}
```
If you want to ingest lookml using the **datahub** cli directly, read on for instructions and configuration details.
If you want to ingest lookml using the **datahub** cli directly, read on for instructions and configuration details.