Revert "feat: adding gradle, pip cache via gh cache, docker cache via dockerhub (#4387)" (#4637)

This reverts commit b6de051d9a5240cefa1a7b217690cfe3a85a70f5.
This commit is contained in:
Dexter Lee 2022-04-11 16:00:16 -07:00 committed by GitHub
parent 80001d2206
commit 2a8ae27261
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 4 additions and 312 deletions

View File

@ -1,11 +1,5 @@
name: build & test
on:
workflow_dispatch:
inputs:
use_cache:
description: 'Whether to use cache'
required: true
default: 'true'
push:
branches:
- master
@ -34,17 +28,6 @@ jobs:
- uses: actions/setup-python@v2
with:
python-version: "3.6"
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: Gradle build (and test)
# there is some race condition in gradle build, which makes gradle never terminate in ~30% of the runs
# running build first without datahub-web-react:yarnBuild and then with it is 100% stable
@ -91,27 +74,6 @@ jobs:
python-version: "3.6"
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Setup Cache for Gradle
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-
baseprefix1-${{ runner.os }}-gradle-
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: Gradle build
run: ./gradlew build -x check -x docs-website:build
- name: Smoke test

View File

@ -68,16 +68,6 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.ORG_DOCKER_PASSWORD }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set env DOCKER_CACHE_TO
run: |
if [[ ${{ needs.setup.outputs.publish }} == 'true' ]]; then
echo "DOCKER_CACHE_TO=type=registry,ref=${{ steps.docker_meta.outputs.tags }}-buildcache,mode=max" >> "$GITHUB_ENV"
else
echo "DOCKER_CACHE_TO=type=inline" >> "$GITHUB_ENV"
fi
- name: Build and Push image
uses: docker/build-push-action@v2
with:
@ -85,5 +75,3 @@ jobs:
file: ./docker/datahub-upgrade/Dockerfile
tags: ${{ steps.docker_meta.outputs.tags }}
push: ${{ needs.setup.outputs.publish == 'true' }}
cache-from: type=registry,ref=acryldata/datahub-upgrade:head-buildcache
cache-to: ${{ env.DOCKER_CACHE_TO }}

View File

@ -73,4 +73,4 @@ jobs:
with:
context: ./metadata-ingestion/src/datahub/ingestion/source/feast_image
tags: ${{ steps.docker_meta.outputs.tags }}
push: ${{ needs.setup.outputs.publish == 'true' }}
push: ${{ needs.setup.outputs.publish == 'true' }}

View File

@ -71,14 +71,6 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Set env DOCKER_CACHE_TO
run: |
if [[ ${{ needs.setup.outputs.publish }} == 'true' ]]; then
echo "DOCKER_CACHE_TO=type=registry,ref=${{ steps.docker_meta.outputs.tags }}-buildcache,mode=max" >> "$GITHUB_ENV"
else
echo "DOCKER_CACHE_TO=type=inline" >> "$GITHUB_ENV"
fi
- name: Build and Push image
uses: docker/build-push-action@v2
with:
@ -87,5 +79,3 @@ jobs:
platforms: linux/amd64,linux/arm64
tags: ${{ steps.docker_meta.outputs.tags }}
push: ${{ needs.setup.outputs.publish == 'true' }}
cache-from: type=registry,ref=linkedin/datahub-frontend-react:head-buildcache
cache-to: ${{ env.DOCKER_CACHE_TO }}

View File

@ -71,14 +71,6 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Set env DOCKER_CACHE_TO
run: |
if [[ ${{ needs.setup.outputs.publish }} == 'true' ]]; then
echo "DOCKER_CACHE_TO=type=registry,ref=${{ steps.docker_meta.outputs.tags }}-buildcache,mode=max" >> "$GITHUB_ENV"
else
echo "DOCKER_CACHE_TO=type=inline" >> "$GITHUB_ENV"
fi
- name: Build and Push image
uses: docker/build-push-action@v2
with:
@ -87,5 +79,3 @@ jobs:
platforms: linux/amd64,linux/arm64
tags: ${{ steps.docker_meta.outputs.tags }}
push: ${{ needs.setup.outputs.publish == 'true' }}
cache-from: type=registry,ref=linkedin/datahub-gms:head-buildcache
cache-to: ${{ env.DOCKER_CACHE_TO }}

View File

@ -80,14 +80,6 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Set env DOCKER_CACHE_TO
run: |
if [[ ${{ needs.setup.outputs.publish }} == 'true' ]]; then
echo "DOCKER_CACHE_TO=type=registry,ref=${{ steps.docker_meta.outputs.tags }}-buildcache,mode=max" >> "$GITHUB_ENV"
else
echo "DOCKER_CACHE_TO=type=inline" >> "$GITHUB_ENV"
fi
- name: Build and Push image
uses: docker/build-push-action@v2
with:
@ -98,5 +90,3 @@ jobs:
push: ${{ needs.setup.outputs.publish == 'true' }}
build-args: |
RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
cache-from: type=registry,ref=linkedin/datahub-ingestion:head-buildcache
cache-to: ${{ env.DOCKER_CACHE_TO }}

View File

@ -67,16 +67,6 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set env DOCKER_CACHE_TO
run: |
if [[ ${{ needs.setup.outputs.publish }} == 'true' ]]; then
echo "DOCKER_CACHE_TO=type=registry,ref=${{ steps.docker_meta.outputs.tags }}-buildcache,mode=max" >> "$GITHUB_ENV"
else
echo "DOCKER_CACHE_TO=type=inline" >> "$GITHUB_ENV"
fi
- name: Build and Push image
uses: docker/build-push-action@v2
with:
@ -84,5 +74,3 @@ jobs:
file: ./docker/datahub-mae-consumer/Dockerfile
tags: ${{ steps.docker_meta.outputs.tags }}
push: ${{ needs.setup.outputs.publish == 'true' }}
cache-from: type=registry,ref=linkedin/datahub-mae-consumer:head-buildcache
cache-to: ${{ env.DOCKER_CACHE_TO }}

View File

@ -67,16 +67,6 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set env DOCKER_CACHE_TO
run: |
if [[ ${{ needs.setup.outputs.publish }} == 'true' ]]; then
echo "DOCKER_CACHE_TO=type=registry,ref=${{ steps.docker_meta.outputs.tags }}-buildcache,mode=max" >> "$GITHUB_ENV"
else
echo "DOCKER_CACHE_TO=type=inline" >> "$GITHUB_ENV"
fi
- name: Build and Push image
uses: docker/build-push-action@v2
with:
@ -84,5 +74,3 @@ jobs:
file: ./docker/datahub-mce-consumer/Dockerfile
tags: ${{ steps.docker_meta.outputs.tags }}
push: ${{ needs.setup.outputs.publish == 'true' }}
cache-from: type=registry,ref=linkedin/datahub-mce-consumer:head-buildcache
cache-to: ${{ env.DOCKER_CACHE_TO }}

View File

@ -65,16 +65,6 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.ORG_DOCKER_PASSWORD }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set env DOCKER_CACHE_TO
run: |
if [[ ${{ needs.setup.outputs.publish }} == 'true' ]]; then
echo "DOCKER_CACHE_TO=type=registry,ref=${{ steps.docker_meta.outputs.tags }}-buildcache,mode=max" >> "$GITHUB_ENV"
else
echo "DOCKER_CACHE_TO=type=inline" >> "$GITHUB_ENV"
fi
- name: Build and Push image
uses: docker/build-push-action@v2
with:
@ -82,5 +72,3 @@ jobs:
file: ./docker/mysql-setup/Dockerfile
tags: ${{ steps.docker_meta.outputs.tags }}
push: ${{ needs.setup.outputs.publish == 'true' }}
cache-from: type=registry,ref=acryldata/datahub-mysql-setup:head-buildcache
cache-to: ${{ env.DOCKER_CACHE_TO }}

View File

@ -65,16 +65,6 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.ORG_DOCKER_PASSWORD }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set env DOCKER_CACHE_TO
run: |
if [[ ${{ needs.setup.outputs.publish }} == 'true' ]]; then
echo "DOCKER_CACHE_TO=type=registry,ref=${{ steps.docker_meta.outputs.tags }}-buildcache,mode=max" >> "$GITHUB_ENV"
else
echo "DOCKER_CACHE_TO=type=inline" >> "$GITHUB_ENV"
fi
- name: Build and Push image
uses: docker/build-push-action@v2
with:
@ -82,5 +72,3 @@ jobs:
file: ./docker/postgres-setup/Dockerfile
tags: ${{ steps.docker_meta.outputs.tags }}
push: ${{ needs.setup.outputs.publish == 'true' }}
cache-from: type=registry,ref=acryldata/datahub-postgres-setup:head-buildcache
cache-to: ${{ env.DOCKER_CACHE_TO }}

View File

@ -1,12 +1,6 @@
name: documentation
on:
workflow_dispatch:
inputs:
use_cache:
description: 'Whether to use cache'
required: true
default: 'true'
pull_request:
branches:
- master
@ -30,27 +24,6 @@ jobs:
python-version: 3.9.9
- name: Install Python dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Setup Cache for Gradle
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-
baseprefix1-${{ runner.os }}-gradle-
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: Build Docs
run: |
./gradlew --info docs-website:build

View File

@ -1,11 +1,5 @@
name: metadata ingestion slow integration tests
on:
workflow_dispatch:
inputs:
use_cache:
description: 'Whether to use cache'
required: true
default: 'true'
push:
branches:
- master
@ -34,27 +28,6 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Setup Cache for Gradle
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-
baseprefix1-${{ runner.os }}-gradle-
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: Run metadata-ingestion slow integration tests
run: ./gradlew :metadata-ingestion:testSlowIntegration
- uses: actions/upload-artifact@v2

View File

@ -1,11 +1,5 @@
name: metadata ingestion
on:
workflow_dispatch:
inputs:
use_cache:
description: 'Whether to use cache'
required: true
default: 'true'
push:
branches:
- master
@ -41,27 +35,6 @@ jobs:
hadoop-version: '3.2'
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Setup Cache for Gradle
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-
baseprefix1-${{ runner.os }}-gradle-
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: Run metadata-ingestion tests
run: ./gradlew :metadata-ingestion:build :metadata-ingestion:check
- uses: actions/upload-artifact@v2
@ -90,27 +63,6 @@ jobs:
with:
spark-version: '3.0.3'
hadoop-version: '3.2'
- name: Setup Cache for Gradle
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-
baseprefix1-${{ runner.os }}-gradle-
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh && python -m pip install --upgrade pip && pip install tox tox-gh-actions
- name: Codegen

View File

@ -1,11 +1,5 @@
name: metadata model generate
on:
workflow_dispatch:
inputs:
use_cache:
description: 'Whether to use cache'
required: true
default: 'true'
push:
branches:
- master
@ -38,27 +32,6 @@ jobs:
python-version: "3.9.9"
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Setup Cache for Gradle
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-
baseprefix1-${{ runner.os }}-gradle-
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: Run model generation
run: ./gradlew :metadata-models:build
- name: Upload metadata to DataHub

View File

@ -1,13 +1,6 @@
name: Publish Datahub Client
on:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
inputs:
use_cache:
description: 'Whether to use cache'
required: true
default: 'true'
workflow_run:
workflows: ["build & test"]
types:
@ -16,6 +9,9 @@ on:
release:
types: [published, edited]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
jobs:
check-secret:
@ -49,27 +45,6 @@ jobs:
run: |
git remote add upstream https://github.com/datahub-project/datahub.git
git fetch upstream --tags --force
- name: Setup Cache for Gradle
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-
baseprefix1-${{ runner.os }}-gradle-
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: publish datahub-client jar
env:
RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}

View File

@ -2,11 +2,6 @@ name: spark smoke test
on:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
inputs:
use_cache:
description: 'Whether to use cache'
required: true
default: 'true'
push:
branches:
- master
@ -38,27 +33,6 @@ jobs:
python-version: "3.6"
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Setup Cache for Gradle
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-
baseprefix1-${{ runner.os }}-gradle-
- name: Setup Cache for pip
uses: actions/cache@v2
if: ${{ github.event.inputs.use_cache != 'false' }}
with:
path: ~/.cache/pip
key: baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-${{ github.job }}
restore-keys: |
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/setup.py') }}-
baseprefix1-${{ runner.os }}-pip-${{ matrix.python-version }}-
baseprefix1-${{ runner.os }}-pip-
- name: Gradle build
run: ./gradlew build -x check -x docs-website:build -x test
- name: Smoke test