From d0bc3c55db129429de156a0d956217a423268bc0 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 12 Feb 2021 20:17:25 -0800 Subject: [PATCH] Setup CI --- .github/workflows/build-and-test.yml | 2 + .../.github/workflows/docker-ingest.yml | 35 ----------------- .../.github/workflows/python-package.yml | 38 ------------------- metadata-ingestion/.gitignore | 2 +- metadata-ingestion/README.md | 2 + metadata-ingestion/scripts/ci.sh | 15 ++++++++ .../integration/sql_server/test_sql_server.py | 4 +- 7 files changed, 22 insertions(+), 76 deletions(-) delete mode 100644 metadata-ingestion/.github/workflows/docker-ingest.yml delete mode 100644 metadata-ingestion/.github/workflows/python-package.yml create mode 100755 metadata-ingestion/scripts/ci.sh diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 4b905db977..a61c8f7bad 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -26,6 +26,8 @@ jobs: java-version: 1.8 - name: Gradle build (and test) run: ./gradlew build + - name: Python ingest framework tests + run: cd metadata-ingestion && ./scripts/ci.sh - name: Slack failure notification if: failure() && github.event_name == 'push' uses: kpritam/slack-job-status-action@v1 diff --git a/metadata-ingestion/.github/workflows/docker-ingest.yml b/metadata-ingestion/.github/workflows/docker-ingest.yml deleted file mode 100644 index 768ae999cb..0000000000 --- a/metadata-ingestion/.github/workflows/docker-ingest.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: datahub-ingest docker -on: - push: - branches: - - main - paths-ignore: - - 'docs/**' - - '**.md' - pull_request: - branches: - - main - paths: - - 'docker/**' - - '.github/workflows/docker-ingest.yml' - paths_ignore: - - '**.md' - - '**.env' - release: - types: [published, edited] - -jobs: - push_to_registries: - name: Push Docker image to repo - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Push to Docker Hub - uses: docker/build-push-action@v1 - with: - dockerfile: ./docker/Dockerfile - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: acryldata/datahub-ingest - tag_with_ref: true diff --git a/metadata-ingestion/.github/workflows/python-package.yml b/metadata-ingestion/.github/workflows/python-package.yml deleted file mode 100644 index 00e87bace3..0000000000 --- a/metadata-ingestion/.github/workflows/python-package.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Python package - -on: [push] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -r test_requirements.txt - - name: Check formatting with black - run: | - black --check --exclude 'gometa/metadata' -S -t py36 src tests - - name: Lint with flake8 - run: | - flake8 src tests --count --statistics - - name: Check import sorting - run: | - isort src tests --check-only - - name: Check with mypy - run: | - mypy -p gometa - - name: Tests with pytest - run: | - pytest diff --git a/metadata-ingestion/.gitignore b/metadata-ingestion/.gitignore index 0104dfea7f..5924c1690d 100644 --- a/metadata-ingestion/.gitignore +++ b/metadata-ingestion/.gitignore @@ -1,7 +1,7 @@ .envrc .vscode/ output -src/gometa/metadata +src/gometa/metadata/ pvenv36/ # Byte-compiled / optimized / DLL files diff --git a/metadata-ingestion/README.md b/metadata-ingestion/README.md index 231edd211f..aae4016cad 100644 --- a/metadata-ingestion/README.md +++ b/metadata-ingestion/README.md @@ -4,6 +4,7 @@ ### Pre-Requisites - Python 3.6+ +- Must have already run `./gradlew build` in the datahub root directory. - On MacOS: `brew install librdkafka` - On Debian/Ubuntu: `sudo apt install librdkafka-dev python3-dev python3-venv` @@ -12,6 +13,7 @@ python3 -m venv venv source venv/bin/activate pip install -e . +./scripts/codegen.sh ``` ### Usage - from source diff --git a/metadata-ingestion/scripts/ci.sh b/metadata-ingestion/scripts/ci.sh new file mode 100755 index 0000000000..2c88f6cfc1 --- /dev/null +++ b/metadata-ingestion/scripts/ci.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euxo pipefail + +python -m pip install --upgrade pip +pip install -e . +pip install -r test_requirements.txt + +./scripts/codegen.sh + +black --check --exclude 'gometa/metadata' -S -t py36 src tests +isort --check-only src tests +flake8 --count --statistics src tests +mypy -p gometa + +pytest diff --git a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py index 20028494be..c28aee2067 100644 --- a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py +++ b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py @@ -12,8 +12,8 @@ def test_ingest(sql_server, pytestconfig): config_file = os.path.join( str(pytestconfig.rootdir), "tests/integration/sql_server", "mssql_to_file.yml" ) - # delete the output directory. TODO: move to a better way to create an output test fixture - os.system("rm -rf output") ingest_command = f'gometa-ingest -c {config_file}' ret = os.system(ingest_command) assert ret == 0 + # TODO: move to a better way to create an output test fixture + os.system("rm ./mssql_mces.json")