Setup CI

.github/workflows/build-and-test.yml

@@ -26,6 +26,8 @@ jobs:
           java-version: 1.8
       - name: Gradle build (and test)
         run: ./gradlew build
+      - name: Python ingest framework tests
+        run: cd metadata-ingestion && ./scripts/ci.sh
       - name: Slack failure notification
         if: failure() && github.event_name == 'push'
         uses: kpritam/slack-job-status-action@v1

metadata-ingestion/.github/workflows/docker-ingest.yml

@@ -1,35 +0,0 @@
-name: datahub-ingest docker
-on:
-  push:
-    branches:
-      - main
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
-  pull_request:
-    branches:
-      - main
-    paths:
-      - 'docker/**'
-      - '.github/workflows/docker-ingest.yml'
-    paths_ignore:
-      - '**.md'
-      - '**.env'
-  release:
-    types: [published, edited]
-
-jobs:
-  push_to_registries:
-    name: Push Docker image to repo
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2
-      - name: Push to Docker Hub
-        uses: docker/build-push-action@v1
-        with:
-            dockerfile: ./docker/Dockerfile
-            username: ${{ secrets.DOCKER_USERNAME }}
-            password: ${{ secrets.DOCKER_PASSWORD }}
-            repository: acryldata/datahub-ingest
-            tag_with_ref: true

metadata-ingestion/.github/workflows/python-package.yml

@@ -1,38 +0,0 @@
-name: Python package
-
-on: [push]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e .
-        pip install -r test_requirements.txt
-    - name: Check formatting with black
-      run: |
-        black --check --exclude 'gometa/metadata' -S -t py36 src tests
-    - name: Lint with flake8
-      run: |
-        flake8 src tests --count --statistics
-    - name: Check import sorting
-      run: |
-        isort src tests --check-only
-    - name: Check with mypy
-      run: |
-        mypy -p gometa
-    - name: Tests with pytest
-      run: |
-        pytest

metadata-ingestion/.gitignore

@@ -1,7 +1,7 @@
 .envrc
 .vscode/
 output
-src/gometa/metadata
+src/gometa/metadata/
 pvenv36/
 
 # Byte-compiled / optimized / DLL files

metadata-ingestion/README.md

@@ -4,6 +4,7 @@
 
 ### Pre-Requisites
 - Python 3.6+
+- Must have already run `./gradlew build` in the datahub root directory.
 - On MacOS: `brew install librdkafka`
 - On Debian/Ubuntu: `sudo apt install librdkafka-dev python3-dev python3-venv`
 
@@ -12,6 +13,7 @@
 python3 -m venv venv
 source venv/bin/activate
 pip install -e .
+./scripts/codegen.sh
 ```
 
 ### Usage - from source

metadata-ingestion/scripts/ci.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -euxo pipefail
+
+python -m pip install --upgrade pip
+pip install -e .
+pip install -r test_requirements.txt
+
+./scripts/codegen.sh
+
+black --check --exclude 'gometa/metadata' -S -t py36 src tests
+isort --check-only src tests
+flake8 --count --statistics src tests
+mypy -p gometa
+
+pytest

metadata-ingestion/tests/integration/sql_server/test_sql_server.py

@@ -12,8 +12,8 @@ def test_ingest(sql_server, pytestconfig):
     config_file = os.path.join(
         str(pytestconfig.rootdir), "tests/integration/sql_server", "mssql_to_file.yml"
     )
-    # delete the output directory. TODO: move to a better way to create an output test fixture
-    os.system("rm -rf output")
     ingest_command = f'gometa-ingest -c {config_file}'
     ret = os.system(ingest_command)
     assert ret == 0
+    # TODO: move to a better way to create an output test fixture
+    os.system("rm ./mssql_mces.json")