From 8cc91c3405638a2e606a11324f2fa8fa6fa14f14 Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Tue, 17 May 2022 21:38:57 +0530 Subject: [PATCH] Modified Ingestion to support 3.6 and later (#5003) --- .github/workflows/py-tests-3_10.yml | 68 +++++++++++++++++++ .github/workflows/py-tests-3_6.yml | 68 +++++++++++++++++++ .github/workflows/py-tests-3_7.yml | 68 +++++++++++++++++++ .github/workflows/py-tests-3_8.yml | 68 +++++++++++++++++++ .../{py-tests.yml => py-tests-3_9.yml} | 0 ingestion/setup.py | 2 +- 6 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/py-tests-3_10.yml create mode 100644 .github/workflows/py-tests-3_6.yml create mode 100644 .github/workflows/py-tests-3_7.yml create mode 100644 .github/workflows/py-tests-3_8.yml rename .github/workflows/{py-tests.yml => py-tests-3_9.yml} (100%) diff --git a/.github/workflows/py-tests-3_10.yml b/.github/workflows/py-tests-3_10.yml new file mode 100644 index 00000000000..debfae2d166 --- /dev/null +++ b/.github/workflows/py-tests-3_10.yml @@ -0,0 +1,68 @@ + +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: py-tests +on: + push: + branches: [main] + paths: + - ingestion/** + - catalog-rest-service/** + pull_request_target: + branches: [main] + paths: + - ingestion/** + - catalog-rest-service/** + +concurrency: + group: py-tests-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +jobs: + py-run-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: 3.10 + + - name: Install Ubuntu dependencies + run: | + sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv + + - name: Install open-metadata dependencies + run: | + python3 -m venv env + source env/bin/activate + make install_test install_dev generate install_all + + - name: Start Server and Ingest Sample Data + env: + INGESTION_DEPENDENCY: "airflow-container,sample-data,elasticsearch" + run: ./docker/run_local_docker.sh + timeout-minutes: 30 + + - name: Run Python Tests & record coverage + run: | + source env/bin/activate + make coverage diff --git a/.github/workflows/py-tests-3_6.yml b/.github/workflows/py-tests-3_6.yml new file mode 100644 index 00000000000..42e52f0c99f --- /dev/null +++ b/.github/workflows/py-tests-3_6.yml @@ -0,0 +1,68 @@ + +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: py-tests +on: + push: + branches: [main] + paths: + - ingestion/** + - catalog-rest-service/** + pull_request_target: + branches: [main] + paths: + - ingestion/** + - catalog-rest-service/** + +concurrency: + group: py-tests-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +jobs: + py-run-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Set up Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 + + - name: Install Ubuntu dependencies + run: | + sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv + + - name: Install open-metadata dependencies + run: | + python3 -m venv env + source env/bin/activate + make install_test install_dev generate install_all + + - name: Start Server and Ingest Sample Data + env: + INGESTION_DEPENDENCY: "airflow-container,sample-data,elasticsearch" + run: ./docker/run_local_docker.sh + timeout-minutes: 30 + + - name: Run Python Tests & record coverage + run: | + source env/bin/activate + make coverage diff --git a/.github/workflows/py-tests-3_7.yml b/.github/workflows/py-tests-3_7.yml new file mode 100644 index 00000000000..aa613f3900a --- /dev/null +++ b/.github/workflows/py-tests-3_7.yml @@ -0,0 +1,68 @@ + +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: py-tests +on: + push: + branches: [main] + paths: + - ingestion/** + - catalog-rest-service/** + pull_request_target: + branches: [main] + paths: + - ingestion/** + - catalog-rest-service/** + +concurrency: + group: py-tests-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +jobs: + py-run-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install Ubuntu dependencies + run: | + sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv + + - name: Install open-metadata dependencies + run: | + python3 -m venv env + source env/bin/activate + make install_test install_dev generate install_all + + - name: Start Server and Ingest Sample Data + env: + INGESTION_DEPENDENCY: "airflow-container,sample-data,elasticsearch" + run: ./docker/run_local_docker.sh + timeout-minutes: 30 + + - name: Run Python Tests & record coverage + run: | + source env/bin/activate + make coverage diff --git a/.github/workflows/py-tests-3_8.yml b/.github/workflows/py-tests-3_8.yml new file mode 100644 index 00000000000..456d9b5b1c5 --- /dev/null +++ b/.github/workflows/py-tests-3_8.yml @@ -0,0 +1,68 @@ + +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: py-tests +on: + push: + branches: [main] + paths: + - ingestion/** + - catalog-rest-service/** + pull_request_target: + branches: [main] + paths: + - ingestion/** + - catalog-rest-service/** + +concurrency: + group: py-tests-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +jobs: + py-run-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install Ubuntu dependencies + run: | + sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv + + - name: Install open-metadata dependencies + run: | + python3 -m venv env + source env/bin/activate + make install_test install_dev generate install_all + + - name: Start Server and Ingest Sample Data + env: + INGESTION_DEPENDENCY: "airflow-container,sample-data,elasticsearch" + run: ./docker/run_local_docker.sh + timeout-minutes: 30 + + - name: Run Python Tests & record coverage + run: | + source env/bin/activate + make coverage \ No newline at end of file diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests-3_9.yml similarity index 100% rename from .github/workflows/py-tests.yml rename to .github/workflows/py-tests-3_9.yml diff --git a/ingestion/setup.py b/ingestion/setup.py index 67e41d882d2..5645202dc2f 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -169,7 +169,7 @@ setup( description="Ingestion Framework for OpenMetadata", long_description=get_long_description(), long_description_content_type="text/markdown", - python_requires=">=3.8", + python_requires=">=3.6", options={"build_exe": build_options}, package_dir={"": "src"}, zip_safe=False,