diff --git a/.github/scripts/check_python_package.py b/.github/scripts/check_python_package.py new file mode 100644 index 0000000000..f1f3005691 --- /dev/null +++ b/.github/scripts/check_python_package.py @@ -0,0 +1,18 @@ +import setuptools + +folders = ["./smoke-test/tests"] + +for folder in folders: + print(f"Checking folder {folder}") + a = [i for i in setuptools.find_packages(folder) if "cypress" not in i] + b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i] + + in_a_not_b = set(a) - set(b) + in_b_not_a = set(b) - set(a) + + assert ( + len(in_a_not_b) == 0 + ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}" + assert ( + len(in_b_not_a) == 0 + ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}" diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 24490ef83a..ae323181a5 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -37,8 +37,29 @@ env: DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion" jobs: + check_lint: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: hsheth2/sane-checkout-action@v1 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: 17 + - name: Ensure packages are correct + run: | + python ./.github/scripts/check_python_package.py + - name: Run lint on smoke test + run: | + ./gradlew :smoke-test:lint setup: runs-on: ubuntu-latest + needs: check_lint outputs: tag: ${{ steps.tag.outputs.tag }} slim_tag: ${{ steps.tag.outputs.slim_tag }} @@ -64,18 +85,6 @@ jobs: steps: - name: Check out the repo uses: hsheth2/sane-checkout-action@v1 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: 17 - - name: Run lint on smoke test - run: | - ./gradlew :smoke-test:lint - name: Compute Tag id: tag run: | diff --git a/metadata-ingestion/src/datahub/cli/config_utils.py b/metadata-ingestion/src/datahub/cli/config_utils.py index 4b69aec808..7877a6bf6d 100644 --- a/metadata-ingestion/src/datahub/cli/config_utils.py +++ b/metadata-ingestion/src/datahub/cli/config_utils.py @@ -1,3 +1,7 @@ +""" +For helper methods to contain manipulation of the config file in local system. +""" + import logging import os import sys @@ -9,9 +13,6 @@ from pydantic import BaseModel, ValidationError from datahub.cli.env_utils import get_boolean_env_variable -__help__ = ( - "For helper methods to contain manipulation of the config file in local system." -) log = logging.getLogger(__name__) DEFAULT_GMS_HOST = "http://localhost:8080" diff --git a/smoke-test/tests/dataproduct/queries/__init__.py b/smoke-test/tests/dataproduct/queries/__init__.py new file mode 100644 index 0000000000..e69de29bb2