feat(ci): check for packages, change location of lint (#9759)

.github/scripts/check_python_package.py

@@ -0,0 +1,18 @@
+import setuptools
+
+folders = ["./smoke-test/tests"]
+
+for folder in folders:
+    print(f"Checking folder {folder}")
+    a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+    b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
+
+    in_a_not_b = set(a) - set(b)
+    in_b_not_a = set(b) - set(a)
+
+    assert (
+        len(in_a_not_b) == 0
+    ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
+    assert (
+        len(in_b_not_a) == 0
+    ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"

.github/workflows/docker-unified.yml

@@ -37,8 +37,29 @@ env:
   DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion"
 
 jobs:
+  check_lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - name: Ensure packages are correct
+        run: |
+          python ./.github/scripts/check_python_package.py
+      - name: Run lint on smoke test
+        run: |
+          ./gradlew :smoke-test:lint
   setup:
     runs-on: ubuntu-latest
+    needs: check_lint
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
       slim_tag: ${{ steps.tag.outputs.slim_tag }}
@@ -64,18 +85,6 @@ jobs:
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-      - name: Set up JDK 17
-        uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: 17
-      - name: Run lint on smoke test
-        run: |
-          ./gradlew :smoke-test:lint
       - name: Compute Tag
         id: tag
         run: |

metadata-ingestion/src/datahub/cli/config_utils.py

@@ -1,3 +1,7 @@
+"""
+For helper methods to contain manipulation of the config file in local system.
+"""
+
 import logging
 import os
 import sys
@@ -9,9 +13,6 @@ from pydantic import BaseModel, ValidationError
 
 from datahub.cli.env_utils import get_boolean_env_variable
 
-__help__ = (
-    "For helper methods to contain manipulation of the config file in local system."
-)
 log = logging.getLogger(__name__)
 
 DEFAULT_GMS_HOST = "http://localhost:8080"