fix/nox-ci-missing-steps (#21426)

* Fix nox-ci

* Fix wrong path

* Fix wrong path

* Use working-directory for gha

* Fix wrong section in gha yml

* Diable some lint to diagnose failures

* Rm version matrix for debugging

* Fix type in nox invocation

* Fix style

* Add version and update checkout version

* Add required system dependencies

* WIP

* Add python code generation

* Remove version extraction from nox, as it not needed

---------

Co-authored-by: Pere Menal <pere.menal@getcollate.io>

.github/workflows/py-nox-ci.yml

@@ -11,25 +11,50 @@ on:
 jobs:
   format-and-unit:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./ingestion
     strategy:
       matrix:
         python-version: ["3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install uv and nox
+      - name: Install system dependencies
         run: |
+          sudo apt-get update
+          sudo apt-get install -y libkrb5-dev build-essential
+
+      - name: Setup JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Generate Python Data Models
+        run: |
+          cd ..
+          make install_antlr_cli
+          pip install datamodel-code-generator==0.25.6
+          mkdir -p ingestion/src/metadata/generated
+          python scripts/datamodel_generation.py
+          antlr4 -Dlanguage=Python3 -o ingestion/src/metadata/generated/antlr \
+            ${PWD}/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4
+
+
+      - name: Set up Nox
+        run: |
+          # Install uv to speed up virtual environments installation
           curl -LsSf https://astral.sh/uv/install.sh | sh
           pip install nox
 
       - name: Run Code Quality Checks
-        # We only want to check the format for a single Python version,
-        # no need to run it for all versions
         if: ${{ matrix.python-version == '3.10' }}
         run: |
           nox -s lint

ingestion/noxfile.py

@@ -11,15 +11,14 @@
 """
 Nox sessions for testing and formatting checks.
 """
-import ast
 import os
-from pathlib import Path
+
+import nox
 
 # NOTE: This is still a work in progress! We still need to:
 #    - Fix ignored unit tests
 #    - Add integration tests
 #    - Address the TODOs in the code
-import nox
 
 # TODO: Add python 3.9. PYTHON 3.9 fails in Mac os due to problem with `psycopg2-binary` package
 
@@ -38,7 +37,7 @@ def get_python_versions():
 
 @nox.session(
     name="lint",
-    reuse_venv=False,
+    reuse_venv=True,
     venv_backend="uv|venv",
 )
 def lint(session):
@@ -57,7 +56,7 @@ def lint(session):
 
 
 @nox.session(
-    name="unit", reuse_venv=False, venv_backend="uv|venv", python=get_python_versions()
+    name="unit", reuse_venv=True, venv_backend="uv|venv", python=get_python_versions()
 )
 def unit(session):
     session.install(".[all-dev-env, test-unit]")
@@ -83,6 +82,7 @@ def unit(session):
     ]
     ignore_args = [f"--ignore=tests/unit/{test}" for test in ignored_tests]
 
+    # run pytest with the ignore arguments and in parallel mode
     session.run("pytest", "tests/unit/", *ignore_args)
 
 
@@ -95,44 +95,13 @@ PLUGINS = list(PLUGINS_TESTS.keys())
 
 @nox.session(
     name="unit-plugins",
-    reuse_venv=True,
+    reuse_venv=False,
     venv_backend="uv|venv",
     python=get_python_versions(),
 )
 @nox.parametrize("plugin", PLUGINS)
 def unit_plugins(session, plugin):
-    versions = extract_attribute_from_setup("VERSIONS", "setup.py")
-
-    if not versions:
-        session.error("Not able to extract VERSIONS from setup.py")
-        session.exit(1)
-
-    if plugin not in versions:
-        session.error(
-            f"Plugin {plugin} not found in VERSIONS. Available plugins: {list(versions)}"
-        )
-        session.exit(1)
-
     session.install(".[test-unit]")
-    session.install(versions[plugin])
+    session.install(f".[{plugin}]")
     # Assuming the plugin has its own tests in a specific directory
     session.run("pytest", PLUGINS_TESTS[plugin])
-
-
-def extract_attribute_from_setup(attr_name: str, setup_path: str = "setup.py"):
-    # TODO: We should consider using a more robust method to extract attributes
-    # such as moving out the attributes to a separate file.
-    setup_file = Path(setup_path)
-    if not setup_file.exists():
-        raise FileNotFoundError(f"{setup_path} not found")
-
-    with setup_file.open("r") as f:
-        tree = ast.parse(f.read(), filename=setup_path)
-
-    for node in ast.iter_child_nodes(tree):
-        if isinstance(node, ast.Assign):
-            for target in node.targets:
-                if isinstance(target, ast.Name) and target.id == attr_name:
-                    return ast.literal_eval(node.value)
-
-    return None  # Not found