diff --git a/.github/workflows/py-nox-ci.yml b/.github/workflows/py-nox-ci.yml index 145f10d9ef7..c3801d420c4 100644 --- a/.github/workflows/py-nox-ci.yml +++ b/.github/workflows/py-nox-ci.yml @@ -11,25 +11,50 @@ on: jobs: format-and-unit: runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./ingestion strategy: matrix: python-version: ["3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install uv and nox + - name: Install system dependencies run: | + sudo apt-get update + sudo apt-get install -y libkrb5-dev build-essential + + - name: Setup JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Generate Python Data Models + run: | + cd .. + make install_antlr_cli + pip install datamodel-code-generator==0.25.6 + mkdir -p ingestion/src/metadata/generated + python scripts/datamodel_generation.py + antlr4 -Dlanguage=Python3 -o ingestion/src/metadata/generated/antlr \ + ${PWD}/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4 + + + - name: Set up Nox + run: | + # Install uv to speed up virtual environments installation curl -LsSf https://astral.sh/uv/install.sh | sh pip install nox - name: Run Code Quality Checks - # We only want to check the format for a single Python version, - # no need to run it for all versions if: ${{ matrix.python-version == '3.10' }} run: | nox -s lint diff --git a/ingestion/noxfile.py b/ingestion/noxfile.py index b90f828cf26..3cda6fd30f1 100644 --- a/ingestion/noxfile.py +++ b/ingestion/noxfile.py @@ -11,15 +11,14 @@ """ Nox sessions for testing and formatting checks. """ -import ast import os -from pathlib import Path + +import nox # NOTE: This is still a work in progress! We still need to: # - Fix ignored unit tests # - Add integration tests # - Address the TODOs in the code -import nox # TODO: Add python 3.9. PYTHON 3.9 fails in Mac os due to problem with `psycopg2-binary` package @@ -38,7 +37,7 @@ def get_python_versions(): @nox.session( name="lint", - reuse_venv=False, + reuse_venv=True, venv_backend="uv|venv", ) def lint(session): @@ -57,7 +56,7 @@ def lint(session): @nox.session( - name="unit", reuse_venv=False, venv_backend="uv|venv", python=get_python_versions() + name="unit", reuse_venv=True, venv_backend="uv|venv", python=get_python_versions() ) def unit(session): session.install(".[all-dev-env, test-unit]") @@ -83,6 +82,7 @@ def unit(session): ] ignore_args = [f"--ignore=tests/unit/{test}" for test in ignored_tests] + # run pytest with the ignore arguments and in parallel mode session.run("pytest", "tests/unit/", *ignore_args) @@ -95,44 +95,13 @@ PLUGINS = list(PLUGINS_TESTS.keys()) @nox.session( name="unit-plugins", - reuse_venv=True, + reuse_venv=False, venv_backend="uv|venv", python=get_python_versions(), ) @nox.parametrize("plugin", PLUGINS) def unit_plugins(session, plugin): - versions = extract_attribute_from_setup("VERSIONS", "setup.py") - - if not versions: - session.error("Not able to extract VERSIONS from setup.py") - session.exit(1) - - if plugin not in versions: - session.error( - f"Plugin {plugin} not found in VERSIONS. Available plugins: {list(versions)}" - ) - session.exit(1) - session.install(".[test-unit]") - session.install(versions[plugin]) + session.install(f".[{plugin}]") # Assuming the plugin has its own tests in a specific directory session.run("pytest", PLUGINS_TESTS[plugin]) - - -def extract_attribute_from_setup(attr_name: str, setup_path: str = "setup.py"): - # TODO: We should consider using a more robust method to extract attributes - # such as moving out the attributes to a separate file. - setup_file = Path(setup_path) - if not setup_file.exists(): - raise FileNotFoundError(f"{setup_path} not found") - - with setup_file.open("r") as f: - tree = ast.parse(f.read(), filename=setup_path) - - for node in ast.iter_child_nodes(tree): - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name) and target.id == attr_name: - return ast.literal_eval(node.value) - - return None # Not found