From fa750573e2b0168a3fbeae7dcb870c8d4b8094fc Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 21 Apr 2025 17:19:25 -0400 Subject: [PATCH] fix(actions): fix datahub-actions publishing + wheels (#13276) --- .pre-commit-config.yaml | 9 ++++- datahub-actions/scripts/release.sh | 34 +++++++------------ datahub-actions/setup.py | 2 +- .../src/datahub_actions/__init__.py | 14 +------- .../src/datahub_actions/_version.py | 13 +++++++ .../src/datahub_actions/cli/actions.py | 10 ++---- .../src/datahub_actions/entrypoints.py | 8 ++--- python-build/build.gradle | 1 + python-build/copy_wheels.py | 1 + python-build/generate_release_scripts.py | 1 + 10 files changed, 45 insertions(+), 48 deletions(-) create mode 100644 datahub-actions/src/datahub_actions/_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 87854d95b8..02da599fb5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-04-15 22:20:18 UTC +# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-04-21 19:41:02 UTC # Do not edit this file directly. Run the script to regenerate. # Add additional hooks in .github/scripts/pre-commit-override.yaml repos: @@ -18,6 +18,13 @@ repos: files: ^\.github/.*\.(yml|yaml)$ pass_filenames: false + - id: datahub-actions-lint-fix + name: datahub-actions Lint Fix + entry: ./gradlew :datahub-actions:lintFix + language: system + files: ^datahub-actions/.*\.py$ + pass_filenames: false + - id: datahub-graphql-core-spotless name: datahub-graphql-core Spotless Apply entry: ./gradlew :datahub-graphql-core:spotlessApply diff --git a/datahub-actions/scripts/release.sh b/datahub-actions/scripts/release.sh index 017ab17ac4..ef68ca4bd4 100755 --- a/datahub-actions/scripts/release.sh +++ b/datahub-actions/scripts/release.sh @@ -1,21 +1,10 @@ -# Copyright 2021 Acryl Data, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - #!/bin/bash +# Auto-generated by python-build/generate_release_scripts.py. Do not edit manually. + set -euxo pipefail -ROOT=../.. +ROOT=.. +MODULE=datahub_actions if [[ ! ${RELEASE_SKIP_TEST:-} ]] && [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then ${ROOT}/gradlew build # also runs tests @@ -26,16 +15,17 @@ fi # Check packaging constraint. python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"' -if [[ ${RELEASE_VERSION:-} ]]; then - # Replace version with RELEASE_VERSION env variable - sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub_actions/__init__.py -else - vim src/datahub_actions/__init__.py +# Update the release version. +if [[ ! ${RELEASE_VERSION:-} ]]; then + echo "RELEASE_VERSION is not set" + exit 1 fi +sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/_version.py +# Build and upload the release. rm -rf build dist || true python -m build if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then - python -m twine upload 'dist/*' --verbose + python -m twine upload 'dist/*' fi -git restore src/datahub_actions/__init__.py \ No newline at end of file +mv src/${MODULE}/_version.py.bak src/${MODULE}/_version.py diff --git a/datahub-actions/setup.py b/datahub-actions/setup.py index b4d8efaba4..624f159184 100644 --- a/datahub-actions/setup.py +++ b/datahub-actions/setup.py @@ -18,7 +18,7 @@ from typing import Dict, Set import setuptools package_metadata: dict = {} -with open("./src/datahub_actions/__init__.py") as fp: +with open("./src/datahub_actions/_version.py") as fp: exec(fp.read(), package_metadata) diff --git a/datahub-actions/src/datahub_actions/__init__.py b/datahub-actions/src/datahub_actions/__init__.py index 024703200d..0584a2fa1f 100644 --- a/datahub-actions/src/datahub_actions/__init__.py +++ b/datahub-actions/src/datahub_actions/__init__.py @@ -12,16 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Published at https://pypi.org/project/acryl-datahub-actions/. -__package_name__ = "acryl-datahub-actions" -__version__ = "1!0.0.0.dev0" - - -def is_dev_mode() -> bool: - return __version__ == "1!0.0.0.dev0" - - -def nice_version_name() -> str: - if is_dev_mode(): - return "unavailable (installed editable via git)" - return __version__ +from datahub_actions._version import __package_name__, __version__ diff --git a/datahub-actions/src/datahub_actions/_version.py b/datahub-actions/src/datahub_actions/_version.py new file mode 100644 index 0000000000..8b7d137b5d --- /dev/null +++ b/datahub-actions/src/datahub_actions/_version.py @@ -0,0 +1,13 @@ +# Published at https://pypi.org/project/acryl-datahub-actions/. +__package_name__ = "acryl-datahub-actions" +__version__ = "1!0.0.0.dev0" + + +def is_dev_mode() -> bool: + return __version__.endswith("dev0") + + +def nice_version_name() -> str: + if is_dev_mode(): + return "unavailable (installed in develop mode)" + return __version__ diff --git a/datahub-actions/src/datahub_actions/cli/actions.py b/datahub-actions/src/datahub_actions/cli/actions.py index 7a39d01101..61d9fbb05c 100644 --- a/datahub-actions/src/datahub_actions/cli/actions.py +++ b/datahub-actions/src/datahub_actions/cli/actions.py @@ -23,7 +23,7 @@ import click from click_default_group import DefaultGroup from expandvars import UnboundVariable -import datahub_actions as datahub_actions_package +import datahub_actions._version as actions_version from datahub.configuration.config_loader import load_config_file from datahub_actions.pipeline.pipeline import Pipeline from datahub_actions.pipeline.pipeline_manager import PipelineManager @@ -103,9 +103,7 @@ def is_pipeline_enabled(config: dict) -> bool: def run(ctx: Any, config: List[str], debug: bool) -> None: """Execute one or more Actions Pipelines""" - logger.info( - "DataHub Actions version: %s", datahub_actions_package.nice_version_name() - ) + logger.info("DataHub Actions version: %s", actions_version.nice_version_name()) if debug: logging.getLogger().setLevel(logging.DEBUG) @@ -179,9 +177,7 @@ def run(ctx: Any, config: List[str], debug: bool) -> None: @actions.command() def version() -> None: """Print version number and exit.""" - click.echo( - f"DataHub Actions version: {datahub_actions_package.nice_version_name()}" - ) + click.echo(f"DataHub Actions version: {actions_version.nice_version_name()}") click.echo(f"Python version: {sys.version}") diff --git a/datahub-actions/src/datahub_actions/entrypoints.py b/datahub-actions/src/datahub_actions/entrypoints.py index 9354da322a..78d94c212f 100644 --- a/datahub-actions/src/datahub_actions/entrypoints.py +++ b/datahub-actions/src/datahub_actions/entrypoints.py @@ -20,7 +20,7 @@ import click import stackprinter from prometheus_client import start_http_server -import datahub_actions as datahub_package +import datahub_actions._version as actions_version from datahub.cli.env_utils import get_boolean_env_variable from datahub_actions.cli.actions import actions @@ -59,8 +59,8 @@ MAX_CONTENT_WIDTH = 120 ) @click.option("--debug/--no-debug", default=False) @click.version_option( - version=datahub_package.nice_version_name(), - prog_name=datahub_package.__package_name__, + version=actions_version.nice_version_name(), + prog_name=actions_version.__package_name__, ) @click.option( "-dl", @@ -129,7 +129,7 @@ def main(**kwargs): ) ) logger.info( - f"DataHub Actions version: {datahub_package.__version__} at {datahub_package.__file__}" + f"DataHub Actions version: {actions_version.__version__} at {actions_version.__file__}" ) logger.info( f"Python version: {sys.version} at {sys.executable} on {platform.platform()}" diff --git a/python-build/build.gradle b/python-build/build.gradle index e90bffd468..2811f6e66c 100644 --- a/python-build/build.gradle +++ b/python-build/build.gradle @@ -14,6 +14,7 @@ task checkPythonVersion(type: Exec) { task buildWheels(type: Exec, dependsOn: [ checkPythonVersion, ':metadata-ingestion:buildWheel', + ':datahub-actions:buildWheel', ':metadata-ingestion-modules:airflow-plugin:buildWheel', ':metadata-ingestion-modules:dagster-plugin:buildWheel', ':metadata-ingestion-modules:prefect-plugin:buildWheel', diff --git a/python-build/copy_wheels.py b/python-build/copy_wheels.py index b66662cbfe..eabdfcd169 100644 --- a/python-build/copy_wheels.py +++ b/python-build/copy_wheels.py @@ -8,6 +8,7 @@ WHEEL_OUTPUT_DIR = PYTHON_BUILD_DIR / "wheels" # These should line up with the build.gradle file. wheel_dirs = [ ROOT_DIR / "metadata-ingestion/dist", + ROOT_DIR / "datahub-actions/dist", ROOT_DIR / "metadata-ingestion-modules/airflow-plugin/dist", ROOT_DIR / "metadata-ingestion-modules/dagster-plugin/dist", ROOT_DIR / "metadata-ingestion-modules/prefect-plugin/dist", diff --git a/python-build/generate_release_scripts.py b/python-build/generate_release_scripts.py index 36253a24cf..e8d77febd8 100644 --- a/python-build/generate_release_scripts.py +++ b/python-build/generate_release_scripts.py @@ -18,6 +18,7 @@ class Package: packages = [ Package(directory="metadata-ingestion", main_module_name="datahub"), + Package(directory="datahub-actions", main_module_name="datahub_actions"), Package( directory="metadata-ingestion-modules/airflow-plugin", main_module_name="datahub_airflow_plugin",