fix(actions): fix datahub-actions publishing + wheels (#13276)

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-04-15 22:20:18 UTC
+# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-04-21 19:41:02 UTC
 # Do not edit this file directly. Run the script to regenerate.
 # Add additional hooks in .github/scripts/pre-commit-override.yaml
 repos:
@@ -18,6 +18,13 @@ repos:
         files: ^\.github/.*\.(yml|yaml)$
         pass_filenames: false
 
+      - id: datahub-actions-lint-fix
+        name: datahub-actions Lint Fix
+        entry: ./gradlew :datahub-actions:lintFix
+        language: system
+        files: ^datahub-actions/.*\.py$
+        pass_filenames: false
+
       - id: datahub-graphql-core-spotless
         name: datahub-graphql-core Spotless Apply
         entry: ./gradlew :datahub-graphql-core:spotlessApply

datahub-actions/scripts/release.sh

@@ -1,21 +1,10 @@
-# Copyright 2021 Acryl Data, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 #!/bin/bash
+# Auto-generated by python-build/generate_release_scripts.py. Do not edit manually.
+
 set -euxo pipefail
 
-ROOT=../..
+ROOT=..
+MODULE=datahub_actions
 
 if [[ ! ${RELEASE_SKIP_TEST:-} ]] && [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
     ${ROOT}/gradlew build  # also runs tests
@@ -26,16 +15,17 @@ fi
 # Check packaging constraint.
 python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
 
-if [[ ${RELEASE_VERSION:-} ]]; then
-    # Replace version with RELEASE_VERSION env variable
-    sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub_actions/__init__.py
-else
-    vim src/datahub_actions/__init__.py
+# Update the release version.
+if [[ ! ${RELEASE_VERSION:-} ]]; then
+    echo "RELEASE_VERSION is not set"
+    exit 1
 fi
+sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/_version.py
 
+# Build and upload the release.
 rm -rf build dist || true
 python -m build
 if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
-    python -m twine upload 'dist/*' --verbose
+    python -m twine upload 'dist/*'
 fi
-git restore src/datahub_actions/__init__.py
+mv src/${MODULE}/_version.py.bak src/${MODULE}/_version.py

datahub-actions/setup.py

@@ -18,7 +18,7 @@ from typing import Dict, Set
 import setuptools
 
 package_metadata: dict = {}
-with open("./src/datahub_actions/__init__.py") as fp:
+with open("./src/datahub_actions/_version.py") as fp:
     exec(fp.read(), package_metadata)
 
 

datahub-actions/src/datahub_actions/__init__.py

@@ -12,16 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Published at https://pypi.org/project/acryl-datahub-actions/.
-__package_name__ = "acryl-datahub-actions"
-__version__ = "1!0.0.0.dev0"
-
-
-def is_dev_mode() -> bool:
-    return __version__ == "1!0.0.0.dev0"
-
-
-def nice_version_name() -> str:
-    if is_dev_mode():
-        return "unavailable (installed editable via git)"
-    return __version__
+from datahub_actions._version import __package_name__, __version__

datahub-actions/src/datahub_actions/_version.py

@@ -0,0 +1,13 @@
+# Published at https://pypi.org/project/acryl-datahub-actions/.
+__package_name__ = "acryl-datahub-actions"
+__version__ = "1!0.0.0.dev0"
+
+
+def is_dev_mode() -> bool:
+    return __version__.endswith("dev0")
+
+
+def nice_version_name() -> str:
+    if is_dev_mode():
+        return "unavailable (installed in develop mode)"
+    return __version__

datahub-actions/src/datahub_actions/cli/actions.py

@@ -23,7 +23,7 @@ import click
 from click_default_group import DefaultGroup
 from expandvars import UnboundVariable
 
-import datahub_actions as datahub_actions_package
+import datahub_actions._version as actions_version
 from datahub.configuration.config_loader import load_config_file
 from datahub_actions.pipeline.pipeline import Pipeline
 from datahub_actions.pipeline.pipeline_manager import PipelineManager
@@ -103,9 +103,7 @@ def is_pipeline_enabled(config: dict) -> bool:
 def run(ctx: Any, config: List[str], debug: bool) -> None:
     """Execute one or more Actions Pipelines"""
 
-    logger.info(
-        "DataHub Actions version: %s", datahub_actions_package.nice_version_name()
-    )
+    logger.info("DataHub Actions version: %s", actions_version.nice_version_name())
 
     if debug:
         logging.getLogger().setLevel(logging.DEBUG)
@@ -179,9 +177,7 @@ def run(ctx: Any, config: List[str], debug: bool) -> None:
 @actions.command()
 def version() -> None:
     """Print version number and exit."""
-    click.echo(
-        f"DataHub Actions version: {datahub_actions_package.nice_version_name()}"
-    )
+    click.echo(f"DataHub Actions version: {actions_version.nice_version_name()}")
     click.echo(f"Python version: {sys.version}")
 
 

datahub-actions/src/datahub_actions/entrypoints.py

@@ -20,7 +20,7 @@ import click
 import stackprinter
 from prometheus_client import start_http_server
 
-import datahub_actions as datahub_package
+import datahub_actions._version as actions_version
 from datahub.cli.env_utils import get_boolean_env_variable
 from datahub_actions.cli.actions import actions
 
@@ -59,8 +59,8 @@ MAX_CONTENT_WIDTH = 120
 )
 @click.option("--debug/--no-debug", default=False)
 @click.version_option(
-    version=datahub_package.nice_version_name(),
-    prog_name=datahub_package.__package_name__,
+    version=actions_version.nice_version_name(),
+    prog_name=actions_version.__package_name__,
 )
 @click.option(
     "-dl",
@@ -129,7 +129,7 @@ def main(**kwargs):
             )
         )
         logger.info(
-            f"DataHub Actions version: {datahub_package.__version__} at {datahub_package.__file__}"
+            f"DataHub Actions version: {actions_version.__version__} at {actions_version.__file__}"
         )
         logger.info(
             f"Python version: {sys.version} at {sys.executable} on {platform.platform()}"

python-build/build.gradle

@@ -14,6 +14,7 @@ task checkPythonVersion(type: Exec) {
 task buildWheels(type: Exec, dependsOn: [
   checkPythonVersion,
   ':metadata-ingestion:buildWheel',
+  ':datahub-actions:buildWheel',
   ':metadata-ingestion-modules:airflow-plugin:buildWheel',
   ':metadata-ingestion-modules:dagster-plugin:buildWheel',
   ':metadata-ingestion-modules:prefect-plugin:buildWheel',

python-build/copy_wheels.py

@@ -8,6 +8,7 @@ WHEEL_OUTPUT_DIR = PYTHON_BUILD_DIR / "wheels"
 # These should line up with the build.gradle file.
 wheel_dirs = [
     ROOT_DIR / "metadata-ingestion/dist",
+    ROOT_DIR / "datahub-actions/dist",
     ROOT_DIR / "metadata-ingestion-modules/airflow-plugin/dist",
     ROOT_DIR / "metadata-ingestion-modules/dagster-plugin/dist",
     ROOT_DIR / "metadata-ingestion-modules/prefect-plugin/dist",

python-build/generate_release_scripts.py

@@ -18,6 +18,7 @@ class Package:
 
 packages = [
     Package(directory="metadata-ingestion", main_module_name="datahub"),
+    Package(directory="datahub-actions", main_module_name="datahub_actions"),
     Package(
         directory="metadata-ingestion-modules/airflow-plugin",
         main_module_name="datahub_airflow_plugin",