datahub/metadata-ingestion-modules/airflow-plugin/setup.py

import os
import pathlib
from typing import Dict, Set

import setuptools

package_metadata: dict = {}
with open("./src/datahub_airflow_plugin/_version.py") as fp:
    exec(fp.read(), package_metadata)


def get_long_description():
    root = os.path.dirname(__file__)
    return pathlib.Path(os.path.join(root, "README.md")).read_text()


_version: str = package_metadata["__version__"]
_self_pin = (
    f"=={_version}"
    if not (_version.endswith(("dev0", "dev1")) or "docker" in _version)
    else ""
)


base_requirements = {
    f"acryl-datahub[datahub-rest]{_self_pin}",
    # We require Airflow 2.3.x at minimum, since we need the new DAG listener API.
    # We pin to 2.5.x, since we also need typing-extensions>=4.5 in acryl-datahub.
    "apache-airflow>=2.5.0,<3",
}

plugins: Dict[str, Set[str]] = {
    "datahub-rest": {
        f"acryl-datahub[datahub-rest]{_self_pin}",
    },
    "datahub-kafka": {
        f"acryl-datahub[datahub-kafka]{_self_pin}",
    },
    "datahub-file": {
        f"acryl-datahub[sync-file-emitter]{_self_pin}",
    },
    "plugin-v1": set(),
    "plugin-v2": {
        f"acryl-datahub[sql-parser]{_self_pin}",
        # We remain restrictive on the versions allowed here to prevent
        # us from being broken by backwards-incompatible changes in the
        # underlying package.
        "openlineage-airflow>=1.2.0,<=1.30.1",
    },
}

# Require some plugins by default.
base_requirements.update(plugins["datahub-rest"])
base_requirements.update(plugins["plugin-v2"])


mypy_stubs = {
    "types-dataclasses",
    "sqlalchemy-stubs",
    "types-setuptools",
    "types-six",
    "types-python-dateutil",
    "types-requests",
    "types-toml",
    "types-PyYAML",
    "types-cachetools",
    # versions 0.1.13 and 0.1.14 seem to have issues
    "types-click==0.1.12",
    "types-tabulate",
}

dev_requirements = {
    *base_requirements,
    *mypy_stubs,
    "coverage>=5.1",
    "mypy==1.14.1",
    "ruff==0.11.7",
    # pydantic 1.8.2 is incompatible with mypy 0.910.
    # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.
    "pydantic>=1.10.16",
    "pytest>=6.2.2",
    "pytest-cov>=2.8.1",
    "tox",
    "tox-uv",
    # Missing numpy requirement in 8.0.0
    "deepdiff!=8.0.0",
    "tenacity",
    "build",
    "twine",
    "packaging",
}

integration_test_requirements = {
    *plugins["datahub-file"],
    *plugins["datahub-kafka"],
    f"acryl-datahub[testing-utils]{_self_pin}",
    # Extra requirements for loading our test dags.
    "apache-airflow[snowflake,amazon]>=2.0.2",
    # A collection of issues we've encountered:
    # - Connexion's new version breaks Airflow:
    #   See https://github.com/apache/airflow/issues/35234.
    # - https://github.com/snowflakedb/snowflake-sqlalchemy/issues/350
    #   Eventually we want to set this to "snowflake-sqlalchemy>=1.4.3".
    # - To avoid https://github.com/snowflakedb/snowflake-connector-python/issues/1188,
    #   we need https://github.com/snowflakedb/snowflake-connector-python/pull/1193
    "snowflake-connector-python>=2.7.10",
    "virtualenv",  # needed by PythonVirtualenvOperator
    "apache-airflow-providers-sqlite",
}
per_version_test_requirements = {
    "test-airflow25": {
        "pendulum<3.0",
        "Flask-Session<0.6.0",
        "connexion<3.0",
        "marshmallow<3.24.0",
        "apache-airflow-providers-amazon==7.3.0",
    },
}


entry_points = {
    "airflow.plugins": "acryl-datahub-airflow-plugin = datahub_airflow_plugin.datahub_plugin:DatahubPlugin",
    "apache_airflow_provider": ["provider_info=datahub_provider:get_provider_info"],
}


setuptools.setup(
    # Package metadata.
    name=package_metadata["__package_name__"],
    version=_version,
    url="https://docs.datahub.com/",
    project_urls={
        "Documentation": "https://docs.datahub.com/docs/",
        "Source": "https://github.com/datahub-project/datahub",
        "Changelog": "https://github.com/datahub-project/datahub/releases",
    },
    license="Apache License 2.0",
    description="Datahub Airflow plugin to capture executions and send to Datahub",
    long_description=get_long_description(),
    long_description_content_type="text/markdown",
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3 :: Only",
        "Intended Audience :: Developers",
        "Intended Audience :: Information Technology",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: Unix",
        "Operating System :: POSIX :: Linux",
        "Environment :: Console",
        "Environment :: MacOS X",
        "Topic :: Software Development",
    ],
    # Package info.
    zip_safe=False,
    python_requires=">=3.8",
    package_data={
        "datahub_airflow_plugin": ["py.typed"],
    },
    package_dir={"": "src"},
    packages=setuptools.find_namespace_packages(where="./src"),
    entry_points=entry_points,
    # Dependencies.
    install_requires=list(base_requirements),
    extras_require={
        "ignore": [],  # This is a dummy extra to allow for trailing commas in the list.
        **{plugin: list(dependencies) for plugin, dependencies in plugins.items()},
        "dev": list(dev_requirements),
        "integration-tests": list(integration_test_requirements),
        **{
            plugin: list(dependencies)
            for plugin, dependencies in per_version_test_requirements.items()
        },
    },
)