feat(cli): build and upload Python wheels in CI (#7537)

.github/workflows/metadata-ingestion.yml

@@ -51,13 +51,13 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install package
         run: ./gradlew :metadata-ingestion:installPackageOnly
       - name: Run metadata-ingestion tests (extras ${{ matrix.extraPythonRequirement }})
-        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }} -x :metadata-ingestion:installPackageOnly
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }}
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion/venv/bin/activate && pip freeze

docs-website/build.gradle

@@ -68,7 +68,10 @@ task yarnInstall(type: YarnTask) {
   }
 }
 
-task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall, generateGraphQLSchema, generateJsonSchema, ':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen'] ) {
+task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall,
+    generateGraphQLSchema, generateJsonSchema,
+    ':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen',
+    ':metadata-ingestion:buildWheel', ':metadata-ingestion-modules:airflow-plugin:buildWheel'] ) {
   inputs.files(projectMdFiles)
   outputs.cacheIf { true }
   args = ['run', 'generate']

docs-website/docusaurus.config.js

@@ -11,6 +11,7 @@ module.exports = {
   favicon: "img/favicon.ico",
   organizationName: "datahub-project", // Usually your GitHub org/user name.
   projectName: "datahub", // Usually your repo name.
+  staticDirectories: ["static", "genStatic"],
   stylesheets: [
     "https://fonts.googleapis.com/css2?family=Manrope:wght@400;600&display=swap",
   ],

docs-website/generateDocsDir.ts

@@ -16,6 +16,7 @@ const GITHUB_BROWSE_URL =
   "https://github.com/datahub-project/datahub/blob/master";
 
 const OUTPUT_DIRECTORY = "docs";
+const STATIC_DIRECTORY = "genStatic/artifacts";
 
 const SIDEBARS_DEF_PATH = "./sidebars.js";
 const sidebars = require(SIDEBARS_DEF_PATH);
@@ -535,6 +536,28 @@ function write_markdown_file(
   }
 }
 
+function copy_python_wheels(): void {
+  // Copy the built wheel files to the static directory.
+  const wheel_dirs = [
+    "../metadata-ingestion/dist",
+    "../metadata-ingestion-modules/airflow-plugin/dist",
+  ];
+
+  const wheel_output_directory = path.join(STATIC_DIRECTORY, "wheels");
+  fs.mkdirSync(wheel_output_directory, { recursive: true });
+
+  for (const wheel_dir of wheel_dirs) {
+    const wheel_files = fs.readdirSync(wheel_dir);
+    for (const wheel_file of wheel_files) {
+      const src = path.join(wheel_dir, wheel_file);
+      const dest = path.join(wheel_output_directory, wheel_file);
+
+      // console.log(`Copying artifact ${src} to ${dest}...`);
+      fs.copyFileSync(src, dest);
+    }
+  }
+}
+
 (async function main() {
   for (const filepath of markdown_files) {
     //console.log("Processing:", filepath);
@@ -586,4 +609,8 @@ function write_markdown_file(
       );
     }
   }
+
+  // Generate static directory.
+  copy_python_wheels();
+  // TODO: copy over the source json schemas + other artifacts.
 })();

docs-website/package.json

@@ -12,8 +12,8 @@
     "clear": "docusaurus clear && rm -rf genDocs/*",
     "_generate-graphql": "mkdir -p genDocs && docusaurus docs:generate:graphql",
     "_generate-docs": "rm -rf docs && mkdir docs && yarn _generate-graphql && ts-node -O '{ \"lib\": [\"es2020\"], \"target\": \"es6\" }' generateDocsDir.ts",
-    "generate": "rm -rf genDocs && mkdir genDocs && yarn _generate-docs && mv docs/* genDocs/ && rmdir docs",
-    "generate-rsync": "mkdir -p genDocs && yarn _generate-docs && rsync -v --checksum -r -h -i --delete docs/ genDocs && rm -rf docs",
+    "generate": "rm -rf genDocs genStatic && mkdir genDocs genStatic && yarn _generate-docs && mv docs/* genDocs/ && rmdir docs",
+    "generate-rsync": "mkdir -p genDocs genStatic && yarn _generate-docs && rsync -v --checksum -r -h -i --delete docs/ genDocs && rm -rf docs",
     "lint": "prettier -w generateDocsDir.ts sidebars.js src/pages/index.js",
     "lint-check": "prettier -l generateDocsDir.ts sidebars.js src/pages/index.js",
     "lint-fix": "prettier --write generateDocsDir.ts sidebars.js src/pages/index.js"

metadata-ingestion-modules/airflow-plugin/.gitignore

@@ -1,4 +1,5 @@
 .envrc
+src/datahub_airflow_plugin/__init__.py.bak
 .vscode/
 output
 pvenv36/

metadata-ingestion-modules/airflow-plugin/build.gradle

@@ -88,6 +88,9 @@ task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
   commandLine 'bash', '-x', '-c',
     "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
 }
+task buildWheel(type: Exec, dependsOn: [install]) {
+  commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
+}
 
 task cleanPythonCache(type: Exec) {
   commandLine 'bash', '-c',

metadata-ingestion-modules/airflow-plugin/setup.py

@@ -3,7 +3,6 @@ import pathlib
 
 import setuptools
 
-
 package_metadata: dict = {}
 with open("./src/datahub_airflow_plugin/__init__.py") as fp:
     exec(fp.read(), package_metadata)
@@ -125,6 +124,8 @@ setuptools.setup(
     install_requires=list(base_requirements),
     extras_require={
         "dev": list(dev_requirements),
-        "datahub-kafka": f"acryl-datahub[datahub-kafka] == {package_metadata['__version__']}",
+        "datahub-kafka": [
+            f"acryl-datahub[datahub-kafka] == {package_metadata['__version__']}"
+        ],
     },
 )

metadata-ingestion/.gitignore

@@ -1,4 +1,5 @@
 .envrc
+src/datahub/__init__.py.bak
 .vscode/
 output
 pvenv36/

metadata-ingestion/build.gradle

@@ -171,6 +171,9 @@ task testSlowIntegration(type: Exec, dependsOn: [installDevTest]) {
 task docGen(type: Exec, dependsOn: [codegen, installDevTest]) {
   commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/docgen.sh"
 }
+task buildWheel(type: Exec, dependsOn: [install, codegen]) {
+  commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
+}
 
 
 

metadata-ingestion/src/datahub/ingestion/source/metadata_common.py

@@ -1,10 +0,0 @@
-import warnings
-
-# This import retains backwards compatability, but is deprecated
-# and should be avoided.
-from datahub.ingestion.api.workunit import MetadataWorkUnit  # noqa: F401
-
-warnings.warn(
-    "importing from datahub.ingestion.source.metadata_common is deprecated; "
-    "use datahub.ingestion.api.workunit instead"
-)