fix(ingest): bug fixes and docs updates (#9422)

docs/modeling/extending-the-metadata-model.md

@@ -282,7 +282,7 @@ Generated package at custom-package/my-company-datahub-models
 This package should be installed alongside the main acryl-datahub package.
 
 Install the custom package locally with `pip install custom-package/my-company-datahub-models`
-To enable others to use it, share the file at custom-package/my-company-datahub-models/dist/*.whl and have them install it with `pip install <wheel file>.whl`
+To enable others to use it, share the file at custom-package/my-company-datahub-models/dist/<wheel file>.whl and have them install it with `pip install <wheel file>.whl`
 Alternatively, publish it to PyPI with `twine upload custom-package/my-company-datahub-models/dist/*`
 ```
 

metadata-ingestion/build.gradle

@@ -57,7 +57,10 @@ task installPackage(type: Exec, dependsOn: installPackageOnly) {
 }
 
 task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) {
-  inputs.files(project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc"))
+  inputs.files(
+    project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc"),
+    project.fileTree(dir: "scripts"),
+  )
   outputs.dir('src/datahub/metadata')
   commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/codegen.sh"
 }

metadata-ingestion/scripts/custom_package_codegen.py

@@ -116,7 +116,9 @@ setup(
     click.echo()
     click.echo(f"Install the custom package locally with `pip install {package_path}`")
     click.echo(
-        f"To enable others to use it, share the file at {package_path}/dist/*.whl and have them install it with `pip install <wheel file>.whl`"
+        "To enable others to use it, share the file at "
+        f"{package_path}/dist/{package_name}-{package_version}-py3-none-any.whl "
+        "and have them install it with `pip install <wheel file>.whl`"
     )
     click.echo(
         f"Alternatively, publish it to PyPI with `twine upload {package_path}/dist/*`"

metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py

@@ -1315,8 +1315,6 @@ class DBTSourceBase(StatefulIngestionSourceBase):
             self.config.strip_user_ids_from_email,
         )
 
-        # TODO if infer_dbt_schemas, load from saved schemas too
-
         canonical_schema: List[SchemaField] = []
         for column in node.columns:
             description = None

metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py

@@ -466,6 +466,19 @@ class DBTCoreSource(DBTSourceBase):
             catalog_version,
         ) = self.loadManifestAndCatalog()
 
+        # If catalog_version is between 1.7.0 and 1.7.2, report a warning.
+        if (
+            catalog_version
+            and catalog_version.startswith("1.7.")
+            and catalog_version < "1.7.3"
+        ):
+            self.report.report_warning(
+                "dbt_catalog_version",
+                f"Due to a bug in dbt, dbt version {catalog_version} will have incomplete metadata on sources. "
+                "Please upgrade to dbt version 1.7.3 or later. "
+                "See https://github.com/dbt-labs/dbt-core/issues/9119 for details on the bug.",
+            )
+
         additional_custom_props = {
             "manifest_schema": manifest_schema,
             "manifest_version": manifest_version,

metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py

@@ -157,7 +157,7 @@ class OracleInspectorObjectWrapper:
 
 @platform_name("Oracle")
 @config_class(OracleConfig)
-@support_status(SupportStatus.CERTIFIED)
+@support_status(SupportStatus.INCUBATING)
 @capability(SourceCapability.DOMAINS, "Enabled by default")
 class OracleSource(SQLAlchemySource):
     """

metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json

@@ -865,9 +865,6 @@
                 }
             },
             "inputs": [
-                {
-                    "string": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
-                },
                 {
                     "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
                 },

node_modules/.yarn-integrity

@@ -1,12 +0,0 @@
-{
-  "systemParams": "darwin-arm64-93",
-  "modulesFolders": [
-    "node_modules"
-  ],
-  "flags": [],
-  "linkedModules": [],
-  "topLevelPatterns": [],
-  "lockfileEntries": {},
-  "files": [],
-  "artifacts": {}
-}

yarn.lock

@@ -1,4 +0,0 @@
-# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
-# yarn lockfile v1
-
-