yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-03 04:56:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/dbt): enable incremental lineage by default (#7674)

Browse Source
This commit is contained in:
Harshal Sheth 2023-03-25 03:44:19 +05:30 committed by GitHub
parent d71463041f
commit c8abf9a1d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 1 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs-website/.gitignore vendored
View File

@ -2,6 +2,7 @@
/docs /docs
/genDocs /genDocs
/genStatic
# Generated GraphQL # Generated GraphQL
/graphql/combined.graphql /graphql/combined.graphql

5
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View File

@ -267,11 +267,6 @@ class DBTCommonConfig(
description='Regex string to extract owner from the dbt node using the `(?P<name>...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r"(?P<owner>(.*)): (\\w+) (\\w+)"` will extract `jdoe` as the owner from `"jdoe: John Doe"` (2) `r"@(?P<owner>(.*))"` will extract `alice` as the owner from `"@alice"`.', description='Regex string to extract owner from the dbt node using the `(?P<name>...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r"(?P<owner>(.*)): (\\w+) (\\w+)"` will extract `jdoe` as the owner from `"jdoe: John Doe"` (2) `r"@(?P<owner>(.*))"` will extract `alice` as the owner from `"@alice"`.',
) )
incremental_lineage: bool = Field(
# Copied from LineageConfig, and changed the default.
default=False,
description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
)
include_env_in_assertion_guid: bool = Field( include_env_in_assertion_guid: bool = Field(
default=False, default=False,
description="Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion " description="Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion "

3
metadata-ingestion/tests/integration/dbt/test_dbt.py
View File

@ -47,8 +47,6 @@ class DbtTestConfig:
self.output_path = f"{tmp_path}/{self.output_file}" self.output_path = f"{tmp_path}/{self.output_file}"
self.golden_path = f"{test_resources_dir}/{self.golden_file}" self.golden_path = f"{test_resources_dir}/{self.golden_file}"
self.source_config_modifiers.setdefault("incremental_lineage", True)
self.source_config = dict( self.source_config = dict(
{ {
"manifest_path": self.manifest_path, "manifest_path": self.manifest_path,
@ -254,7 +252,6 @@ def test_dbt_tests(pytestconfig, tmp_path, mock_time, **kwargs):
), ),
# this is just here to avoid needing to access datahub server # this is just here to avoid needing to access datahub server
write_semantics="OVERRIDE", write_semantics="OVERRIDE",
incremental_lineage=True,
), ),
), ),
sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}), sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}),