yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-31 02:37:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/fivetran): support filtering on destination ids (#11277)

Browse Source
This commit is contained in:
matthew-coudert-cko 2024-08-30 21:25:18 +01:00 committed by GitHub
parent 3e5c18f5d2
commit c513e17dbb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 19 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py
View File

@ -161,6 +161,10 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin
default=AllowDenyPattern.allow_all(),
description="Regex patterns for connectors to filter in ingestion.",
)
destination_patterns: AllowDenyPattern = Field(
default=AllowDenyPattern.allow_all(),
description="Regex patterns for destinations to filter in ingestion.",
)
include_column_lineage: bool = Field(
default=True,
description="Populates table->table column lineage.",

1
metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
View File

@ -283,6 +283,7 @@ class FivetranSource(StatefulIngestionSourceBase):
logger.info("Fivetran plugin execution is started")
connectors = self.audit_log.get_allowed_connectors_list(
self.config.connector_patterns,
self.config.destination_patterns,
self.report,
self.config.history_sync_lookback_period,
)

4
metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
View File

@ -251,6 +251,7 @@ class FivetranLogAPI:
def get_allowed_connectors_list(
self,
connector_patterns: AllowDenyPattern,
destination_patterns: AllowDenyPattern,
report: FivetranSourceReport,
syncs_interval: int,
) -> List[Connector]:
@ -261,6 +262,9 @@ class FivetranLogAPI:
if not connector_patterns.allowed(connector[Constant.CONNECTOR_NAME]):
report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME])
continue
if not destination_patterns.allowed(connector[Constant.DESTINATION_ID]):
report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME])
continue
connectors.append(
Connector(
connector_id=connector[Constant.CONNECTOR_ID],

10
metadata-ingestion/tests/integration/fivetran/test_fivetran.py
View File

@ -205,6 +205,11 @@ def test_fivetran_with_snowflake_dest(pytestconfig, tmp_path):
"postgres",
]
},
"destination_patterns": {
"allow": [
"interval_unconstitutional",
]
},
"sources_to_database": {
"calendar_elected": "postgres_db",
},
@ -291,6 +296,11 @@ def test_fivetran_with_snowflake_dest_and_null_connector_user(pytestconfig, tmp_
"postgres",
]
},
"destination_patterns": {
"allow": [
"interval_unconstitutional",
]
},
"sources_to_database": {
"calendar_elected": "postgres_db",
},