yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-31 02:37:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/fivetran): support filtering on destination ids (#11277)

Browse Source
This commit is contained in:
matthew-coudert-cko 2024-08-30 21:25:18 +01:00 committed by GitHub
parent 3e5c18f5d2
commit c513e17dbb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 19 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py
View File

@ -161,6 +161,10 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin
default=AllowDenyPattern.allow_all(), default=AllowDenyPattern.allow_all(),
description="Regex patterns for connectors to filter in ingestion.", description="Regex patterns for connectors to filter in ingestion.",
) )
destination_patterns: AllowDenyPattern = Field(
default=AllowDenyPattern.allow_all(),
description="Regex patterns for destinations to filter in ingestion.",
)
include_column_lineage: bool = Field( include_column_lineage: bool = Field(
default=True, default=True,
description="Populates table->table column lineage.", description="Populates table->table column lineage.",

1
metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
View File

@ -283,6 +283,7 @@ class FivetranSource(StatefulIngestionSourceBase):
logger.info("Fivetran plugin execution is started") logger.info("Fivetran plugin execution is started")
connectors = self.audit_log.get_allowed_connectors_list( connectors = self.audit_log.get_allowed_connectors_list(
self.config.connector_patterns, self.config.connector_patterns,
self.config.destination_patterns,
self.report, self.report,
self.config.history_sync_lookback_period, self.config.history_sync_lookback_period,
) )

4
metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
View File

@ -251,6 +251,7 @@ class FivetranLogAPI:
def get_allowed_connectors_list( def get_allowed_connectors_list(
self, self,
connector_patterns: AllowDenyPattern, connector_patterns: AllowDenyPattern,
destination_patterns: AllowDenyPattern,
report: FivetranSourceReport, report: FivetranSourceReport,
syncs_interval: int, syncs_interval: int,
) -> List[Connector]: ) -> List[Connector]:
@ -261,6 +262,9 @@ class FivetranLogAPI:
if not connector_patterns.allowed(connector[Constant.CONNECTOR_NAME]): if not connector_patterns.allowed(connector[Constant.CONNECTOR_NAME]):
report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME]) report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME])
continue continue
if not destination_patterns.allowed(connector[Constant.DESTINATION_ID]):
report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME])
continue
connectors.append( connectors.append(
Connector( Connector(
connector_id=connector[Constant.CONNECTOR_ID], connector_id=connector[Constant.CONNECTOR_ID],

10
metadata-ingestion/tests/integration/fivetran/test_fivetran.py
View File

@ -205,6 +205,11 @@ def test_fivetran_with_snowflake_dest(pytestconfig, tmp_path):
"postgres", "postgres",
] ]
}, },
"destination_patterns": {
"allow": [
"interval_unconstitutional",
]
},
"sources_to_database": { "sources_to_database": {
"calendar_elected": "postgres_db", "calendar_elected": "postgres_db",
}, },
@ -291,6 +296,11 @@ def test_fivetran_with_snowflake_dest_and_null_connector_user(pytestconfig, tmp_
"postgres", "postgres",
] ]
}, },
"destination_patterns": {
"allow": [
"interval_unconstitutional",
]
},
"sources_to_database": { "sources_to_database": {
"calendar_elected": "postgres_db", "calendar_elected": "postgres_db",
}, },