yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-28 18:38:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix bug

Browse Source
This commit is contained in:
Jonny Dixon 2025-06-11 14:59:09 +01:00
parent e1f26096d3
commit e7bb418988
3 changed files with 2 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
View File

@ -380,16 +380,6 @@ class FivetranSource(StatefulIngestionSourceBase):
)
return
# Add table-level lineage if we're working with the tables
fine_grained_lineage.append(
FineGrainedLineage(
upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
upstreams=[str(source_urn)],
downstreamType=FineGrainedLineageDownstreamType.FIELD,
downstreams=[str(dest_urn)],
)
)
logger.info(f"Creating column lineage from {source_urn} to {dest_urn}")
# Extract destination platform from the URN
@ -1131,7 +1121,6 @@ class FivetranSource(StatefulIngestionSourceBase):
# Process job history
if len(connector.jobs) >= MAX_JOBS_PER_CONNECTOR:
self._report_lineage_truncation(connector)
self.report.warning(
title="Not all sync history was captured",
message=f"The connector had more than {MAX_JOBS_PER_CONNECTOR} sync runs in the past {self.config.history_sync_lookback_period} days. "

36
metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json
View File

@ -163,17 +163,6 @@
],
"inputDatajobs": [],
"fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
@ -196,17 +185,6 @@
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
@ -805,19 +783,7 @@
"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.confluent_cloud.my-destination-topic,PROD)"
],
"inputDatajobs": [],
"fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:dataset:(urn:li:dataPlatform:kafka,kafka_prod.my-source-topic,PROD)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.confluent_cloud.my-destination-topic,PROD)"
],
"confidenceScore": 1.0
}
]
"fineGrainedLineages": []
}
},
"systemMetadata": {

36
metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json
View File

@ -146,17 +146,6 @@
],
"inputDatajobs": [],
"fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
@ -179,17 +168,6 @@
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
@ -779,19 +757,7 @@
"urn:li:dataset:(urn:li:dataPlatform:kafka,kafka_prod.my-destination-topic,PROD)"
],
"inputDatajobs": [],
"fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:dataset:(urn:li:dataPlatform:kafka,kafka_prod.my-source-topic,PROD)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:dataset:(urn:li:dataPlatform:kafka,kafka_prod.my-destination-topic,PROD)"
],
"confidenceScore": 1.0
}
]
"fineGrainedLineages": []
}
},
"systemMetadata": {