yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-30 12:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/snowflake): avoid overfetching schemas from datahub (#10527)

Browse Source
This commit is contained in:
Harshal Sheth 2024-06-06 15:54:32 -07:00 committed by GitHub
parent 78ab7acab6
commit e62e4b18c0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
View File

@ -254,7 +254,18 @@ class SnowflakeV2Source(
platform=self.platform,
platform_instance=self.config.platform_instance,
env=self.config.env,
graph=self.ctx.graph,
graph=(
# If we're ingestion schema metadata for tables/views, then we will populate
# schemas into the resolver as we go. We only need to do a bulk fetch
# if we're not ingesting schema metadata as part of ingestion.
self.ctx.graph
if not (
self.config.include_technical_schema
and self.config.include_tables
and self.config.include_views
)
else None
),
generate_usage_statistics=False,
generate_operations=False,
format_queries=self.config.format_sql_queries,
@ -1252,7 +1263,7 @@ class SnowflakeV2Source(
foreignKeys=foreign_keys,
)
if self.aggregator and self.config.parse_view_ddl:
if self.aggregator:
self.aggregator.register_schema(urn=dataset_urn, schema=schema_metadata)
return schema_metadata